diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index c4ee846ba564f..776b1ab944f69 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -62,7 +62,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.23", "8.14.3", "8.15.0", "8.16.0"] + BWC_VERSION: ["7.17.23", "8.14.4", "8.15.0", "8.16.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 982c1f69856c0..e9c743885d78d 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -577,8 +577,8 @@ steps: env: BWC_VERSION: 8.13.4 - - label: "{{matrix.image}} / 8.14.3 / packaging-tests-upgrade" - command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.3 + - label: "{{matrix.image}} / 8.14.4 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.4 timeout_in_minutes: 300 matrix: setup: @@ -592,7 +592,7 @@ steps: buildDirectory: /dev/shm/bk diskSizeGb: 250 env: - BWC_VERSION: 8.14.3 + BWC_VERSION: 8.14.4 - label: "{{matrix.image}} / 8.15.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.0 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 5bc33433bbc72..f908b946bb523 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -642,8 +642,8 @@ steps: - signal_reason: agent_stop limit: 3 - - label: 8.14.3 / bwc - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.3#bwcTest + - label: 8.14.4 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.4#bwcTest timeout_in_minutes: 300 agents: provider: gcp @@ -653,7 +653,7 @@ steps: preemptible: true diskSizeGb: 250 env: - BWC_VERSION: 8.14.3 + BWC_VERSION: 8.14.4 retry: automatic: - exit_status: "-1" @@ -771,7 +771,7 @@ steps: setup: ES_RUNTIME_JAVA: - openjdk17 - BWC_VERSION: ["7.17.23", "8.14.3", "8.15.0", "8.16.0"] + BWC_VERSION: ["7.17.23", "8.14.4", "8.15.0", "8.16.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 @@ -821,7 +821,7 @@ steps: - openjdk21 - openjdk22 - openjdk23 - BWC_VERSION: ["7.17.23", "8.14.3", "8.15.0", "8.16.0"] + BWC_VERSION: ["7.17.23", "8.14.4", "8.15.0", "8.16.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 9de7dbfa2a5c2..776be80e0d291 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -31,6 +31,6 @@ BWC_VERSION: - "8.11.4" - "8.12.2" - "8.13.4" - - "8.14.3" + - "8.14.4" - "8.15.0" - "8.16.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index 90a3dcba977c8..f5f7f7a7d4ecb 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,5 +1,5 @@ BWC_VERSION: - "7.17.23" - - "8.14.3" + - "8.14.4" - "8.15.0" - "8.16.0" diff --git a/README.asciidoc b/README.asciidoc index dc27735d3c015..fa479d9c76340 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -1,6 +1,6 @@ = Elasticsearch -Elasticsearch is a distributed search and analytics engine optimized for speed and relevance on production-scale workloads. Elasticsearch is the foundation of Elastic's open Stack platform. Search in near real-time over massive datasets, perform vector searches, integrate with generative AI applications, and much more. +Elasticsearch is a distributed search and analytics engine, scalable data store and vector database optimized for speed and relevance on production-scale workloads. Elasticsearch is the foundation of Elastic's open Stack platform. Search in near real-time over massive datasets, perform vector searches, integrate with generative AI applications, and much more. Use cases enabled by Elasticsearch include: diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java index 4bb33937579c2..2185c6d1df611 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java @@ -41,6 +41,7 @@ import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator; import org.elasticsearch.compute.operator.topn.TopNOperator; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; @@ -189,6 +190,11 @@ public String indexName() { return "benchmark"; } + @Override + public IndexSettings indexSettings() { + throw new UnsupportedOperationException(); + } + @Override public MappedFieldType.FieldExtractPreference fieldExtractPreference() { return MappedFieldType.FieldExtractPreference.NONE; diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties index 515ab9d5f1822..efe2ff3449216 100644 --- a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties +++ b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=f8b4f4772d302c8ff580bc40d0f56e715de69b163546944f787c87abf209c961 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-all.zip +distributionSha256Sum=258e722ec21e955201e31447b0aed14201765a3bfbae296a46cf60b70e66db70 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy index e454d2ee38fff..658e2623cbbd7 100644 --- a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy +++ b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy @@ -13,6 +13,7 @@ import org.elasticsearch.gradle.OS import org.elasticsearch.gradle.internal.test.AntFixture import org.gradle.api.file.FileSystemOperations import org.gradle.api.file.ProjectLayout +import org.gradle.api.provider.ProviderFactory import org.gradle.api.tasks.Internal import org.gradle.process.ExecOperations @@ -24,14 +25,17 @@ abstract class AntFixtureStop extends LoggedExec implements FixtureStop { AntFixture fixture @Inject - AntFixtureStop(ProjectLayout projectLayout, ExecOperations execOperations, FileSystemOperations fileSystemOperations) { - super(projectLayout, execOperations, fileSystemOperations) + AntFixtureStop(ProjectLayout projectLayout, + ExecOperations execOperations, + FileSystemOperations fileSystemOperations, + ProviderFactory providerFactory) { + super(projectLayout, execOperations, fileSystemOperations, providerFactory) } void setFixture(AntFixture fixture) { assert this.fixture == null this.fixture = fixture; - final Object pid = "${ -> this.fixture.pid }" + final Object pid = "${-> this.fixture.pid}" onlyIf("pidFile exists") { fixture.pidFile.exists() } doFirst { logger.info("Shutting down ${fixture.name} with pid ${pid}") diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java index 7010ed92d4c57..4112d96c7296b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java @@ -26,7 +26,6 @@ import org.gradle.api.tasks.TaskProvider; import org.gradle.jvm.toolchain.JavaLanguageVersion; import org.gradle.jvm.toolchain.JavaToolchainService; -import org.gradle.jvm.toolchain.JvmVendorSpec; import java.io.File; import java.io.IOException; @@ -161,10 +160,8 @@ private static TaskProvider createRunBwcGradleTask( /** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */ private static Provider getJavaHome(ObjectFactory objectFactory, JavaToolchainService toolChainService, final int version) { Property value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version)); - return toolChainService.launcherFor(javaToolchainSpec -> { - javaToolchainSpec.getLanguageVersion().value(value); - javaToolchainSpec.getVendor().set(JvmVendorSpec.ORACLE); - }).map(launcher -> launcher.getMetadata().getInstallationPath().getAsFile().getAbsolutePath()); + return toolChainService.launcherFor(javaToolchainSpec -> { javaToolchainSpec.getLanguageVersion().value(value); }) + .map(launcher -> launcher.getMetadata().getInstallationPath().getAsFile().getAbsolutePath()); } private static String readFromFile(File file) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java index 4f9498c8f33a6..b513fd7b93631 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java @@ -8,7 +8,7 @@ package org.elasticsearch.gradle.internal; -import com.gradle.scan.plugin.BuildScanExtension; +import com.gradle.develocity.agent.gradle.DevelocityConfiguration; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; @@ -64,7 +64,7 @@ public void apply(Project target) { File targetFile = target.file("build/" + buildNumber + ".tar.bz2"); File projectDir = target.getProjectDir(); File gradleWorkersDir = new File(target.getGradle().getGradleUserHomeDir(), "workers/"); - BuildScanExtension extension = target.getExtensions().getByType(BuildScanExtension.class); + DevelocityConfiguration extension = target.getExtensions().getByType(DevelocityConfiguration.class); File daemonsLogDir = new File(target.getGradle().getGradleUserHomeDir(), "daemon/" + target.getGradle().getGradleVersion()); getFlowScope().always(BuildFinishedFlowAction.class, spec -> { @@ -125,7 +125,7 @@ interface Parameters extends FlowParameters { ListProperty getFilteredFiles(); @Input - Property getBuildScan(); + Property getBuildScan(); } @@ -198,7 +198,7 @@ public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNo + System.getenv("BUILDKITE_JOB_ID") + "/artifacts/" + artifactUuid; - parameters.getBuildScan().get().link("Artifact Upload", targetLink); + parameters.getBuildScan().get().getBuildScan().link("Artifact Upload", targetLink); } } catch (Exception e) { System.out.println("Failed to upload buildkite artifact " + e.getMessage()); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java index 13f265388fe3f..a4412cd3db247 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java @@ -59,7 +59,6 @@ public class InternalDistributionModuleCheckTaskProvider { "org.elasticsearch.plugin", "org.elasticsearch.plugin.analysis", "org.elasticsearch.pluginclassloader", - "org.elasticsearch.preallocate", "org.elasticsearch.securesm", "org.elasticsearch.server", "org.elasticsearch.simdvec", diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java index 42834928bafed..e6059c729e8c8 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java @@ -48,9 +48,9 @@ import java.nio.file.Files; import java.time.ZoneOffset; import java.time.ZonedDateTime; -import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.Optional; import java.util.Random; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -97,24 +97,25 @@ public void apply(Project project) { JavaVersion minimumCompilerVersion = JavaVersion.toVersion(getResourceContents("/minimumCompilerVersion")); JavaVersion minimumRuntimeVersion = JavaVersion.toVersion(getResourceContents("/minimumRuntimeVersion")); - File runtimeJavaHome = findRuntimeJavaHome(); - boolean isRuntimeJavaHomeSet = Jvm.current().getJavaHome().equals(runtimeJavaHome) == false; + Optional selectedRuntimeJavaHome = findRuntimeJavaHome(); + File actualRuntimeJavaHome = selectedRuntimeJavaHome.orElse(Jvm.current().getJavaHome()); + boolean isRuntimeJavaHomeSet = selectedRuntimeJavaHome.isPresent(); GitInfo gitInfo = GitInfo.gitInfo(project.getRootDir()); BuildParams.init(params -> { params.reset(); - params.setRuntimeJavaHome(runtimeJavaHome); + params.setRuntimeJavaHome(actualRuntimeJavaHome); params.setJavaToolChainSpec(resolveToolchainSpecFromEnv()); params.setRuntimeJavaVersion( determineJavaVersion( "runtime java.home", - runtimeJavaHome, + actualRuntimeJavaHome, isRuntimeJavaHomeSet ? minimumRuntimeVersion : Jvm.current().getJavaVersion() ) ); params.setIsRuntimeJavaHomeSet(isRuntimeJavaHomeSet); - JvmInstallationMetadata runtimeJdkMetaData = metadataDetector.getMetadata(getJavaInstallation(runtimeJavaHome)); + JvmInstallationMetadata runtimeJdkMetaData = metadataDetector.getMetadata(getJavaInstallation(actualRuntimeJavaHome)); params.setRuntimeJavaDetails(formatJavaVendorDetails(runtimeJdkMetaData)); params.setJavaVersions(getAvailableJavaVersions()); params.setMinimumCompilerVersion(minimumCompilerVersion); @@ -298,49 +299,19 @@ private static void assertMinimumCompilerVersion(JavaVersion minimumCompilerVers } } - private File findRuntimeJavaHome() { + private Optional findRuntimeJavaHome() { String runtimeJavaProperty = System.getProperty("runtime.java"); if (runtimeJavaProperty != null) { - return resolveJavaHomeFromToolChainService(runtimeJavaProperty); + return Optional.of(resolveJavaHomeFromToolChainService(runtimeJavaProperty)); } String env = System.getenv("RUNTIME_JAVA_HOME"); if (env != null) { - return new File(env); + return Optional.of(new File(env)); } // fall back to tool chain if set. env = System.getenv("JAVA_TOOLCHAIN_HOME"); - return env == null ? Jvm.current().getJavaHome() : new File(env); - } - - @NotNull - private String resolveJavaHomeFromEnvVariable(String javaHomeEnvVar) { - Provider javaHomeNames = providers.gradleProperty("org.gradle.java.installations.fromEnv"); - // Provide a useful error if we're looking for a Java home version that we haven't told Gradle about yet - Arrays.stream(javaHomeNames.get().split(",")) - .filter(s -> s.equals(javaHomeEnvVar)) - .findFirst() - .orElseThrow( - () -> new GradleException( - "Environment variable '" - + javaHomeEnvVar - + "' is not registered with Gradle installation supplier. Ensure 'org.gradle.java.installations.fromEnv' is " - + "updated in gradle.properties file." - ) - ); - String versionedJavaHome = System.getenv(javaHomeEnvVar); - if (versionedJavaHome == null) { - final String exceptionMessage = String.format( - Locale.ROOT, - "$%s must be set to build Elasticsearch. " - + "Note that if the variable was just set you " - + "might have to run `./gradlew --stop` for " - + "it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details.", - javaHomeEnvVar - ); - throw new GradleException(exceptionMessage); - } - return versionedJavaHome; + return env == null ? Optional.empty() : Optional.of(new File(env)); } @NotNull @@ -348,15 +319,10 @@ private File resolveJavaHomeFromToolChainService(String version) { Property value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version)); Provider javaLauncherProvider = toolChainService.launcherFor(javaToolchainSpec -> { javaToolchainSpec.getLanguageVersion().value(value); - javaToolchainSpec.getVendor().set(JvmVendorSpec.ORACLE); }); return javaLauncherProvider.get().getMetadata().getInstallationPath().getAsFile(); } - private static String getJavaHomeEnvVarName(String version) { - return "JAVA" + version + "_HOME"; - } - public static String getResourceContents(String resourcePath) { try ( BufferedReader reader = new BufferedReader(new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath))) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java index 4263ef2b1f76f..489cff65976b1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java @@ -193,6 +193,11 @@ public Set getMissingClassExcludes() { @SkipWhenEmpty public abstract ConfigurableFileCollection getJarsToScan(); + @Classpath + public FileCollection getClasspath() { + return classpath; + } + @TaskAction public void runThirdPartyAudit() throws IOException { Set jars = getJarsToScan().getFiles(); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolver.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolver.java index b8cffae0189ce..913a15517f0af 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolver.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolver.java @@ -23,9 +23,12 @@ import java.util.Map; import java.util.Optional; +/** + * Resolves released Oracle JDKs that are EOL. + */ public abstract class ArchivedOracleJdkToolchainResolver extends AbstractCustomJavaToolchainResolver { - private static final Map ARCHIVED_BASE_VERSIONS = Maps.of(20, "20.0.2", 19, "19.0.2", 18, "18.0.2.1", 17, "17.0.7"); + private static final Map ARCHIVED_BASE_VERSIONS = Maps.of(20, "20.0.2", 19, "19.0.2", 18, "18.0.2.1"); @Override public Optional resolve(JavaToolchainRequest request) { diff --git a/build-tools-internal/src/main/resources/changelog-schema.json b/build-tools-internal/src/main/resources/changelog-schema.json index a38eb32062146..d8fc7d780ae58 100644 --- a/build-tools-internal/src/main/resources/changelog-schema.json +++ b/build-tools-internal/src/main/resources/changelog-schema.json @@ -277,6 +277,7 @@ "compatibilityChangeArea": { "type": "string", "enum": [ + "Analysis", "Authorization", "Cluster and node setting", "Command line tool", diff --git a/build-tools-internal/src/main/resources/minimumGradleVersion b/build-tools-internal/src/main/resources/minimumGradleVersion index 83ea3179ddacc..f7b1c8ff61774 100644 --- a/build-tools-internal/src/main/resources/minimumGradleVersion +++ b/build-tools-internal/src/main/resources/minimumGradleVersion @@ -1 +1 @@ -8.8 \ No newline at end of file +8.9 \ No newline at end of file diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolverSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolverSpec.groovy index b7f08b6016679..dd6e7b324e745 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolverSpec.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolverSpec.groovy @@ -40,12 +40,6 @@ class ArchivedOracleJdkToolchainResolverSpec extends AbstractToolchainResolverSp [18, ORACLE, LINUX, X86_64, "https://download.oracle.com/java/18/archive/jdk-18.0.2.1_linux-x64_bin.tar.gz"], [18, ORACLE, LINUX, AARCH64, "https://download.oracle.com/java/18/archive/jdk-18.0.2.1_linux-aarch64_bin.tar.gz"], [18, ORACLE, WINDOWS, X86_64, "https://download.oracle.com/java/18/archive/jdk-18.0.2.1_windows-x64_bin.zip"], - - [17, ORACLE, MAC_OS, X86_64, "https://download.oracle.com/java/17/archive/jdk-17.0.7_macos-x64_bin.tar.gz"], - [17, ORACLE, MAC_OS, AARCH64, "https://download.oracle.com/java/17/archive/jdk-17.0.7_macos-aarch64_bin.tar.gz"], - [17, ORACLE, LINUX, X86_64, "https://download.oracle.com/java/17/archive/jdk-17.0.7_linux-x64_bin.tar.gz"], - [17, ORACLE, LINUX, AARCH64, "https://download.oracle.com/java/17/archive/jdk-17.0.7_linux-aarch64_bin.tar.gz"], - [17, ORACLE, WINDOWS, X86_64, "https://download.oracle.com/java/17/archive/jdk-17.0.7_windows-x64_bin.zip"] ] } diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java index 2bc4aa1a1be36..d4747c9a6c38e 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java @@ -14,6 +14,7 @@ import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Plugin; import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.dsl.DependencyHandler; import org.gradle.api.artifacts.repositories.IvyArtifactRepository; import org.gradle.api.artifacts.type.ArtifactTypeDefinition; @@ -42,8 +43,10 @@ public class DistributionDownloadPlugin implements Plugin { private static final String FAKE_SNAPSHOT_IVY_GROUP = "elasticsearch-distribution-snapshot"; private static final String DOWNLOAD_REPO_NAME = "elasticsearch-downloads"; private static final String SNAPSHOT_REPO_NAME = "elasticsearch-snapshots"; - public static final String DISTRO_EXTRACTED_CONFIG_PREFIX = "es_distro_extracted_"; - public static final String DISTRO_CONFIG_PREFIX = "es_distro_file_"; + + public static final String ES_DISTRO_CONFIG_PREFIX = "es_distro_"; + public static final String DISTRO_EXTRACTED_CONFIG_PREFIX = ES_DISTRO_CONFIG_PREFIX + "extracted_"; + public static final String DISTRO_CONFIG_PREFIX = ES_DISTRO_CONFIG_PREFIX + "file_"; private final ObjectFactory objectFactory; private NamedDomainObjectContainer distributionsContainer; @@ -51,6 +54,8 @@ public class DistributionDownloadPlugin implements Plugin { private Property dockerAvailability; + private boolean writingDependencies = false; + @Inject public DistributionDownloadPlugin(ObjectFactory objectFactory) { this.objectFactory = objectFactory; @@ -63,6 +68,7 @@ public void setDockerAvailability(Provider dockerAvailability) { @Override public void apply(Project project) { + writingDependencies = project.getGradle().getStartParameter().getWriteDependencyVerifications().isEmpty() == false; project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> { transformSpec.getFrom().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.ZIP_TYPE); transformSpec.getTo().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE); @@ -85,7 +91,6 @@ private void setupDistributionContainer(Project project) { var extractedConfiguration = project.getConfigurations().create(DISTRO_EXTRACTED_CONFIG_PREFIX + name); extractedConfiguration.getAttributes() .attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE); - var distribution = new ElasticsearchDistribution( name, objectFactory, @@ -94,16 +99,20 @@ private void setupDistributionContainer(Project project) { objectFactory.fileCollection().from(extractedConfiguration) ); - registerDistributionDependencies(project, distribution); + // when running with --write-dependency-verification to update dependency verification data, + // we do not register the dependencies as we ignore elasticsearch internal dependencies anyhow and + // want to reduce general resolution time + if (writingDependencies == false) { + registerDistributionDependencies(project, distribution); + } return distribution; }); project.getExtensions().add(CONTAINER_NAME, distributionsContainer); } private void registerDistributionDependencies(Project project, ElasticsearchDistribution distribution) { - project.getConfigurations() - .getByName(DISTRO_CONFIG_PREFIX + distribution.getName()) - .getDependencies() + Configuration distroConfig = project.getConfigurations().getByName(DISTRO_CONFIG_PREFIX + distribution.getName()); + distroConfig.getDependencies() .addLater( project.provider(() -> distribution.maybeFreeze()) .map( @@ -112,9 +121,9 @@ private void registerDistributionDependencies(Project project, ElasticsearchDist ) ); - project.getConfigurations() - .getByName(DISTRO_EXTRACTED_CONFIG_PREFIX + distribution.getName()) - .getDependencies() + Configuration extractedDistroConfig = project.getConfigurations() + .getByName(DISTRO_EXTRACTED_CONFIG_PREFIX + distribution.getName()); + extractedDistroConfig.getDependencies() .addAllLater( project.provider(() -> distribution.maybeFreeze()) .map( diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java b/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java index 6087482db278d..3a425d11ccf17 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java @@ -17,6 +17,8 @@ import org.gradle.api.provider.ListProperty; import org.gradle.api.provider.MapProperty; import org.gradle.api.provider.Property; +import org.gradle.api.provider.Provider; +import org.gradle.api.provider.ProviderFactory; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.Optional; @@ -92,17 +94,45 @@ public abstract class LoggedExec extends DefaultTask implements FileSystemOperat private String output; @Inject - public LoggedExec(ProjectLayout projectLayout, ExecOperations execOperations, FileSystemOperations fileSystemOperations) { + public LoggedExec( + ProjectLayout projectLayout, + ExecOperations execOperations, + FileSystemOperations fileSystemOperations, + ProviderFactory providerFactory + ) { this.projectLayout = projectLayout; this.execOperations = execOperations; this.fileSystemOperations = fileSystemOperations; getWorkingDir().convention(projectLayout.getProjectDirectory().getAsFile()); // For now mimic default behaviour of Gradle Exec task here - getEnvironment().putAll(System.getenv()); + setupDefaultEnvironment(providerFactory); getCaptureOutput().convention(false); getSpoolOutput().convention(false); } + /** + * We explicitly configure the environment variables that are passed to the executed process. + * This is required to make sure that the build cache and Gradle configuration cache is correctly configured + * can be reused across different build invocations. + * */ + private void setupDefaultEnvironment(ProviderFactory providerFactory) { + getEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("BUILDKITE")); + getEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("GRADLE_BUILD_CACHE")); + getEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("VAULT")); + Provider javaToolchainHome = providerFactory.environmentVariable("JAVA_TOOLCHAIN_HOME"); + if (javaToolchainHome.isPresent()) { + getEnvironment().put("JAVA_TOOLCHAIN_HOME", javaToolchainHome); + } + Provider javaRuntimeHome = providerFactory.environmentVariable("RUNTIME_JAVA_HOME"); + if (javaRuntimeHome.isPresent()) { + getEnvironment().put("RUNTIME_JAVA_HOME", javaRuntimeHome); + } + Provider path = providerFactory.environmentVariable("PATH"); + if (path.isPresent()) { + getEnvironment().put("PATH", path); + } + } + @TaskAction public void run() { boolean spoolOutput = getSpoolOutput().get(); diff --git a/build.gradle b/build.gradle index 3869d21b49bfe..01fdace570ce0 100644 --- a/build.gradle +++ b/build.gradle @@ -19,6 +19,7 @@ import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.util.GradleUtils import org.gradle.plugins.ide.eclipse.model.AccessRule import org.gradle.plugins.ide.eclipse.model.ProjectDependency +import org.elasticsearch.gradle.DistributionDownloadPlugin import java.nio.file.Files @@ -284,11 +285,16 @@ allprojects { } tasks.register('resolveAllDependencies', ResolveAllDependencies) { - configs = project.configurations + def ignoredPrefixes = [DistributionDownloadPlugin.ES_DISTRO_CONFIG_PREFIX, "jdbcDriver"] + configs = project.configurations.matching { config -> ignoredPrefixes.any { config.name.startsWith(it) } == false } resolveJavaToolChain = true if (project.path.contains("fixture")) { dependsOn tasks.withType(ComposePull) } + if (project.path.contains(":distribution:docker")) { + enabled = false + } + } plugins.withId('lifecycle-base') { diff --git a/distribution/build.gradle b/distribution/build.gradle index 77f1a2d032c73..47367ab0261a2 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -280,8 +280,6 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { dependencies { libs project(':server') - // this is a special circumstance of a jar that is not a dependency of server, but needs to be in the module path - libs project(':libs:elasticsearch-preallocate') libsVersionChecker project(':distribution:tools:java-version-checker') libsCliLauncher project(':distribution:tools:cli-launcher') diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java index 298b4671582b5..2a89f18209d11 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java @@ -69,11 +69,6 @@ static List systemJvmOptions(Settings nodeSettings, final Map> token filter before and after the -`edge_ngram` filter to achieve the same results. +deprecated:[8.16.0, use <> token filter before and after `edge_ngram` for same results]. +Indicates whether to truncate tokens from the `front` or `back`. Defaults to `front`. -- [[analysis-edgengram-tokenfilter-customize]] diff --git a/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc index 3efb8f6de9b3e..e37118019a55c 100644 --- a/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc @@ -85,45 +85,45 @@ Additional settings are: <> search analyzers to pick up changes to synonym files. Only to be used for search analyzers. * `expand` (defaults to `true`). -* `lenient` (defaults to `false`). If `true` ignores exceptions while parsing the synonym configuration. It is important -to note that only those synonym rules which cannot get parsed are ignored. For instance consider the following request: - -[source,console] --------------------------------------------------- -PUT /test_index -{ - "settings": { - "index": { - "analysis": { - "analyzer": { - "synonym": { - "tokenizer": "standard", - "filter": [ "my_stop", "synonym_graph" ] - } - }, - "filter": { - "my_stop": { - "type": "stop", - "stopwords": [ "bar" ] - }, - "synonym_graph": { - "type": "synonym_graph", - "lenient": true, - "synonyms": [ "foo, bar => baz" ] - } - } - } - } - } -} --------------------------------------------------- +Expands definitions for equivalent synonym rules. +See <>. +* `lenient` (defaults to `false`). +If `true` ignores errors while parsing the synonym configuration. +It is important to note that only those synonym rules which cannot get parsed are ignored. +See <> for an example of `lenient` behaviour for invalid synonym rules. + +[discrete] +[[synonym-graph-tokenizer-expand-equivalent-synonyms]] +===== `expand` equivalent synonym rules + +The `expand` parameter controls whether to expand equivalent synonym rules. +Consider a synonym defined like: + +`foo, bar, baz` + +Using `expand: true`, the synonym rule would be expanded into: -With the above request the word `bar` gets skipped but a mapping `foo => baz` is still added. However, if the mapping -being added was `foo, baz => bar` nothing would get added to the synonym list. This is because the target word for the -mapping is itself eliminated because it was a stop word. Similarly, if the mapping was "bar, foo, baz" and `expand` was -set to `false` no mapping would get added as when `expand=false` the target mapping is the first word. However, if -`expand=true` then the mappings added would be equivalent to `foo, baz => foo, baz` i.e, all mappings other than the -stop word. +``` +foo => foo +foo => bar +foo => baz +bar => foo +bar => bar +bar => baz +baz => foo +baz => bar +baz => baz +``` + +When `expand` is set to `false`, the synonym rule is not expanded and the first synonym is treated as the canonical representation. The synonym would be equivalent to: + +``` +foo => foo +bar => foo +baz => foo +``` + +The `expand` parameter does not affect explicit synonym rules, like `foo, bar => baz`. [discrete] [[synonym-graph-tokenizer-ignore_case-deprecated]] @@ -160,12 +160,65 @@ Text will be processed first through filters preceding the synonym filter before {es} will also use the token filters preceding the synonym filter in a tokenizer chain to parse the entries in a synonym file or synonym set. In the above example, the synonyms graph token filter is placed after a stemmer. The stemmer will also be applied to the synonym entries. -The synonym rules should not contain words that are removed by a filter that appears later in the chain (like a `stop` filter). -Removing a term from a synonym rule means there will be no matching for it at query time. - Because entries in the synonym map cannot have stacked positions, some token filters may cause issues here. Token filters that produce multiple versions of a token may choose which version of the token to emit when parsing synonyms. For example, `asciifolding` will only produce the folded version of the token. Others, like `multiplexer`, `word_delimiter_graph` or `ngram` will throw an error. If you need to build analyzers that include both multi-token filters and synonym filters, consider using the <> filter, with the multi-token filters in one branch and the synonym filter in the other. + +[discrete] +[[synonym-graph-tokenizer-stop-token-filter]] +===== Synonyms and `stop` token filters + +Synonyms and <> interact with each other in the following ways: + +[discrete] +====== Stop token filter *before* synonym token filter + +Stop words will be removed from the synonym rule definition. +This can can cause errors on the synonym rule. + +[WARNING] +==== +Invalid synonym rules can cause errors when applying analyzer changes. +For reloadable analyzers, this prevents reloading and applying changes. +You must correct errors in the synonym rules and reload the analyzer. + +An index with invalid synonym rules cannot be reopened, making it inoperable when: + +* A node containing the index starts +* The index is opened from a closed state +* A node restart occurs (which reopens the node assigned shards) +==== + +For *explicit synonym rules* like `foo, bar => baz` with a stop filter that removes `bar`: + +- If `lenient` is set to `false`, an error will be raised as `bar` would be removed from the left hand side of the synonym rule. +- If `lenient` is set to `true`, the rule `foo => baz` will be added and `bar => baz` will be ignored. + +If the stop filter removed `baz` instead: + +- If `lenient` is set to `false`, an error will be raised as `baz` would be removed from the right hand side of the synonym rule. +- If `lenient` is set to `true`, the synonym will have no effect as the target word is removed. + +For *equivalent synonym rules* like `foo, bar, baz` and `expand: true, with a stop filter that removes `bar`: + +- If `lenient` is set to `false`, an error will be raised as `bar` would be removed from the synonym rule. +- If `lenient` is set to `true`, the synonyms added would be equivalent to the following synonym rules, which do not contain the removed word: + +``` +foo => foo +foo => baz +baz => foo +baz => baz +``` + +[discrete] +====== Stop token filter *after* synonym token filter + +The stop filter will remove the terms from the resulting synonym expansion. + +For example, a synonym rule like `foo, bar => baz` and a stop filter that removes `baz` will get no matches for `foo` or `bar`, as both would get expanded to `baz` which is removed by the stop filter. + +If the stop filter removed `foo` instead, then searching for `foo` would get expanded to `baz`, which is not removed by the stop filter thus potentially providing matches for `baz`. diff --git a/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc index 046cd297b5092..1658f016db60b 100644 --- a/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc @@ -73,47 +73,45 @@ Additional settings are: <> search analyzers to pick up changes to synonym files. Only to be used for search analyzers. * `expand` (defaults to `true`). -* `lenient` (defaults to `false`). If `true` ignores exceptions while parsing the synonym configuration. It is important -to note that only those synonym rules which cannot get parsed are ignored. For instance consider the following request: - - -[source,console] --------------------------------------------------- -PUT /test_index -{ - "settings": { - "index": { - "analysis": { - "analyzer": { - "synonym": { - "tokenizer": "standard", - "filter": [ "my_stop", "synonym" ] - } - }, - "filter": { - "my_stop": { - "type": "stop", - "stopwords": [ "bar" ] - }, - "synonym": { - "type": "synonym", - "lenient": true, - "synonyms": [ "foo, bar => baz" ] - } - } - } - } - } -} --------------------------------------------------- +Expands definitions for equivalent synonym rules. +See <>. +* `lenient` (defaults to `false`). +If `true` ignores errors while parsing the synonym configuration. +It is important to note that only those synonym rules which cannot get parsed are ignored. +See <> for an example of `lenient` behaviour for invalid synonym rules. + +[discrete] +[[synonym-tokenizer-expand-equivalent-synonyms]] +===== `expand` equivalent synonym rules + +The `expand` parameter controls whether to expand equivalent synonym rules. +Consider a synonym defined like: + +`foo, bar, baz` + +Using `expand: true`, the synonym rule would be expanded into: -With the above request the word `bar` gets skipped but a mapping `foo => baz` is still added. However, if the mapping -being added was `foo, baz => bar` nothing would get added to the synonym list. This is because the target word for the -mapping is itself eliminated because it was a stop word. Similarly, if the mapping was "bar, foo, baz" and `expand` was -set to `false` no mapping would get added as when `expand=false` the target mapping is the first word. However, if -`expand=true` then the mappings added would be equivalent to `foo, baz => foo, baz` i.e, all mappings other than the -stop word. +``` +foo => foo +foo => bar +foo => baz +bar => foo +bar => bar +bar => baz +baz => foo +baz => bar +baz => baz +``` +When `expand` is set to `false`, the synonym rule is not expanded and the first synonym is treated as the canonical representation. The synonym would be equivalent to: + +``` +foo => foo +bar => foo +baz => foo +``` + +The `expand` parameter does not affect explicit synonym rules, like `foo, bar => baz`. [discrete] [[synonym-tokenizer-ignore_case-deprecated]] @@ -135,7 +133,7 @@ To apply synonyms, you will need to include a synonym token filters into an anal "my_analyzer": { "type": "custom", "tokenizer": "standard", - "filter": ["stemmer", "synonym_graph"] + "filter": ["stemmer", "synonym"] } } ---- @@ -148,10 +146,7 @@ Order is important for your token filters. Text will be processed first through filters preceding the synonym filter before being processed by the synonym filter. {es} will also use the token filters preceding the synonym filter in a tokenizer chain to parse the entries in a synonym file or synonym set. -In the above example, the synonyms graph token filter is placed after a stemmer. The stemmer will also be applied to the synonym entries. - -The synonym rules should not contain words that are removed by a filter that appears later in the chain (like a `stop` filter). -Removing a term from a synonym rule means there will be no matching for it at query time. +In the above example, the synonyms token filter is placed after a stemmer. The stemmer will also be applied to the synonym entries. Because entries in the synonym map cannot have stacked positions, some token filters may cause issues here. Token filters that produce multiple versions of a token may choose which version of the token to emit when parsing synonyms. @@ -159,3 +154,59 @@ For example, `asciifolding` will only produce the folded version of the token. Others, like `multiplexer`, `word_delimiter_graph` or `ngram` will throw an error. If you need to build analyzers that include both multi-token filters and synonym filters, consider using the <> filter, with the multi-token filters in one branch and the synonym filter in the other. + +[discrete] +[[synonym-tokenizer-stop-token-filter]] +===== Synonyms and `stop` token filters + +Synonyms and <> interact with each other in the following ways: + +[discrete] +====== Stop token filter *before* synonym token filter + +Stop words will be removed from the synonym rule definition. +This can can cause errors on the synonym rule. + +[WARNING] +==== +Invalid synonym rules can cause errors when applying analyzer changes. +For reloadable analyzers, this prevents reloading and applying changes. +You must correct errors in the synonym rules and reload the analyzer. + +An index with invalid synonym rules cannot be reopened, making it inoperable when: + +* A node containing the index starts +* The index is opened from a closed state +* A node restart occurs (which reopens the node assigned shards) +==== + +For *explicit synonym rules* like `foo, bar => baz` with a stop filter that removes `bar`: + +- If `lenient` is set to `false`, an error will be raised as `bar` would be removed from the left hand side of the synonym rule. +- If `lenient` is set to `true`, the rule `foo => baz` will be added and `bar => baz` will be ignored. + +If the stop filter removed `baz` instead: + +- If `lenient` is set to `false`, an error will be raised as `baz` would be removed from the right hand side of the synonym rule. +- If `lenient` is set to `true`, the synonym will have no effect as the target word is removed. + +For *equivalent synonym rules* like `foo, bar, baz` and `expand: true, with a stop filter that removes `bar`: + +- If `lenient` is set to `false`, an error will be raised as `bar` would be removed from the synonym rule. +- If `lenient` is set to `true`, the synonyms added would be equivalent to the following synonym rules, which do not contain the removed word: + +``` +foo => foo +foo => baz +baz => foo +baz => baz +``` + +[discrete] +====== Stop token filter *after* synonym token filter + +The stop filter will remove the terms from the resulting synonym expansion. + +For example, a synonym rule like `foo, bar => baz` and a stop filter that removes `baz` will get no matches for `foo` or `bar`, as both would get expanded to `baz` which is removed by the stop filter. + +If the stop filter removed `foo` instead, then searching for `foo` would get expanded to `baz`, which is not removed by the stop filter thus potentially providing matches for `baz`. diff --git a/docs/reference/analysis/tokenfilters/synonyms-format.asciidoc b/docs/reference/analysis/tokenfilters/synonyms-format.asciidoc index 63dd72dade8d0..e780c24963312 100644 --- a/docs/reference/analysis/tokenfilters/synonyms-format.asciidoc +++ b/docs/reference/analysis/tokenfilters/synonyms-format.asciidoc @@ -15,7 +15,7 @@ This format uses two different definitions: ipod, i-pod, i pod computer, pc, laptop ---- -* Explicit mappings: Matches a group of words to other words. Words on the left hand side of the rule definition are expanded into all the possibilities described on the right hand side. Example: +* Explicit synonyms: Matches a group of words to other words. Words on the left hand side of the rule definition are expanded into all the possibilities described on the right hand side. Example: + [source,synonyms] ---- diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc index 084ff471367ce..f188a5f2ddf04 100644 --- a/docs/reference/cluster/nodes-stats.asciidoc +++ b/docs/reference/cluster/nodes-stats.asciidoc @@ -808,6 +808,14 @@ This is not shown for the `shards` level, since mappings may be shared across th `total_estimated_overhead_in_bytes`:: (integer) Estimated heap overhead, in bytes, of mappings on this node, which allows for 1kiB of heap for every mapped field. +`total_segments`:: +(integer) Estimated number of Lucene segments on this node + +`total_segment_fields`:: +(integer) Estimated number of fields at the segment level on this node + +`average_fields_per_segment`:: +(integer) Estimated average number of fields per segment on this node ======= `dense_vector`:: diff --git a/docs/reference/connector/apis/claim-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/claim-connector-sync-job-api.asciidoc new file mode 100644 index 0000000000000..2fb28f9e9fb37 --- /dev/null +++ b/docs/reference/connector/apis/claim-connector-sync-job-api.asciidoc @@ -0,0 +1,68 @@ +[[claim-connector-sync-job-api]] +=== Claim connector sync job API +++++ +Claim connector sync job +++++ + +preview::[] + +Claims a connector sync job. + +The `_claim` endpoint is not intended for direct connector management by users. It is there to support the implementation of services that utilize the https://github.com/elastic/connectors/blob/main/docs/CONNECTOR_PROTOCOL.md[Connector Protocol] to communicate with {es}. + +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + +[[claim-connector-sync-job-api-request]] +==== {api-request-title} +`PUT _connector/_sync_job//_claim` + +[[claim-connector-sync-job-api-prereqs]] +==== {api-prereq-title} + +* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* The `connector_sync_job_id` parameter should reference an existing connector sync job. + +[[claim-connector-sync-job-api-desc]] +==== {api-description-title} + +Claims a connector sync job. This action updates the job's status to `in_progress` and sets the `last_seen` and `started_at` timestamps to the current time. Additionally, it can set the `sync_cursor` property for the sync job. + +[[claim-connector-sync-job-api-path-params]] +==== {api-path-parms-title} + +`connector_sync_job_id`:: +(Required, string) + +[role="child_attributes"] +[[claim-connector-sync-job-api-request-body]] +==== {api-request-body-title} + +`worker_hostname`:: +(Required, string) The host name of the current system that will execute the job. + +`sync_cursor`:: +(Optional, Object) The cursor object from the last incremental sync job. This should reference the `sync_cursor` field in the connector state for which the job is executed. + + +[[claim-connector-sync-job-api-response-codes]] +==== {api-response-codes-title} + +`200`:: +Connector sync job was successfully claimed. + +`404`:: +No connector sync job matching `connector_sync_job_id` could be found. + +[[claim-connector-sync-job-api-example]] +==== {api-examples-title} + +The following example claims the connector sync job with ID `my-connector-sync-job-id`: + +[source,console] +---- +PUT _connector/_sync_job/my-connector-sync-job-id/_claim +{ + "worker_hostname": "some-machine" +} +---- +// TEST[skip:there's no way to clean up after creating a connector sync job, as we don't know the id ahead of time. Therefore, skip this test.] diff --git a/docs/reference/connector/apis/connector-apis.asciidoc b/docs/reference/connector/apis/connector-apis.asciidoc index 41186ff6326f2..987f82f6b4ce4 100644 --- a/docs/reference/connector/apis/connector-apis.asciidoc +++ b/docs/reference/connector/apis/connector-apis.asciidoc @@ -108,6 +108,8 @@ preview:[] * <> preview:[] +* <> +preview:[] * <> preview:[] * <> @@ -141,5 +143,6 @@ include::update-connector-last-sync-api.asciidoc[] include::update-connector-status-api.asciidoc[] include::check-in-connector-sync-job-api.asciidoc[] +include::claim-connector-sync-job-api.asciidoc[] include::set-connector-sync-job-error-api.asciidoc[] include::set-connector-sync-job-stats-api.asciidoc[] diff --git a/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc b/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc index 217b29451937d..730dad852adee 100644 --- a/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc +++ b/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc @@ -31,7 +31,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec (Optional, integer) The offset from the first result to fetch. Defaults to `0`. `status`:: -(Optional, job status) A comma-separated list of job statuses to filter the results. Available statuses include: `canceling`, `canceled`, `completed`, `error`, `in_progress`, `pending`, `suspended`. +(Optional, job status) A job status to filter the results for. Available statuses include: `canceling`, `canceled`, `completed`, `error`, `in_progress`, `pending`, `suspended`. `connector_id`:: (Optional, string) The connector id the fetched sync jobs need to have. diff --git a/docs/reference/data-streams/data-streams.asciidoc b/docs/reference/data-streams/data-streams.asciidoc index 9c7137563caef..1484e21febdb3 100644 --- a/docs/reference/data-streams/data-streams.asciidoc +++ b/docs/reference/data-streams/data-streams.asciidoc @@ -157,4 +157,5 @@ include::set-up-a-data-stream.asciidoc[] include::use-a-data-stream.asciidoc[] include::change-mappings-and-settings.asciidoc[] include::tsds.asciidoc[] +include::logs.asciidoc[] include::lifecycle/index.asciidoc[] diff --git a/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc index 6bd157071f54e..7d33a5b5f880c 100644 --- a/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc @@ -54,7 +54,7 @@ duration the document could be deleted. When empty, every document in this data `enabled`:: (Optional, boolean) -If defined, it turns data streqm lifecycle on/off (`true`/`false`) for this data stream. +If defined, it turns data stream lifecycle on/off (`true`/`false`) for this data stream. A data stream lifecycle that's disabled (`enabled: false`) will have no effect on the data stream. Defaults to `true`. diff --git a/docs/reference/data-streams/logs.asciidoc b/docs/reference/data-streams/logs.asciidoc new file mode 100644 index 0000000000000..e870289bcf7be --- /dev/null +++ b/docs/reference/data-streams/logs.asciidoc @@ -0,0 +1,52 @@ +[[logs-data-stream]] +== Logs data stream + +preview::[Logs data streams and the logsdb index mode are in tech preview and may be changed or removed in the future. Don't use logs data streams or logsdb index mode in production.] + +A logs data stream is a data stream type that stores log data more efficiently. + +In benchmarks, log data stored in a logs data stream used ~2.5 times less disk space than a regular data +stream. The exact impact will vary depending on your data set. + +The following features are enabled in a logs data stream: + +* <>, which omits storing the `_source` field. When the document source is requested, it is synthesized from document fields upon retrieval. + +* Index sorting. This yields a lower storage footprint. By default indices are sorted by `host.name` and `@timestamp` fields at index time. + +* More space efficient compression for fields with <> enabled. + +[discrete] +[[how-to-use-logsds]] +=== Create a logs data stream + +To create a logs data stream, set your index template `index.mode` to `logsdb`: + +[source,console] +---- +PUT _index_template/my-index-template +{ + "index_patterns": ["logs-*"], + "data_stream": { }, + "template": { + "settings": { + "index.mode": "logsdb" <1> + } + }, + "priority": 101 <2> +} +---- +// TEST + +<1> The index mode setting. +<2> The index template priority. By default, Elasticsearch ships with an index template with a `logs-*-*` pattern with a priority of 100. You need to define a priority higher than 100 to ensure that this index template gets selected over the default index template for the `logs-*-*` pattern. See the <> for more information. + +After the index template is created, new indices that use the template will be configured as a logs data stream. You can start indexing data and <>. + +//// +[source,console] +---- +DELETE _index_template/my-index-template +---- +// TEST[continued] +//// diff --git a/docs/reference/datatiers.asciidoc b/docs/reference/datatiers.asciidoc index 0981e80804383..c37f54b5c9cae 100644 --- a/docs/reference/datatiers.asciidoc +++ b/docs/reference/datatiers.asciidoc @@ -2,13 +2,24 @@ [[data-tiers]] == Data tiers -A _data tier_ is a collection of nodes with the same data role that -typically share the same hardware profile: +A _data tier_ is a collection of <> within a cluster that share the same +<>, and a hardware profile that's appropriately sized for the role. Elastic recommends that nodes in the same tier share the same +hardware profile to avoid <>. -* <> nodes handle the indexing and query load for content such as a product catalog. -* <> nodes handle the indexing load for time series data such as logs or metrics -and hold your most recent, most-frequently-accessed data. -* <> nodes hold time series data that is accessed less-frequently +The data tiers that you use, and the way that you use them, depends on the data's <>. + +The following data tiers are can be used with each data category: + +Content data: + +* <> nodes handle the indexing and query load for non-timeseries +indices, such as a product catalog. + +Time series data: + +* <> nodes handle the indexing load for time series data, +such as logs or metrics. They hold your most recent, most-frequently-accessed data. +* <> nodes hold time series data that is accessed less-frequently and rarely needs to be updated. * <> nodes hold time series data that is accessed infrequently and not normally updated. To save space, you can keep @@ -16,29 +27,40 @@ infrequently and not normally updated. To save space, you can keep <> on the cold tier. These fully mounted indices eliminate the need for replicas, reducing required disk space by approximately 50% compared to the regular indices. -* <> nodes hold time series data that is accessed +* <> nodes hold time series data that is accessed rarely and never updated. The frozen tier stores <> of <> exclusively. This extends the storage capacity even further — by up to 20 times compared to the warm tier. -TIP: The performance of an {es} node is often limited by the performance of the underlying storage. +TIP: The performance of an {es} node is often limited by the performance of the underlying storage and hardware profile. +For example hardware profiles, refer to Elastic Cloud's {cloud}/ec-reference-hardware.html[instance configurations]. Review our recommendations for optimizing your storage for <> and <>. IMPORTANT: {es} generally expects nodes within a data tier to share the same hardware profile. Variations not following this recommendation should be carefully architected to avoid <>. -When you index documents directly to a specific index, they remain on content tier nodes indefinitely. +The way data tiers are used often depends on the data's category: + +- Content data remains on the <> for its entire +data lifecycle. -When you index documents to a data stream, they initially reside on hot tier nodes. -You can configure <> ({ilm-init}) policies -to automatically transition your time series data through the hot, warm, and cold tiers -according to your performance, resiliency and data retention requirements. +- Time series data may progress through the +descending temperature data tiers (hot, warm, cold, and frozen) according to your +performance, resiliency, and data retention requirements. ++ +You can automate these lifecycle transitions using the <>, or custom <>. + +[discrete] +[[available-tier]] +=== Available data tiers + +Learn more about each data tier, including when and how it should be used. [discrete] [[content-tier]] -=== Content tier +==== Content tier // tag::content-tier[] Data stored in the content tier is generally a collection of items such as a product catalog or article archive. @@ -53,13 +75,14 @@ While they are also responsible for indexing, content data is generally not inge as time series data such as logs and metrics. From a resiliency perspective the indices in this tier should be configured to use one or more replicas. -The content tier is required. System indices and other indices that aren't part -of a data stream are automatically allocated to the content tier. +The content tier is required and is often deployed within the same node +grouping as the hot tier. System indices and other indices that aren't part +of a data stream are automatically allocated to the content tier. // end::content-tier[] [discrete] [[hot-tier]] -=== Hot tier +==== Hot tier // tag::hot-tier[] The hot tier is the {es} entry point for time series data and holds your most-recent, @@ -74,7 +97,7 @@ data stream>> are automatically allocated to the hot tier. [discrete] [[warm-tier]] -=== Warm tier +==== Warm tier // tag::warm-tier[] Time series data can move to the warm tier once it is being queried less frequently @@ -87,7 +110,7 @@ For resiliency, indices in the warm tier should be configured to use one or more [discrete] [[cold-tier]] -=== Cold tier +==== Cold tier // tag::cold-tier[] When you no longer need to search time series data regularly, it can move from @@ -109,7 +132,7 @@ but doesn't reduce required disk space compared to the warm tier. [discrete] [[frozen-tier]] -=== Frozen tier +==== Frozen tier // tag::frozen-tier[] Once data is no longer being queried, or being queried rarely, it may move from @@ -123,9 +146,15 @@ sometimes fetch frozen data from the snapshot repository, searches on the frozen tier are typically slower than on the cold tier. // end::frozen-tier[] +[discrete] +[[configure-data-tiers]] +=== Configure data tiers + +Follow the instructions for your deployment type to configure data tiers. + [discrete] [[configure-data-tiers-cloud]] -=== Configure data tiers on {ess} or {ece} +==== {ess} or {ece} The default configuration for an {ecloud} deployment includes a shared tier for hot and content data. This tier is required and can't be removed. @@ -159,7 +188,7 @@ tier]. [discrete] [[configure-data-tiers-on-premise]] -=== Configure data tiers for self-managed deployments +==== Self-managed deployments For self-managed deployments, each node's <> is configured in `elasticsearch.yml`. For example, the highest-performance nodes in a cluster @@ -177,25 +206,59 @@ tier. [[data-tier-allocation]] === Data tier index allocation -When you create an index, by default {es} sets -<> +The <> setting determines which tier the index should be allocated to. + +When you create an index, by default {es} sets the `_tier_preference` to `data_content` to automatically allocate the index shards to the content tier. When {es} creates an index as part of a <>, -by default {es} sets -<> +by default {es} sets the `_tier_preference` to `data_hot` to automatically allocate the index shards to the hot tier. -You can explicitly set `index.routing.allocation.include._tier_preference` -to opt out of the default tier-based allocation. +At the time of index creation, you can override the default setting by explicitly setting +the preferred value in one of two ways: + +- Using an <>. Refer to <> for details. +- Within the <> request body. + +You can override this +setting after index creation by <> to the preferred +value. + +This setting also accepts multiple tiers in order of preference. This prevents indices from remaining unallocated if no nodes are available in the preferred tier. For example, when {ilm} migrates an index to the cold phase, it sets the index `_tier_preference` to `data_cold,data_warm,data_hot`. + +To remove the data tier preference +setting, set the `_tier_preference` value to `null`. This allows the index to allocate to any data node within the cluster. Setting the `_tier_preference` to `null` does not restore the default value. Note that, in the case of managed indices, a <> action might apply a new value in its place. + +[discrete] +[[data-tier-allocation-value]] +==== Determine the current data tier preference + +You can check an existing index's data tier preference by <> for `index.routing.allocation.include._tier_preference`: + +[source,console] +-------------------------------------------------- +GET /my-index-000001/_settings?filter_path=*.settings.index.routing.allocation.include._tier_preference +-------------------------------------------------- +// TEST[setup:my_index] + +[discrete] +[[data-tier-allocation-troubleshooting]] +==== Troubleshooting + +The `_tier_preference` setting might conflict with other allocation settings. This conflict might prevent the shard from allocating. A conflict might occur when a cluster has not yet been completely <>. + +This setting will not unallocate a currently allocated shard, but might prevent it from migrating from its current location to its designated data tier. To troubleshoot, call the <> and specify the suspected problematic shard. [discrete] [[data-tier-migration]] -=== Automatic data tier migration +==== Automatic data tier migration {ilm-init} automatically transitions managed indices through the available data tiers using the <> action. By default, this action is automatically injected in every phase. -You can explicitly specify the migrate action with `"enabled": false` to disable automatic migration, +You can explicitly specify the migrate action with `"enabled": false` to <>, for example, if you're using the <> to manually specify allocation rules. diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc index 9d359fd7d7f02..ccc8e67f39bc0 100644 --- a/docs/reference/docs/index_.asciidoc +++ b/docs/reference/docs/index_.asciidoc @@ -211,7 +211,7 @@ creates a dynamic mapping. By default, new fields and objects are automatically added to the mapping if needed. For more information about field mapping, see <> and the <> API. -Automatic index creation is controlled by the `action.auto_create_index` +Automatic index creation is controlled by the <> setting. This setting defaults to `true`, which allows any index to be created automatically. You can modify this setting to explicitly allow or block automatic creation of indices that match specified patterns, or set it to diff --git a/docs/reference/esql/esql-across-clusters.asciidoc b/docs/reference/esql/esql-across-clusters.asciidoc index 6231b4f4f0a69..8bc1e2a83fc19 100644 --- a/docs/reference/esql/esql-across-clusters.asciidoc +++ b/docs/reference/esql/esql-across-clusters.asciidoc @@ -8,6 +8,11 @@ preview::["{ccs-cap} for {esql} is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] +[NOTE] +==== +For {ccs-cap} with {esql} on version 8.16 or later, remote clusters must also be on version 8.16 or later. +==== + With {esql}, you can execute a single query across multiple clusters. [discrete] @@ -64,7 +69,7 @@ You will need to: * Create an API key on the *remote cluster* using the <> API or using the {kibana-ref}/api-keys.html[Kibana API keys UI]. * Add the API key to the keystore on the *local cluster*, as part of the steps in <>. All cross-cluster requests from the local cluster are bound by the API key’s privileges. -Using {esql} with the API key based security model requires some additional permissions that may not be needed when using the traditional query DSL based search. +Using {esql} with the API key based security model requires some additional permissions that may not be needed when using the traditional query DSL based search. The following example API call creates a role that can query remote indices using {esql} when using the API key based security model. [source,console] @@ -73,11 +78,11 @@ POST /_security/role/remote1 { "cluster": ["cross_cluster_search"], <1> "indices": [ - { + { "names" : [""], <2> "privileges": ["read"] } - ], + ], "remote_indices": [ <3> { "names": [ "logs-*" ], @@ -93,7 +98,7 @@ POST /_security/role/remote1 <3> The indices allowed read access to the remote cluster. The configured <> must also allow this index to be read. <4> The `read_cross_cluster` privilege is always required when using {esql} across clusters with the API key based security model. <5> The remote clusters to which these privileges apply. -This remote cluster must be configured with a <> and connected to the remote cluster before the remote index can be queried. +This remote cluster must be configured with a <> and connected to the remote cluster before the remote index can be queried. Verify connection using the <> API. You will then need a user or API key with the permissions you created above. The following example API call creates a user with the `remote1` role. diff --git a/docs/reference/esql/esql-commands.asciidoc b/docs/reference/esql/esql-commands.asciidoc index bed79299b1cc1..235113ac1394a 100644 --- a/docs/reference/esql/esql-commands.asciidoc +++ b/docs/reference/esql/esql-commands.asciidoc @@ -37,6 +37,9 @@ image::images/esql/processing-command.svg[A processing command changing an input * <> * <> * <> +ifeval::["{release-state}"=="unreleased"] +* experimental:[] <> +endif::[] * <> * <> ifeval::["{release-state}"=="unreleased"] @@ -59,6 +62,9 @@ include::processing-commands/drop.asciidoc[] include::processing-commands/enrich.asciidoc[] include::processing-commands/eval.asciidoc[] include::processing-commands/grok.asciidoc[] +ifeval::["{release-state}"=="unreleased"] +include::processing-commands/inlinestats.asciidoc[] +endif::[] include::processing-commands/keep.asciidoc[] include::processing-commands/limit.asciidoc[] ifeval::["{release-state}"=="unreleased"] diff --git a/docs/reference/esql/esql-limitations.asciidoc b/docs/reference/esql/esql-limitations.asciidoc index 11e3fd7ae9883..8accc7550edbb 100644 --- a/docs/reference/esql/esql-limitations.asciidoc +++ b/docs/reference/esql/esql-limitations.asciidoc @@ -85,6 +85,11 @@ Some <> are not supported in all contexts: ** `cartesian_point` ** `cartesian_shape` +In addition, when <>, +it's possible for the same field to be mapped to multiple types. +These fields cannot be directly used in queries or returned in results, +unless they're <>. + [discrete] [[esql-_source-availability]] === _source availability diff --git a/docs/reference/esql/esql-multi-index.asciidoc b/docs/reference/esql/esql-multi-index.asciidoc new file mode 100644 index 0000000000000..41ff6a27417b1 --- /dev/null +++ b/docs/reference/esql/esql-multi-index.asciidoc @@ -0,0 +1,175 @@ +[[esql-multi-index]] +=== Using {esql} to query multiple indices +++++ +Using {esql} to query multiple indices +++++ + +With {esql}, you can execute a single query across multiple indices, data streams, or aliases. +To do so, use wildcards and date arithmetic. The following example uses a comma-separated list and a wildcard: + +[source,esql] +---- +FROM employees-00001,other-employees-* +---- + +Use the format `:` to <>: + +[source,esql] +---- +FROM cluster_one:employees-00001,cluster_two:other-employees-* +---- + +[discrete] +[[esql-multi-index-invalid-mapping]] +=== Field type mismatches + +When querying multiple indices, data streams, or aliases, you might find that the same field is mapped to multiple different types. +For example, consider the two indices with the following field mappings: + +*index: events_ip* +``` +{ + "mappings": { + "properties": { + "@timestamp": { "type": "date" }, + "client_ip": { "type": "ip" }, + "event_duration": { "type": "long" }, + "message": { "type": "keyword" } + } + } +} +``` + +*index: events_keyword* +``` +{ + "mappings": { + "properties": { + "@timestamp": { "type": "date" }, + "client_ip": { "type": "keyword" }, + "event_duration": { "type": "long" }, + "message": { "type": "keyword" } + } + } +} +``` + +When you query each of these individually with a simple query like `FROM events_ip`, the results are provided with type-specific columns: + +[source.merge.styled,esql] +---- +FROM events_ip +| SORT @timestamp DESC +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +@timestamp:date | client_ip:ip | event_duration:long | message:keyword +2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +|=== + +Note how the `client_ip` column is correctly identified as type `ip`, and all values are displayed. +However, if instead the query sources two conflicting indices with `FROM events_*`, the type of the `client_ip` column cannot be determined +and is reported as `unsupported` with all values returned as `null`. + +[[query-unsupported]] +[source.merge.styled,esql] +---- +FROM events_* +| SORT @timestamp DESC +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +@timestamp:date | client_ip:unsupported | event_duration:long | message:keyword +2023-10-23T13:55:01.543Z | null | 1756467 | Connected to 10.1.0.1 +2023-10-23T13:53:55.832Z | null | 5033755 | Connection error +2023-10-23T13:52:55.015Z | null | 8268153 | Connection error +2023-10-23T13:51:54.732Z | null | 725448 | Connection error +2023-10-23T13:33:34.937Z | null | 1232382 | Disconnected +2023-10-23T12:27:28.948Z | null | 2764889 | Connected to 10.1.0.2 +2023-10-23T12:15:03.360Z | null | 3450233 | Connected to 10.1.0.3 +|=== + +In addition, if the query refers to this unsupported field directly, the query fails: + +[source.merge.styled,esql] +---- +FROM events_* +| KEEP @timestamp, client_ip, event_duration, message +| SORT @timestamp DESC +---- + +[source,bash] +---- +Cannot use field [client_ip] due to ambiguities being mapped as +[2] incompatible types: + [ip] in [events_ip], + [keyword] in [events_keyword] +---- + +[discrete] +[[esql-multi-index-union-types]] +=== Union types + +{esql} has a way to handle <>. When the same field is mapped to multiple types in multiple indices, +the type of the field is understood to be a _union_ of the various types in the index mappings. +As seen in the preceding examples, this _union type_ cannot be used in the results, +and cannot be referred to by the query +-- except when it's passed to a type conversion function that accepts all the types in the _union_ and converts the field +to a single type. {esql} offers a suite of <> to achieve this. + +In the above examples, the query can use a command like `EVAL client_ip = TO_IP(client_ip)` to resolve +the union of `ip` and `keyword` to just `ip`. +You can also use the type-conversion syntax `EVAL client_ip = client_ip::IP`. +Alternatively, the query could use <> to convert all supported types into `KEYWORD`. + +For example, the <> that returned `client_ip:unsupported` with `null` values can be improved using the `TO_IP` function or the equivalent `field::ip` syntax. +These changes also resolve the error message. +As long as the only reference to the original field is to pass it to a conversion function that resolves the type ambiguity, no error results. + +[source.merge.styled,esql] +---- +FROM events_* +| EVAL client_ip = TO_IP(client_ip) +| KEEP @timestamp, client_ip, event_duration, message +| SORT @timestamp DESC +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +@timestamp:date | client_ip:ip | event_duration:long | message:keyword +2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +|=== + +[discrete] +[[esql-multi-index-index-metadata]] +=== Index metadata + +It can be helpful to know the particular index from which each row is sourced. +To get this information, use the <> option on the <> command. + +[source.merge.styled,esql] +---- +FROM events_* METADATA _index +| EVAL client_ip = TO_IP(client_ip) +| KEEP _index, @timestamp, client_ip, event_duration, message +| SORT @timestamp DESC +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +_index:keyword | @timestamp:date | client_ip:ip | event_duration:long | message:keyword +events_ip | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +events_ip | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +events_ip | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +events_keyword | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +events_keyword | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +events_keyword | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +events_keyword | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +|=== diff --git a/docs/reference/esql/esql-query-api.asciidoc b/docs/reference/esql/esql-query-api.asciidoc index 2cdd97ceab176..e8cfa03e3ee88 100644 --- a/docs/reference/esql/esql-query-api.asciidoc +++ b/docs/reference/esql/esql-query-api.asciidoc @@ -75,6 +75,11 @@ For syntax, refer to <>. (Optional, array) Values for parameters in the `query`. For syntax, refer to <>. +`profile`:: +(Optional, boolean) If provided and `true` the response will include an extra `profile` object +with information about how the query was executed. It provides insight into the performance +of each part of the query. This is for human debugging as the object's format might change at any time. + `query`:: (Required, string) {esql} query to run. For syntax, refer to <>. @@ -100,3 +105,8 @@ returned if `drop_null_columns` is sent with the request. `rows`:: (array of arrays) Values for the search results. + +`profile`:: +(object) +Profile describing the execution of the query. Only returned if `profile` was sent in the body. +The object itself is for human debugging and can change at any time. diff --git a/docs/reference/esql/esql-rest.asciidoc b/docs/reference/esql/esql-rest.asciidoc index 5b90e96d7a734..2c8c5e81e273d 100644 --- a/docs/reference/esql/esql-rest.asciidoc +++ b/docs/reference/esql/esql-rest.asciidoc @@ -278,6 +278,47 @@ POST /_query ---- // TEST[setup:library] +The parameters can be named parameters or positional parameters. + +Named parameters use question mark placeholders (`?`) followed by a string. + +[source,console] +---- +POST /_query +{ + "query": """ + FROM library + | EVAL year = DATE_EXTRACT("year", release_date) + | WHERE page_count > ?page_count AND author == ?author + | STATS count = COUNT(*) by year + | WHERE count > ?count + | LIMIT 5 + """, + "params": [{"page_count" : 300}, {"author" : "Frank Herbert"}, {"count" : 0}] +} +---- +// TEST[setup:library] + +Positional parameters use question mark placeholders (`?`) followed by an +integer. + +[source,console] +---- +POST /_query +{ + "query": """ + FROM library + | EVAL year = DATE_EXTRACT("year", release_date) + | WHERE page_count > ?1 AND author == ?2 + | STATS count = COUNT(*) by year + | WHERE count > ?3 + | LIMIT 5 + """, + "params": [300, "Frank Herbert", 0] +} +---- +// TEST[setup:library] + [discrete] [[esql-rest-async-query]] ==== Running an async {esql} query diff --git a/docs/reference/esql/esql-using.asciidoc b/docs/reference/esql/esql-using.asciidoc index 3e045163069ec..d2e18bf1b91a3 100644 --- a/docs/reference/esql/esql-using.asciidoc +++ b/docs/reference/esql/esql-using.asciidoc @@ -12,6 +12,9 @@ and set up alerts. Using {esql} in {elastic-sec} to investigate events in Timeline, create detection rules, and build {esql} queries using Elastic AI Assistant. +<>:: +Using {esql} to query multiple indexes and resolve field type mismatches. + <>:: Using {esql} to query across multiple clusters. @@ -21,5 +24,6 @@ Using the <> to list and cancel {esql} queries. include::esql-rest.asciidoc[] include::esql-kibana.asciidoc[] include::esql-security-solution.asciidoc[] +include::esql-multi-index.asciidoc[] include::esql-across-clusters.asciidoc[] include::task-management.asciidoc[] diff --git a/docs/reference/esql/functions/aggregation-functions.asciidoc b/docs/reference/esql/functions/aggregation-functions.asciidoc index 82931b84fd44a..fb840687427df 100644 --- a/docs/reference/esql/functions/aggregation-functions.asciidoc +++ b/docs/reference/esql/functions/aggregation-functions.asciidoc @@ -15,9 +15,9 @@ The <> command supports these aggregate functions: * <> * <> * <> -* <> +* <> * experimental:[] <> -* <> +* <> * <> * <> * experimental:[] <> @@ -27,12 +27,12 @@ include::count.asciidoc[] include::count-distinct.asciidoc[] include::median.asciidoc[] include::median-absolute-deviation.asciidoc[] -include::percentile.asciidoc[] include::st_centroid_agg.asciidoc[] -include::sum.asciidoc[] include::layout/avg.asciidoc[] include::layout/max.asciidoc[] include::layout/min.asciidoc[] +include::layout/percentile.asciidoc[] +include::layout/sum.asciidoc[] include::layout/top.asciidoc[] include::values.asciidoc[] include::weighted-avg.asciidoc[] diff --git a/docs/reference/esql/functions/appendix/percentile.asciidoc b/docs/reference/esql/functions/appendix/percentile.asciidoc new file mode 100644 index 0000000000000..c4800a3fb0e1e --- /dev/null +++ b/docs/reference/esql/functions/appendix/percentile.asciidoc @@ -0,0 +1,14 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-percentile-approximate]] +==== `PERCENTILE` is (usually) approximate + +include::../../../aggregations/metrics/percentile-aggregation.asciidoc[tag=approximate] + +[WARNING] +==== +`PERCENTILE` is also {wikipedia}/Nondeterministic_algorithm[non-deterministic]. +This means you can get slightly different results using the same data. +==== + diff --git a/docs/reference/esql/functions/case.asciidoc b/docs/reference/esql/functions/case.asciidoc deleted file mode 100644 index b5fda636135b2..0000000000000 --- a/docs/reference/esql/functions/case.asciidoc +++ /dev/null @@ -1,70 +0,0 @@ -[discrete] -[[esql-case]] -=== `CASE` - -*Syntax* - -[source,esql] ----- -CASE(condition1, value1[, ..., conditionN, valueN][, default_value]) ----- - -*Parameters* - -`conditionX`:: -A condition. - -`valueX`:: -The value that's returned when the corresponding condition is the first to -evaluate to `true`. - -`default_value`:: -The default value that's is returned when no condition matches. - -*Description* - -Accepts pairs of conditions and values. The function returns the value that -belongs to the first condition that evaluates to `true`. - -If the number of arguments is odd, the last argument is the default value which -is returned when no condition matches. If the number of arguments is even, and -no condition matches, the function returns `null`. - -*Example* - -Determine whether employees are monolingual, bilingual, or polyglot: - -[source,esql] -[source.merge.styled,esql] ----- -include::{esql-specs}/docs.csv-spec[tag=case] ----- -[%header.monospaced.styled,format=dsv,separator=|] -|=== -include::{esql-specs}/docs.csv-spec[tag=case-result] -|=== - -Calculate the total connection success rate based on log messages: - -[source,esql] -[source.merge.styled,esql] ----- -include::{esql-specs}/conditional.csv-spec[tag=docsCaseSuccessRate] ----- -[%header.monospaced.styled,format=dsv,separator=|] -|=== -include::{esql-specs}/conditional.csv-spec[tag=docsCaseSuccessRate-result] -|=== - -Calculate an hourly error rate as a percentage of the total number of log -messages: - -[source,esql] -[source.merge.styled,esql] ----- -include::{esql-specs}/conditional.csv-spec[tag=docsCaseHourlyErrorRate] ----- -[%header.monospaced.styled,format=dsv,separator=|] -|=== -include::{esql-specs}/conditional.csv-spec[tag=docsCaseHourlyErrorRate-result] -|=== diff --git a/docs/reference/esql/functions/count.asciidoc b/docs/reference/esql/functions/count.asciidoc index 38732336413ad..66cfe76350cdd 100644 --- a/docs/reference/esql/functions/count.asciidoc +++ b/docs/reference/esql/functions/count.asciidoc @@ -56,3 +56,28 @@ include::{esql-specs}/stats.csv-spec[tag=docsCountWithExpression] |=== include::{esql-specs}/stats.csv-spec[tag=docsCountWithExpression-result] |=== + +[[esql-agg-count-or-null]] +To count the number of times an expression returns `TRUE` use +a <> command to remove rows that shouldn't be included: + +[source.merge.styled,esql] +---- +include::{esql-specs}/stats.csv-spec[tag=count-where] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/stats.csv-spec[tag=count-where-result] +|=== + +To count the same stream of data based on two different expressions +use the pattern `COUNT( OR NULL)`: + +[source.merge.styled,esql] +---- +include::{esql-specs}/stats.csv-spec[tag=count-or-null] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/stats.csv-spec[tag=count-or-null-result] +|=== diff --git a/docs/reference/esql/functions/description/exp.asciidoc b/docs/reference/esql/functions/description/exp.asciidoc new file mode 100644 index 0000000000000..116bfbece34ba --- /dev/null +++ b/docs/reference/esql/functions/description/exp.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Returns the value of e raised to the power of the given number. diff --git a/docs/reference/esql/functions/description/locate.asciidoc b/docs/reference/esql/functions/description/locate.asciidoc index 60a6d435e37b6..e5a6fba512432 100644 --- a/docs/reference/esql/functions/description/locate.asciidoc +++ b/docs/reference/esql/functions/description/locate.asciidoc @@ -2,4 +2,4 @@ *Description* -Returns an integer that indicates the position of a keyword substring within another string +Returns an integer that indicates the position of a keyword substring within another string. diff --git a/docs/reference/esql/functions/description/percentile.asciidoc b/docs/reference/esql/functions/description/percentile.asciidoc new file mode 100644 index 0000000000000..9ef823577ddee --- /dev/null +++ b/docs/reference/esql/functions/description/percentile.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Returns the value at which a certain percentage of observed values occur. For example, the 95th percentile is the value which is greater than 95% of the observed values and the 50th percentile is the `MEDIAN`. diff --git a/docs/reference/esql/functions/description/substring.asciidoc b/docs/reference/esql/functions/description/substring.asciidoc index edb97b219bbe0..3d8091f26c04d 100644 --- a/docs/reference/esql/functions/description/substring.asciidoc +++ b/docs/reference/esql/functions/description/substring.asciidoc @@ -2,4 +2,4 @@ *Description* -Returns a substring of a string, specified by a start position and an optional length +Returns a substring of a string, specified by a start position and an optional length. diff --git a/docs/reference/esql/functions/description/sum.asciidoc b/docs/reference/esql/functions/description/sum.asciidoc new file mode 100644 index 0000000000000..e3956567b8656 --- /dev/null +++ b/docs/reference/esql/functions/description/sum.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +The sum of a numeric expression. diff --git a/docs/reference/esql/functions/examples/exp.asciidoc b/docs/reference/esql/functions/examples/exp.asciidoc new file mode 100644 index 0000000000000..bd088d98571c1 --- /dev/null +++ b/docs/reference/esql/functions/examples/exp.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/math.csv-spec[tag=exp] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/math.csv-spec[tag=exp-result] +|=== + diff --git a/docs/reference/esql/functions/examples/percentile.asciidoc b/docs/reference/esql/functions/examples/percentile.asciidoc new file mode 100644 index 0000000000000..51cb22b1622b8 --- /dev/null +++ b/docs/reference/esql/functions/examples/percentile.asciidoc @@ -0,0 +1,22 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Examples* + +[source.merge.styled,esql] +---- +include::{esql-specs}/stats_percentile.csv-spec[tag=percentile] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/stats_percentile.csv-spec[tag=percentile-result] +|=== +The expression can use inline functions. For example, to calculate a percentile of the maximum values of a multivalued column, first use `MV_MAX` to get the maximum value per row, and use the result with the `PERCENTILE` function +[source.merge.styled,esql] +---- +include::{esql-specs}/stats_percentile.csv-spec[tag=docsStatsPercentileNestedExpression] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/stats_percentile.csv-spec[tag=docsStatsPercentileNestedExpression-result] +|=== + diff --git a/docs/reference/esql/functions/sum.asciidoc b/docs/reference/esql/functions/examples/sum.asciidoc similarity index 62% rename from docs/reference/esql/functions/sum.asciidoc rename to docs/reference/esql/functions/examples/sum.asciidoc index efe65d5503ec6..1c02ccd784a54 100644 --- a/docs/reference/esql/functions/sum.asciidoc +++ b/docs/reference/esql/functions/examples/sum.asciidoc @@ -1,22 +1,6 @@ -[discrete] -[[esql-agg-sum]] -=== `SUM` +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. -*Syntax* - -[source,esql] ----- -SUM(expression) ----- - -`expression`:: -Numeric expression. - -*Description* - -Returns the sum of a numeric expression. - -*Example* +*Examples* [source.merge.styled,esql] ---- @@ -26,11 +10,7 @@ include::{esql-specs}/stats.csv-spec[tag=sum] |=== include::{esql-specs}/stats.csv-spec[tag=sum-result] |=== - -The expression can use inline functions. For example, to calculate -the sum of each employee's maximum salary changes, apply the -`MV_MAX` function to each row and then sum the results: - +The expression can use inline functions. For example, to calculate the sum of each employee's maximum salary changes, apply the `MV_MAX` function to each row and then sum the results [source.merge.styled,esql] ---- include::{esql-specs}/stats.csv-spec[tag=docsStatsSumNestedExpression] @@ -39,3 +19,4 @@ include::{esql-specs}/stats.csv-spec[tag=docsStatsSumNestedExpression] |=== include::{esql-specs}/stats.csv-spec[tag=docsStatsSumNestedExpression-result] |=== + diff --git a/docs/reference/esql/functions/kibana/definition/add.json b/docs/reference/esql/functions/kibana/definition/add.json new file mode 100644 index 0000000000000..e8f971fdf0671 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/add.json @@ -0,0 +1,296 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "add", + "description" : "Add two numbers together. If either field is <> then the result is `null`.", + "signatures" : [ + { + "params" : [ + { + "name" : "lhs", + "type" : "date_period", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "date_period", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "date_period" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_period", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "datetime", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "datetime" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "datetime", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "date_period", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "datetime" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "datetime", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "time_duration", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "datetime" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "integer" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "time_duration", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "datetime", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "datetime" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "time_duration", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "time_duration", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "time_duration" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "unsigned_long" + } + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/div.json b/docs/reference/esql/functions/kibana/definition/div.json new file mode 100644 index 0000000000000..0e49da7bdac61 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/div.json @@ -0,0 +1,189 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "div", + "description" : "Divide one number by another. If either field is <> then the result is `null`.", + "note" : "Division of two integer types will yield an integer result, rounding towards 0. If you need floating point division, <> one of the arguments to a `DOUBLE`.", + "signatures" : [ + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "integer" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "unsigned_long" + } + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/equals.json b/docs/reference/esql/functions/kibana/definition/equals.json new file mode 100644 index 0000000000000..8c6d6d530302e --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/equals.json @@ -0,0 +1,405 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "equals", + "description" : "Check if two fields are equal. If either field is <> then the result is `null`.", + "note" : "This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <> and <>.", + "signatures" : [ + { + "params" : [ + { + "name" : "lhs", + "type" : "boolean", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "boolean", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "cartesian_point", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "cartesian_point", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "cartesian_shape", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "cartesian_shape", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "datetime", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "datetime", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "geo_point", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "geo_point", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "geo_shape", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "geo_shape", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "ip", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "ip", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "version", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "version", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/exp.json b/docs/reference/esql/functions/kibana/definition/exp.json new file mode 100644 index 0000000000000..02f7433ede779 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/exp.json @@ -0,0 +1,59 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "exp", + "description" : "Returns the value of e raised to the power of the given number.", + "signatures" : [ + { + "params" : [ + { + "name" : "number", + "type" : "double", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "integer", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "unsigned_long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "ROW d = 5.0\n| EVAL s = EXP(d)" + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/greater_than.json b/docs/reference/esql/functions/kibana/definition/greater_than.json new file mode 100644 index 0000000000000..371569fdb3489 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/greater_than.json @@ -0,0 +1,315 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "greater_than", + "description" : "Check if one field is greater than another. If either field is <> then the result is `null`.", + "note" : "This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <> and <>.", + "signatures" : [ + { + "params" : [ + { + "name" : "lhs", + "type" : "datetime", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "datetime", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "ip", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "ip", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "version", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "version", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json b/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json new file mode 100644 index 0000000000000..ed368d5677f77 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json @@ -0,0 +1,315 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "greater_than_or_equal", + "description" : "Check if one field is greater than or equal to another. If either field is <> then the result is `null`.", + "note" : "This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <> and <>.", + "signatures" : [ + { + "params" : [ + { + "name" : "lhs", + "type" : "datetime", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "datetime", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "ip", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "ip", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "version", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "version", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/in.json b/docs/reference/esql/functions/kibana/definition/in.json new file mode 100644 index 0000000000000..7dd9889caefe0 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/in.json @@ -0,0 +1,263 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "in", + "description" : "The `IN` operator allows testing whether a field or expression equals an element in a list of literals, fields or expressions.", + "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "boolean", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "inlist", + "type" : "boolean", + "optional" : false, + "description" : "A list of items." + } + ], + "variadic" : true, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "cartesian_point", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "inlist", + "type" : "cartesian_point", + "optional" : false, + "description" : "A list of items." + } + ], + "variadic" : true, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "cartesian_shape", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "inlist", + "type" : "cartesian_shape", + "optional" : false, + "description" : "A list of items." + } + ], + "variadic" : true, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "inlist", + "type" : "double", + "optional" : false, + "description" : "A list of items." + } + ], + "variadic" : true, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "geo_point", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "inlist", + "type" : "geo_point", + "optional" : false, + "description" : "A list of items." + } + ], + "variadic" : true, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "geo_shape", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "inlist", + "type" : "geo_shape", + "optional" : false, + "description" : "A list of items." + } + ], + "variadic" : true, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "inlist", + "type" : "integer", + "optional" : false, + "description" : "A list of items." + } + ], + "variadic" : true, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "ip", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "inlist", + "type" : "ip", + "optional" : false, + "description" : "A list of items." + } + ], + "variadic" : true, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "inlist", + "type" : "keyword", + "optional" : false, + "description" : "A list of items." + } + ], + "variadic" : true, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "inlist", + "type" : "text", + "optional" : false, + "description" : "A list of items." + } + ], + "variadic" : true, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "inlist", + "type" : "long", + "optional" : false, + "description" : "A list of items." + } + ], + "variadic" : true, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "text", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "inlist", + "type" : "keyword", + "optional" : false, + "description" : "A list of items." + } + ], + "variadic" : true, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "text", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "inlist", + "type" : "text", + "optional" : false, + "description" : "A list of items." + } + ], + "variadic" : true, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "version", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "inlist", + "type" : "version", + "optional" : false, + "description" : "A list of items." + } + ], + "variadic" : true, + "returnType" : "boolean" + } + ], + "examples" : [ + "ROW a = 1, b = 4, c = 3\n| WHERE c-a IN (3, b / 2, a)" + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/less_than.json b/docs/reference/esql/functions/kibana/definition/less_than.json new file mode 100644 index 0000000000000..b756319d67142 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/less_than.json @@ -0,0 +1,315 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "less_than", + "description" : "Check if one field is less than another. If either field is <> then the result is `null`.", + "note" : "This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <> and <>.", + "signatures" : [ + { + "params" : [ + { + "name" : "lhs", + "type" : "datetime", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "datetime", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "ip", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "ip", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "version", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "version", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json b/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json new file mode 100644 index 0000000000000..e06ff383c4242 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json @@ -0,0 +1,315 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "less_than_or_equal", + "description" : "Check if one field is less than or equal to another. If either field is <> then the result is `null`.", + "note" : "This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <> and <>.", + "signatures" : [ + { + "params" : [ + { + "name" : "lhs", + "type" : "datetime", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "datetime", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "ip", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "ip", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "version", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "version", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/like.json b/docs/reference/esql/functions/kibana/definition/like.json new file mode 100644 index 0000000000000..a852b4271bb70 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/like.json @@ -0,0 +1,47 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "like", + "description" : "Use `LIKE` to filter data based on string patterns using wildcards. `LIKE`\nusually acts on a field placed on the left-hand side of the operator, but it can\nalso act on a constant (literal) expression. The right-hand side of the operator\nrepresents the pattern.\n\nThe following wildcard characters are supported:\n\n* `*` matches zero or more characters.\n* `?` matches one character.", + "signatures" : [ + { + "params" : [ + { + "name" : "str", + "type" : "keyword", + "optional" : false, + "description" : "A literal expression." + }, + { + "name" : "pattern", + "type" : "keyword", + "optional" : false, + "description" : "Pattern." + } + ], + "variadic" : true, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "str", + "type" : "text", + "optional" : false, + "description" : "A literal expression." + }, + { + "name" : "pattern", + "type" : "text", + "optional" : false, + "description" : "Pattern." + } + ], + "variadic" : true, + "returnType" : "boolean" + } + ], + "examples" : [ + "FROM employees\n| WHERE first_name LIKE \"?b*\"\n| KEEP first_name, last_name" + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/locate.json b/docs/reference/esql/functions/kibana/definition/locate.json index 13b7512e17def..2097c90b41958 100644 --- a/docs/reference/esql/functions/kibana/definition/locate.json +++ b/docs/reference/esql/functions/kibana/definition/locate.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "locate", - "description" : "Returns an integer that indicates the position of a keyword substring within another string", + "description" : "Returns an integer that indicates the position of a keyword substring within another string.", "signatures" : [ { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/max.json b/docs/reference/esql/functions/kibana/definition/max.json index bc7380bd76dd4..853cb9f9a97c3 100644 --- a/docs/reference/esql/functions/kibana/definition/max.json +++ b/docs/reference/esql/functions/kibana/definition/max.json @@ -52,6 +52,18 @@ "variadic" : false, "returnType" : "integer" }, + { + "params" : [ + { + "name" : "field", + "type" : "ip", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "ip" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/kibana/definition/min.json b/docs/reference/esql/functions/kibana/definition/min.json index 937391bf242ac..1c0c02eb9860f 100644 --- a/docs/reference/esql/functions/kibana/definition/min.json +++ b/docs/reference/esql/functions/kibana/definition/min.json @@ -52,6 +52,18 @@ "variadic" : false, "returnType" : "integer" }, + { + "params" : [ + { + "name" : "field", + "type" : "ip", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "ip" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/kibana/definition/mod.json b/docs/reference/esql/functions/kibana/definition/mod.json new file mode 100644 index 0000000000000..78769f6bd4ecf --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/mod.json @@ -0,0 +1,188 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "mod", + "description" : "Divide one number by another and return the remainder. If either field is <> then the result is `null`.", + "signatures" : [ + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "integer" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "unsigned_long" + } + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/mul.json b/docs/reference/esql/functions/kibana/definition/mul.json new file mode 100644 index 0000000000000..6290369fb5b0f --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/mul.json @@ -0,0 +1,188 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "mul", + "description" : "Multiply two numbers together. If either field is <> then the result is `null`.", + "signatures" : [ + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "integer" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "A numeric value." + }, + { + "name" : "rhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "A numeric value." + } + ], + "variadic" : false, + "returnType" : "unsigned_long" + } + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/neg.json b/docs/reference/esql/functions/kibana/definition/neg.json new file mode 100644 index 0000000000000..bf203d8c7c226 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/neg.json @@ -0,0 +1,68 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "neg", + "description" : "Returns the negation of the argument.", + "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "date_period", + "optional" : false, + "description" : "A numeric value or a date time interval." + } + ], + "variadic" : false, + "returnType" : "date_period" + }, + { + "params" : [ + { + "name" : "field", + "type" : "double", + "optional" : false, + "description" : "A numeric value or a date time interval." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "A numeric value or a date time interval." + } + ], + "variadic" : false, + "returnType" : "integer" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "A numeric value or a date time interval." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "field", + "type" : "time_duration", + "optional" : false, + "description" : "A numeric value or a date time interval." + } + ], + "variadic" : false, + "returnType" : "time_duration" + } + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/not_equals.json b/docs/reference/esql/functions/kibana/definition/not_equals.json new file mode 100644 index 0000000000000..c5e61e3adaf44 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/not_equals.json @@ -0,0 +1,405 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "not_equals", + "description" : "Check if two fields are unequal. If either field is <> then the result is `null`.", + "note" : "This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <> and <>.", + "signatures" : [ + { + "params" : [ + { + "name" : "lhs", + "type" : "boolean", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "boolean", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "cartesian_point", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "cartesian_point", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "cartesian_shape", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "cartesian_shape", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "datetime", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "datetime", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "geo_point", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "geo_point", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "geo_shape", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "geo_shape", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "ip", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "ip", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "keyword", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "text", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "version", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "version", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/percentile.json b/docs/reference/esql/functions/kibana/definition/percentile.json new file mode 100644 index 0000000000000..5d8a8371ed454 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/percentile.json @@ -0,0 +1,174 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "agg", + "name" : "percentile", + "description" : "Returns the value at which a certain percentage of observed values occur. For example, the 95th percentile is the value which is greater than 95% of the observed values and the 50th percentile is the `MEDIAN`.", + "signatures" : [ + { + "params" : [ + { + "name" : "number", + "type" : "double", + "optional" : false, + "description" : "" + }, + { + "name" : "percentile", + "type" : "double", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "double", + "optional" : false, + "description" : "" + }, + { + "name" : "percentile", + "type" : "integer", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "double", + "optional" : false, + "description" : "" + }, + { + "name" : "percentile", + "type" : "long", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "integer", + "optional" : false, + "description" : "" + }, + { + "name" : "percentile", + "type" : "double", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "integer", + "optional" : false, + "description" : "" + }, + { + "name" : "percentile", + "type" : "integer", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "integer", + "optional" : false, + "description" : "" + }, + { + "name" : "percentile", + "type" : "long", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "long", + "optional" : false, + "description" : "" + }, + { + "name" : "percentile", + "type" : "double", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "long", + "optional" : false, + "description" : "" + }, + { + "name" : "percentile", + "type" : "integer", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "long", + "optional" : false, + "description" : "" + }, + { + "name" : "percentile", + "type" : "long", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "FROM employees\n| STATS p0 = PERCENTILE(salary, 0)\n , p50 = PERCENTILE(salary, 50)\n , p99 = PERCENTILE(salary, 99)", + "FROM employees\n| STATS p80_max_salary_change = PERCENTILE(MV_MAX(salary_change), 80)" + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/rlike.json b/docs/reference/esql/functions/kibana/definition/rlike.json new file mode 100644 index 0000000000000..bd6b83fe44a75 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/rlike.json @@ -0,0 +1,47 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "rlike", + "description" : "Use `RLIKE` to filter data based on string patterns using using\n<>. `RLIKE` usually acts on a field placed on\nthe left-hand side of the operator, but it can also act on a constant (literal)\nexpression. The right-hand side of the operator represents the pattern.", + "signatures" : [ + { + "params" : [ + { + "name" : "str", + "type" : "keyword", + "optional" : false, + "description" : "A literal value." + }, + { + "name" : "pattern", + "type" : "keyword", + "optional" : false, + "description" : "A regular expression." + } + ], + "variadic" : true, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "str", + "type" : "text", + "optional" : false, + "description" : "A literal value." + }, + { + "name" : "pattern", + "type" : "text", + "optional" : false, + "description" : "A regular expression." + } + ], + "variadic" : true, + "returnType" : "boolean" + } + ], + "examples" : [ + "FROM employees\n| WHERE first_name RLIKE \".leja.*\"\n| KEEP first_name, last_name" + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/sub.json b/docs/reference/esql/functions/kibana/definition/sub.json new file mode 100644 index 0000000000000..15888bd62b06b --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/sub.json @@ -0,0 +1,260 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "sub", + "description" : "Subtract one number from another. If either field is <> then the result is `null`.", + "signatures" : [ + { + "params" : [ + { + "name" : "lhs", + "type" : "date_period", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "date_period", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "date_period" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "datetime", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "date_period", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "datetime" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "datetime", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "time_duration", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "datetime" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "integer" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "double", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "integer", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "long", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "time_duration", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "time_duration", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "time_duration" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "unsigned_long", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "unsigned_long" + } + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/substring.json b/docs/reference/esql/functions/kibana/definition/substring.json index 25f432796cc8d..b38b545822a90 100644 --- a/docs/reference/esql/functions/kibana/definition/substring.json +++ b/docs/reference/esql/functions/kibana/definition/substring.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "substring", - "description" : "Returns a substring of a string, specified by a start position and an optional length", + "description" : "Returns a substring of a string, specified by a start position and an optional length.", "signatures" : [ { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/sum.json b/docs/reference/esql/functions/kibana/definition/sum.json new file mode 100644 index 0000000000000..b9235b6ba04de --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/sum.json @@ -0,0 +1,48 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "agg", + "name" : "sum", + "description" : "The sum of a numeric expression.", + "signatures" : [ + { + "params" : [ + { + "name" : "number", + "type" : "double", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "integer", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "number", + "type" : "long", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "long" + } + ], + "examples" : [ + "FROM employees\n| STATS SUM(languages)", + "FROM employees\n| STATS total_salary_changes = SUM(MV_MAX(salary_change))" + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/top.json b/docs/reference/esql/functions/kibana/definition/top.json index 7ad073d6e7564..4db3aed40a88d 100644 --- a/docs/reference/esql/functions/kibana/definition/top.json +++ b/docs/reference/esql/functions/kibana/definition/top.json @@ -4,6 +4,30 @@ "name" : "top", "description" : "Collects the top values for a field. Includes repeated values.", "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "boolean", + "optional" : false, + "description" : "The field to collect the top values for." + }, + { + "name" : "limit", + "type" : "integer", + "optional" : false, + "description" : "The maximum number of values to collect." + }, + { + "name" : "order", + "type" : "keyword", + "optional" : false, + "description" : "The order to calculate the top values. Either `asc` or `desc`." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { @@ -76,6 +100,30 @@ "variadic" : false, "returnType" : "integer" }, + { + "params" : [ + { + "name" : "field", + "type" : "ip", + "optional" : false, + "description" : "The field to collect the top values for." + }, + { + "name" : "limit", + "type" : "integer", + "optional" : false, + "description" : "The maximum number of values to collect." + }, + { + "name" : "order", + "type" : "keyword", + "optional" : false, + "description" : "The order to calculate the top values. Either `asc` or `desc`." + } + ], + "variadic" : false, + "returnType" : "ip" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/kibana/docs/add.md b/docs/reference/esql/functions/kibana/docs/add.md new file mode 100644 index 0000000000000..3f99bd4c77551 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/add.md @@ -0,0 +1,7 @@ + + +### ADD +Add two numbers together. If either field is <> then the result is `null`. + diff --git a/docs/reference/esql/functions/kibana/docs/div.md b/docs/reference/esql/functions/kibana/docs/div.md new file mode 100644 index 0000000000000..a8b7b4e58f376 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/div.md @@ -0,0 +1,8 @@ + + +### DIV +Divide one number by another. If either field is <> then the result is `null`. + +Note: Division of two integer types will yield an integer result, rounding towards 0. If you need floating point division, <> one of the arguments to a `DOUBLE`. diff --git a/docs/reference/esql/functions/kibana/docs/equals.md b/docs/reference/esql/functions/kibana/docs/equals.md new file mode 100644 index 0000000000000..b8fcb72c2ccd5 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/equals.md @@ -0,0 +1,8 @@ + + +### EQUALS +Check if two fields are equal. If either field is <> then the result is `null`. + +Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <> and <>. diff --git a/docs/reference/esql/functions/kibana/docs/exp.md b/docs/reference/esql/functions/kibana/docs/exp.md new file mode 100644 index 0000000000000..0eb6bef5f4247 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/exp.md @@ -0,0 +1,11 @@ + + +### EXP +Returns the value of e raised to the power of the given number. + +``` +ROW d = 5.0 +| EVAL s = EXP(d) +``` diff --git a/docs/reference/esql/functions/kibana/docs/greater_than.md b/docs/reference/esql/functions/kibana/docs/greater_than.md new file mode 100644 index 0000000000000..67f99eda3aed7 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/greater_than.md @@ -0,0 +1,8 @@ + + +### GREATER_THAN +Check if one field is greater than another. If either field is <> then the result is `null`. + +Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <> and <>. diff --git a/docs/reference/esql/functions/kibana/docs/greater_than_or_equal.md b/docs/reference/esql/functions/kibana/docs/greater_than_or_equal.md new file mode 100644 index 0000000000000..73d3ac6371b07 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/greater_than_or_equal.md @@ -0,0 +1,8 @@ + + +### GREATER_THAN_OR_EQUAL +Check if one field is greater than or equal to another. If either field is <> then the result is `null`. + +Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <> and <>. diff --git a/docs/reference/esql/functions/kibana/docs/in.md b/docs/reference/esql/functions/kibana/docs/in.md new file mode 100644 index 0000000000000..e096146374f38 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/in.md @@ -0,0 +1,11 @@ + + +### IN +The `IN` operator allows testing whether a field or expression equals an element in a list of literals, fields or expressions. + +``` +ROW a = 1, b = 4, c = 3 +| WHERE c-a IN (3, b / 2, a) +``` diff --git a/docs/reference/esql/functions/kibana/docs/less_than.md b/docs/reference/esql/functions/kibana/docs/less_than.md new file mode 100644 index 0000000000000..0d171d06c68d3 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/less_than.md @@ -0,0 +1,8 @@ + + +### LESS_THAN +Check if one field is less than another. If either field is <> then the result is `null`. + +Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <> and <>. diff --git a/docs/reference/esql/functions/kibana/docs/less_than_or_equal.md b/docs/reference/esql/functions/kibana/docs/less_than_or_equal.md new file mode 100644 index 0000000000000..acb92288c2c46 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/less_than_or_equal.md @@ -0,0 +1,8 @@ + + +### LESS_THAN_OR_EQUAL +Check if one field is less than or equal to another. If either field is <> then the result is `null`. + +Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <> and <>. diff --git a/docs/reference/esql/functions/kibana/docs/like.md b/docs/reference/esql/functions/kibana/docs/like.md new file mode 100644 index 0000000000000..4c400bdc65479 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/like.md @@ -0,0 +1,20 @@ + + +### LIKE +Use `LIKE` to filter data based on string patterns using wildcards. `LIKE` +usually acts on a field placed on the left-hand side of the operator, but it can +also act on a constant (literal) expression. The right-hand side of the operator +represents the pattern. + +The following wildcard characters are supported: + +* `*` matches zero or more characters. +* `?` matches one character. + +``` +FROM employees +| WHERE first_name LIKE "?b*" +| KEEP first_name, last_name +``` diff --git a/docs/reference/esql/functions/kibana/docs/locate.md b/docs/reference/esql/functions/kibana/docs/locate.md index 7fffbfd548f20..75275068d3096 100644 --- a/docs/reference/esql/functions/kibana/docs/locate.md +++ b/docs/reference/esql/functions/kibana/docs/locate.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### LOCATE -Returns an integer that indicates the position of a keyword substring within another string +Returns an integer that indicates the position of a keyword substring within another string. ``` row a = "hello" diff --git a/docs/reference/esql/functions/kibana/docs/mod.md b/docs/reference/esql/functions/kibana/docs/mod.md new file mode 100644 index 0000000000000..e6b3c92406072 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/mod.md @@ -0,0 +1,7 @@ + + +### MOD +Divide one number by another and return the remainder. If either field is <> then the result is `null`. + diff --git a/docs/reference/esql/functions/kibana/docs/mul.md b/docs/reference/esql/functions/kibana/docs/mul.md new file mode 100644 index 0000000000000..3f24f3b1a67bb --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/mul.md @@ -0,0 +1,7 @@ + + +### MUL +Multiply two numbers together. If either field is <> then the result is `null`. + diff --git a/docs/reference/esql/functions/kibana/docs/neg.md b/docs/reference/esql/functions/kibana/docs/neg.md new file mode 100644 index 0000000000000..2d1c65487343f --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/neg.md @@ -0,0 +1,7 @@ + + +### NEG +Returns the negation of the argument. + diff --git a/docs/reference/esql/functions/kibana/docs/not_equals.md b/docs/reference/esql/functions/kibana/docs/not_equals.md new file mode 100644 index 0000000000000..cff2130e766ed --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/not_equals.md @@ -0,0 +1,8 @@ + + +### NOT_EQUALS +Check if two fields are unequal. If either field is <> then the result is `null`. + +Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <> and <>. diff --git a/docs/reference/esql/functions/kibana/docs/percentile.md b/docs/reference/esql/functions/kibana/docs/percentile.md new file mode 100644 index 0000000000000..dd907ef817186 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/percentile.md @@ -0,0 +1,13 @@ + + +### PERCENTILE +Returns the value at which a certain percentage of observed values occur. For example, the 95th percentile is the value which is greater than 95% of the observed values and the 50th percentile is the `MEDIAN`. + +``` +FROM employees +| STATS p0 = PERCENTILE(salary, 0) + , p50 = PERCENTILE(salary, 50) + , p99 = PERCENTILE(salary, 99) +``` diff --git a/docs/reference/esql/functions/kibana/docs/rlike.md b/docs/reference/esql/functions/kibana/docs/rlike.md new file mode 100644 index 0000000000000..ed94553e7e44f --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/rlike.md @@ -0,0 +1,15 @@ + + +### RLIKE +Use `RLIKE` to filter data based on string patterns using using +<>. `RLIKE` usually acts on a field placed on +the left-hand side of the operator, but it can also act on a constant (literal) +expression. The right-hand side of the operator represents the pattern. + +``` +FROM employees +| WHERE first_name RLIKE ".leja.*" +| KEEP first_name, last_name +``` diff --git a/docs/reference/esql/functions/kibana/docs/sub.md b/docs/reference/esql/functions/kibana/docs/sub.md new file mode 100644 index 0000000000000..10746ed81cfe3 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/sub.md @@ -0,0 +1,7 @@ + + +### SUB +Subtract one number from another. If either field is <> then the result is `null`. + diff --git a/docs/reference/esql/functions/kibana/docs/substring.md b/docs/reference/esql/functions/kibana/docs/substring.md index 62c4eb33c2e95..5f2601a279f6f 100644 --- a/docs/reference/esql/functions/kibana/docs/substring.md +++ b/docs/reference/esql/functions/kibana/docs/substring.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### SUBSTRING -Returns a substring of a string, specified by a start position and an optional length +Returns a substring of a string, specified by a start position and an optional length. ``` FROM employees diff --git a/docs/reference/esql/functions/kibana/docs/sum.md b/docs/reference/esql/functions/kibana/docs/sum.md new file mode 100644 index 0000000000000..eb72ddb0dece1 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/sum.md @@ -0,0 +1,11 @@ + + +### SUM +The sum of a numeric expression. + +``` +FROM employees +| STATS SUM(languages) +``` diff --git a/docs/reference/esql/functions/layout/exp.asciidoc b/docs/reference/esql/functions/layout/exp.asciidoc new file mode 100644 index 0000000000000..e76532e856041 --- /dev/null +++ b/docs/reference/esql/functions/layout/exp.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-exp]] +=== `EXP` + +*Syntax* + +[.text-center] +image::esql/functions/signature/exp.svg[Embedded,opts=inline] + +include::../parameters/exp.asciidoc[] +include::../description/exp.asciidoc[] +include::../types/exp.asciidoc[] +include::../examples/exp.asciidoc[] diff --git a/docs/reference/esql/functions/layout/percentile.asciidoc b/docs/reference/esql/functions/layout/percentile.asciidoc new file mode 100644 index 0000000000000..a8ceafc66673b --- /dev/null +++ b/docs/reference/esql/functions/layout/percentile.asciidoc @@ -0,0 +1,16 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-percentile]] +=== `PERCENTILE` + +*Syntax* + +[.text-center] +image::esql/functions/signature/percentile.svg[Embedded,opts=inline] + +include::../parameters/percentile.asciidoc[] +include::../description/percentile.asciidoc[] +include::../types/percentile.asciidoc[] +include::../examples/percentile.asciidoc[] +include::../appendix/percentile.asciidoc[] diff --git a/docs/reference/esql/functions/layout/sum.asciidoc b/docs/reference/esql/functions/layout/sum.asciidoc new file mode 100644 index 0000000000000..abac1fdd27b6e --- /dev/null +++ b/docs/reference/esql/functions/layout/sum.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-sum]] +=== `SUM` + +*Syntax* + +[.text-center] +image::esql/functions/signature/sum.svg[Embedded,opts=inline] + +include::../parameters/sum.asciidoc[] +include::../description/sum.asciidoc[] +include::../types/sum.asciidoc[] +include::../examples/sum.asciidoc[] diff --git a/docs/reference/esql/functions/math-functions.asciidoc b/docs/reference/esql/functions/math-functions.asciidoc index db907c8d54061..e311208795533 100644 --- a/docs/reference/esql/functions/math-functions.asciidoc +++ b/docs/reference/esql/functions/math-functions.asciidoc @@ -18,6 +18,7 @@ * <> * <> * <> +* <> * <> * <> * <> @@ -43,6 +44,7 @@ include::layout/ceil.asciidoc[] include::layout/cos.asciidoc[] include::layout/cosh.asciidoc[] include::layout/e.asciidoc[] +include::layout/exp.asciidoc[] include::layout/floor.asciidoc[] include::layout/log.asciidoc[] include::layout/log10.asciidoc[] diff --git a/docs/reference/esql/functions/median-absolute-deviation.asciidoc b/docs/reference/esql/functions/median-absolute-deviation.asciidoc index 796e0797157de..b4f80cced06f6 100644 --- a/docs/reference/esql/functions/median-absolute-deviation.asciidoc +++ b/docs/reference/esql/functions/median-absolute-deviation.asciidoc @@ -25,8 +25,8 @@ It is calculated as the median of each data point's deviation from the median of the entire sample. That is, for a random variable `X`, the median absolute deviation is `median(|median(X) - X|)`. -NOTE: Like <>, `MEDIAN_ABSOLUTE_DEVIATION` is - <>. +NOTE: Like <>, `MEDIAN_ABSOLUTE_DEVIATION` is + <>. [WARNING] ==== diff --git a/docs/reference/esql/functions/median.asciidoc b/docs/reference/esql/functions/median.asciidoc index ef845aafd3915..2f7d70775e38e 100644 --- a/docs/reference/esql/functions/median.asciidoc +++ b/docs/reference/esql/functions/median.asciidoc @@ -17,9 +17,9 @@ Expression from which to return the median value. *Description* Returns the value that is greater than half of all values and less than half of -all values, also known as the 50% <>. +all values, also known as the 50% <>. -NOTE: Like <>, `MEDIAN` is <>. +NOTE: Like <>, `MEDIAN` is <>. [WARNING] ==== diff --git a/docs/reference/esql/functions/parameters/exp.asciidoc b/docs/reference/esql/functions/parameters/exp.asciidoc new file mode 100644 index 0000000000000..65013f4c21265 --- /dev/null +++ b/docs/reference/esql/functions/parameters/exp.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`number`:: +Numeric expression. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/percentile.asciidoc b/docs/reference/esql/functions/parameters/percentile.asciidoc new file mode 100644 index 0000000000000..efabdbd7ca914 --- /dev/null +++ b/docs/reference/esql/functions/parameters/percentile.asciidoc @@ -0,0 +1,9 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`number`:: + + +`percentile`:: + diff --git a/docs/reference/esql/functions/parameters/sum.asciidoc b/docs/reference/esql/functions/parameters/sum.asciidoc new file mode 100644 index 0000000000000..91c56709d182a --- /dev/null +++ b/docs/reference/esql/functions/parameters/sum.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`number`:: + diff --git a/docs/reference/esql/functions/percentile.asciidoc b/docs/reference/esql/functions/percentile.asciidoc deleted file mode 100644 index e00ee436c31cf..0000000000000 --- a/docs/reference/esql/functions/percentile.asciidoc +++ /dev/null @@ -1,60 +0,0 @@ -[discrete] -[[esql-agg-percentile]] -=== `PERCENTILE` - -*Syntax* - -[source,esql] ----- -PERCENTILE(expression, percentile) ----- - -*Parameters* - -`expression`:: -Expression from which to return a percentile. - -`percentile`:: -A constant numeric expression. - -*Description* - -Returns the value at which a certain percentage of observed values occur. For -example, the 95th percentile is the value which is greater than 95% of the -observed values and the 50th percentile is the <>. - -*Example* - -[source.merge.styled,esql] ----- -include::{esql-specs}/stats_percentile.csv-spec[tag=percentile] ----- -[%header.monospaced.styled,format=dsv,separator=|] -|=== -include::{esql-specs}/stats_percentile.csv-spec[tag=percentile-result] -|=== - -The expression can use inline functions. For example, to calculate a percentile -of the maximum values of a multivalued column, first use `MV_MAX` to get the -maximum value per row, and use the result with the `PERCENTILE` function: - -[source.merge.styled,esql] ----- -include::{esql-specs}/stats_percentile.csv-spec[tag=docsStatsPercentileNestedExpression] ----- -[%header.monospaced.styled,format=dsv,separator=|] -|=== -include::{esql-specs}/stats_percentile.csv-spec[tag=docsStatsPercentileNestedExpression-result] -|=== - -[discrete] -[[esql-agg-percentile-approximate]] -==== `PERCENTILE` is (usually) approximate - -include::../../aggregations/metrics/percentile-aggregation.asciidoc[tag=approximate] - -[WARNING] -==== -`PERCENTILE` is also {wikipedia}/Nondeterministic_algorithm[non-deterministic]. -This means you can get slightly different results using the same data. -==== \ No newline at end of file diff --git a/docs/reference/esql/functions/round.asciidoc b/docs/reference/esql/functions/round.asciidoc deleted file mode 100644 index e792db6c1ed69..0000000000000 --- a/docs/reference/esql/functions/round.asciidoc +++ /dev/null @@ -1,34 +0,0 @@ -[discrete] -[[esql-round]] -=== `ROUND` -*Syntax* - -[.text-center] -image::esql/functions/signature/round.svg[Embedded,opts=inline] - -*Parameters* - -`value`:: -Numeric expression. If `null`, the function returns `null`. - -`decimals`:: -Numeric expression. If `null`, the function returns `null`. - -*Description* - -Rounds a number to the closest number with the specified number of digits. -Defaults to 0 digits if no number of digits is provided. If the specified number -of digits is negative, rounds to the number of digits left of the decimal point. - -include::types/round.asciidoc[] - -*Example* - -[source.merge.styled,esql] ----- -include::{esql-specs}/docs.csv-spec[tag=round] ----- -[%header.monospaced.styled,format=dsv,separator=|] -|=== -include::{esql-specs}/docs.csv-spec[tag=round-result] -|=== diff --git a/docs/reference/esql/functions/signature/exp.svg b/docs/reference/esql/functions/signature/exp.svg new file mode 100644 index 0000000000000..7637faefeb57b --- /dev/null +++ b/docs/reference/esql/functions/signature/exp.svg @@ -0,0 +1 @@ +EXP(number) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/percentile.svg b/docs/reference/esql/functions/signature/percentile.svg new file mode 100644 index 0000000000000..bb4090898fa27 --- /dev/null +++ b/docs/reference/esql/functions/signature/percentile.svg @@ -0,0 +1 @@ +PERCENTILE(number,percentile) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/sum.svg b/docs/reference/esql/functions/signature/sum.svg new file mode 100644 index 0000000000000..d1024edc2a5b9 --- /dev/null +++ b/docs/reference/esql/functions/signature/sum.svg @@ -0,0 +1 @@ +SUM(number) \ No newline at end of file diff --git a/docs/reference/esql/functions/types/exp.asciidoc b/docs/reference/esql/functions/types/exp.asciidoc new file mode 100644 index 0000000000000..7cda278abdb56 --- /dev/null +++ b/docs/reference/esql/functions/types/exp.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +number | result +double | double +integer | double +long | double +unsigned_long | double +|=== diff --git a/docs/reference/esql/functions/types/in.asciidoc b/docs/reference/esql/functions/types/in.asciidoc new file mode 100644 index 0000000000000..6ed2c250ef0ac --- /dev/null +++ b/docs/reference/esql/functions/types/in.asciidoc @@ -0,0 +1,22 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +field | inlist | result +boolean | boolean | boolean +cartesian_point | cartesian_point | boolean +cartesian_shape | cartesian_shape | boolean +double | double | boolean +geo_point | geo_point | boolean +geo_shape | geo_shape | boolean +integer | integer | boolean +ip | ip | boolean +keyword | keyword | boolean +keyword | text | boolean +long | long | boolean +text | keyword | boolean +text | text | boolean +version | version | boolean +|=== diff --git a/docs/reference/esql/functions/types/max.asciidoc b/docs/reference/esql/functions/types/max.asciidoc index 6515c6bfc48d2..5b7293d4a4293 100644 --- a/docs/reference/esql/functions/types/max.asciidoc +++ b/docs/reference/esql/functions/types/max.asciidoc @@ -9,5 +9,6 @@ boolean | boolean datetime | datetime double | double integer | integer +ip | ip long | long |=== diff --git a/docs/reference/esql/functions/types/min.asciidoc b/docs/reference/esql/functions/types/min.asciidoc index 6515c6bfc48d2..5b7293d4a4293 100644 --- a/docs/reference/esql/functions/types/min.asciidoc +++ b/docs/reference/esql/functions/types/min.asciidoc @@ -9,5 +9,6 @@ boolean | boolean datetime | datetime double | double integer | integer +ip | ip long | long |=== diff --git a/docs/reference/esql/functions/types/neg.asciidoc b/docs/reference/esql/functions/types/neg.asciidoc index 28d3b2a512dec..c0d0a21711552 100644 --- a/docs/reference/esql/functions/types/neg.asciidoc +++ b/docs/reference/esql/functions/types/neg.asciidoc @@ -4,7 +4,7 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== -v | result +field | result date_period | date_period double | double integer | integer diff --git a/docs/reference/esql/functions/types/percentile.asciidoc b/docs/reference/esql/functions/types/percentile.asciidoc new file mode 100644 index 0000000000000..8152a237a88ea --- /dev/null +++ b/docs/reference/esql/functions/types/percentile.asciidoc @@ -0,0 +1,17 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +number | percentile | result +double | double | double +double | integer | double +double | long | double +integer | double | double +integer | integer | double +integer | long | double +long | double | double +long | integer | double +long | long | double +|=== diff --git a/docs/reference/esql/functions/types/rlike.asciidoc b/docs/reference/esql/functions/types/rlike.asciidoc index 436333fddf5ee..46532f2af3bf3 100644 --- a/docs/reference/esql/functions/types/rlike.asciidoc +++ b/docs/reference/esql/functions/types/rlike.asciidoc @@ -4,7 +4,7 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== -str | pattern | caseInsensitive | result -keyword | keyword | boolean | boolean -text | text | boolean | boolean +str | pattern | result +keyword | keyword | boolean +text | text | boolean |=== diff --git a/docs/reference/esql/functions/types/sum.asciidoc b/docs/reference/esql/functions/types/sum.asciidoc new file mode 100644 index 0000000000000..aa4c3ad0d7dd8 --- /dev/null +++ b/docs/reference/esql/functions/types/sum.asciidoc @@ -0,0 +1,11 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +number | result +double | double +integer | long +long | long +|=== diff --git a/docs/reference/esql/functions/types/top.asciidoc b/docs/reference/esql/functions/types/top.asciidoc index 1874cd8b12bf3..ff71b2d153e3a 100644 --- a/docs/reference/esql/functions/types/top.asciidoc +++ b/docs/reference/esql/functions/types/top.asciidoc @@ -5,8 +5,10 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== field | limit | order | result +boolean | integer | keyword | boolean datetime | integer | keyword | datetime double | integer | keyword | double integer | integer | keyword | integer +ip | integer | keyword | ip long | integer | keyword | long |=== diff --git a/docs/reference/esql/processing-commands/dissect.asciidoc b/docs/reference/esql/processing-commands/dissect.asciidoc index c48b72af0de7e..82138aa238087 100644 --- a/docs/reference/esql/processing-commands/dissect.asciidoc +++ b/docs/reference/esql/processing-commands/dissect.asciidoc @@ -2,6 +2,9 @@ [[esql-dissect]] === `DISSECT` +`DISSECT` enables you to <>. + **Syntax** [source,esql] @@ -17,6 +20,8 @@ multiple values, `DISSECT` will process each value. `pattern`:: A <>. +If a field name conflicts with an existing column, the existing column is dropped. +If a field name is used more than once, only the rightmost duplicate creates a column. ``:: A string used as the separator between appended values, when using the <>. @@ -56,4 +61,4 @@ include::{esql-specs}/docs.csv-spec[tag=dissectWithToDatetime] include::{esql-specs}/docs.csv-spec[tag=dissectWithToDatetime-result] |=== -// end::examples[] \ No newline at end of file +// end::examples[] diff --git a/docs/reference/esql/processing-commands/drop.asciidoc b/docs/reference/esql/processing-commands/drop.asciidoc index 8f03141d5e05a..c81f438f81c3b 100644 --- a/docs/reference/esql/processing-commands/drop.asciidoc +++ b/docs/reference/esql/processing-commands/drop.asciidoc @@ -2,6 +2,8 @@ [[esql-drop]] === `DROP` +The `DROP` processing command removes one or more columns. + **Syntax** [source,esql] @@ -14,10 +16,6 @@ DROP columns `columns`:: A comma-separated list of columns to remove. Supports wildcards. -*Description* - -The `DROP` processing command removes one or more columns. - *Examples* [source,esql] diff --git a/docs/reference/esql/processing-commands/enrich.asciidoc b/docs/reference/esql/processing-commands/enrich.asciidoc index 5470d81b2f40b..2ece5a63e7570 100644 --- a/docs/reference/esql/processing-commands/enrich.asciidoc +++ b/docs/reference/esql/processing-commands/enrich.asciidoc @@ -2,6 +2,9 @@ [[esql-enrich]] === `ENRICH` +`ENRICH` enables you to add data from existing indices as new columns using an +enrich policy. + **Syntax** [source,esql] @@ -28,11 +31,16 @@ name as the `match_field` defined in the <>. The enrich fields from the enrich index that are added to the result as new columns. If a column with the same name as the enrich field already exists, the existing column will be replaced by the new column. If not specified, each of -the enrich fields defined in the policy is added +the enrich fields defined in the policy is added. +A column with the same name as the enrich field will be dropped unless the +enrich field is renamed. `new_nameX`:: Enables you to change the name of the column that's added for each of the enrich fields. Defaults to the enrich field name. +If a column has the same name as the new name, it will be discarded. +If a name (new or original) occurs more than once, only the rightmost duplicate +creates a new column. *Description* diff --git a/docs/reference/esql/processing-commands/eval.asciidoc b/docs/reference/esql/processing-commands/eval.asciidoc index 9b34fca7ceeff..00a7764d24004 100644 --- a/docs/reference/esql/processing-commands/eval.asciidoc +++ b/docs/reference/esql/processing-commands/eval.asciidoc @@ -2,6 +2,9 @@ [[esql-eval]] === `EVAL` +The `EVAL` processing command enables you to append new columns with calculated +values. + **Syntax** [source,esql] @@ -13,10 +16,12 @@ EVAL [column1 =] value1[, ..., [columnN =] valueN] `columnX`:: The column name. +If a column with the same name already exists, the existing column is dropped. +If a column name is used more than once, only the rightmost duplicate creates a column. `valueX`:: The value for the column. Can be a literal, an expression, or a -<>. +<>. Can use columns defined left of this one. *Description* diff --git a/docs/reference/esql/processing-commands/grok.asciidoc b/docs/reference/esql/processing-commands/grok.asciidoc index d5d58a9eaee12..57c55a5bad53f 100644 --- a/docs/reference/esql/processing-commands/grok.asciidoc +++ b/docs/reference/esql/processing-commands/grok.asciidoc @@ -2,6 +2,9 @@ [[esql-grok]] === `GROK` +`GROK` enables you to <>. + **Syntax** [source,esql] @@ -17,6 +20,9 @@ multiple values, `GROK` will process each value. `pattern`:: A grok pattern. +If a field name conflicts with an existing column, the existing column is discarded. +If a field name is used more than once, a multi-valued column will be created with one value +per each occurrence of the field name. *Description* @@ -64,4 +70,16 @@ include::{esql-specs}/docs.csv-spec[tag=grokWithToDatetime] |=== include::{esql-specs}/docs.csv-spec[tag=grokWithToDatetime-result] |=== + +If a field name is used more than once, `GROK` creates a multi-valued +column: + +[source.merge.styled,esql] +---- +include::{esql-specs}/docs.csv-spec[tag=grokWithDuplicateFieldNames] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/docs.csv-spec[tag=grokWithDuplicateFieldNames-result] +|=== // end::examples[] diff --git a/docs/reference/esql/processing-commands/inlinestats.asciidoc b/docs/reference/esql/processing-commands/inlinestats.asciidoc new file mode 100644 index 0000000000000..0b8d38c7b280f --- /dev/null +++ b/docs/reference/esql/processing-commands/inlinestats.asciidoc @@ -0,0 +1,102 @@ +[discrete] +[[esql-inlinestats-by]] +=== `INLINESTATS ... BY` + +experimental::["INLINESTATS is highly experimental and only available in SNAPSHOT versions."] + +The `INLINESTATS` command calculates an aggregate result and adds new columns +with the result to the stream of input data. + +**Syntax** + +[source,esql] +---- +INLINESTATS [column1 =] expression1[, ..., [columnN =] expressionN] +[BY grouping_expression1[, ..., grouping_expressionN]] +---- + +*Parameters* + +`columnX`:: +The name by which the aggregated value is returned. If omitted, the name is +equal to the corresponding expression (`expressionX`). If multiple columns +have the same name, all but the rightmost column with this name will be ignored. + +`expressionX`:: +An expression that computes an aggregated value. If its name coincides with one +of the computed columns, that column will be ignored. + +`grouping_expressionX`:: +An expression that outputs the values to group by. + +NOTE: Individual `null` values are skipped when computing aggregations. + +*Description* + +The `INLINESTATS` command calculates an aggregate result and merges that result +back into the stream of input data. Without the optional `BY` clause this will +produce a single result which is appended to each row. With a `BY` clause this +will produce one result per grouping and merge the result into the stream based on +matching group keys. + +All of the <> are supported. + +*Examples* + +Find the employees that speak the most languages (it's a tie!): + +[source.merge.styled,esql] +---- +include::{esql-specs}/inlinestats.csv-spec[tag=max-languages] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/inlinestats.csv-spec[tag=max-languages-result] +|=== + +Find the longest tenured employee who's last name starts with each letter of the alphabet: + +[source.merge.styled,esql] +---- +include::{esql-specs}/inlinestats.csv-spec[tag=longest-tenured-by-first] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/inlinestats.csv-spec[tag=longest-tenured-by-first-result] +|=== + +Find the northern and southern most airports: + +[source.merge.styled,esql] +---- +include::{esql-specs}/inlinestats.csv-spec[tag=extreme-airports] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/inlinestats.csv-spec[tag=extreme-airports-result] +|=== + +NOTE: Our test data doesn't have many "small" airports. + +If a `BY` field is multivalued then `INLINESTATS` will put the row in *each* +bucket like <>: + +[source.merge.styled,esql] +---- +include::{esql-specs}/inlinestats.csv-spec[tag=mv-group] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/inlinestats.csv-spec[tag=mv-group-result] +|=== + +To treat each group key as its own row use <> before `INLINESTATS`: + +[source.merge.styled,esql] +---- +include::{esql-specs}/inlinestats.csv-spec[tag=mv-expand] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/inlinestats.csv-spec[tag=mv-expand-result] +|=== diff --git a/docs/reference/esql/processing-commands/keep.asciidoc b/docs/reference/esql/processing-commands/keep.asciidoc index 57f32a68aec4c..3dbd0c69d8222 100644 --- a/docs/reference/esql/processing-commands/keep.asciidoc +++ b/docs/reference/esql/processing-commands/keep.asciidoc @@ -2,6 +2,9 @@ [[esql-keep]] === `KEEP` +The `KEEP` processing command enables you to specify what columns are returned +and the order in which they are returned. + **Syntax** [source,esql] @@ -13,6 +16,8 @@ KEEP columns `columns`:: A comma-separated list of columns to keep. Supports wildcards. +See below for the behavior in case an existing column matches multiple +given wildcards or column names. *Description* @@ -26,7 +31,7 @@ Fields are added in the order they appear. If one field matches multiple express 2. Partial wildcard expressions (for example: `fieldNam*`) 3. Wildcard only (`*`) -If a field matches two expressions with the same precedence, the right-most expression wins. +If a field matches two expressions with the same precedence, the rightmost expression wins. Refer to the examples for illustrations of these precedence rules. @@ -70,7 +75,7 @@ include::{esql-specs}/docs.csv-spec[tag=keepDoubleWildcard] include::{esql-specs}/docs.csv-spec[tag=keep-double-wildcard-result] |=== -The following examples show how precedence rules work when a field name matches multiple expressions. +The following examples show how precedence rules work when a field name matches multiple expressions. Complete field name has precedence over wildcard expressions: diff --git a/docs/reference/esql/processing-commands/limit.asciidoc b/docs/reference/esql/processing-commands/limit.asciidoc index 4ccf3024a4c1e..78d05672ea095 100644 --- a/docs/reference/esql/processing-commands/limit.asciidoc +++ b/docs/reference/esql/processing-commands/limit.asciidoc @@ -2,6 +2,9 @@ [[esql-limit]] === `LIMIT` +The `LIMIT` processing command enables you to limit the number of rows that are +returned. + **Syntax** [source,esql] diff --git a/docs/reference/esql/processing-commands/lookup.asciidoc b/docs/reference/esql/processing-commands/lookup.asciidoc index 1944d243968a8..ca456d8e70eed 100644 --- a/docs/reference/esql/processing-commands/lookup.asciidoc +++ b/docs/reference/esql/processing-commands/lookup.asciidoc @@ -2,7 +2,10 @@ [[esql-lookup]] === `LOOKUP` -experimental::["LOOKUP is a highly experimental and only available in SNAPSHOT versions."] +experimental::["LOOKUP is highly experimental and only available in SNAPSHOT versions."] + +`LOOKUP` matches values from the input against a `table` provided in the request, +adding the other fields from the `table` to the output. **Syntax** @@ -15,15 +18,11 @@ LOOKUP table ON match_field1[, match_field2, ...] `table`:: The name of the `table` provided in the request to match. +If the table's column names conflict with existing columns, the existing columns will be dropped. `match_field`:: The fields in the input to match against the table. -*Description* - -`LOOKUP` matches values from the input against a `table` provided in the request, -adding the other fields from the `table` to the output. - *Examples* // tag::examples[] diff --git a/docs/reference/esql/processing-commands/mv_expand.asciidoc b/docs/reference/esql/processing-commands/mv_expand.asciidoc index 9e1cb5573c381..010701f7fc8ee 100644 --- a/docs/reference/esql/processing-commands/mv_expand.asciidoc +++ b/docs/reference/esql/processing-commands/mv_expand.asciidoc @@ -4,6 +4,9 @@ preview::[] +The `MV_EXPAND` processing command expands multivalued columns into one row per +value, duplicating other columns. + **Syntax** [source,esql] @@ -16,11 +19,6 @@ MV_EXPAND column `column`:: The multivalued column to expand. -*Description* - -The `MV_EXPAND` processing command expands multivalued columns into one row per -value, duplicating other columns. - *Example* [source.merge.styled,esql] diff --git a/docs/reference/esql/processing-commands/rename.asciidoc b/docs/reference/esql/processing-commands/rename.asciidoc index 773fe8b640f75..41e2ce9298ae8 100644 --- a/docs/reference/esql/processing-commands/rename.asciidoc +++ b/docs/reference/esql/processing-commands/rename.asciidoc @@ -2,6 +2,8 @@ [[esql-rename]] === `RENAME` +The `RENAME` processing command renames one or more columns. + **Syntax** [source,esql] @@ -15,7 +17,9 @@ RENAME old_name1 AS new_name1[, ..., old_nameN AS new_nameN] The name of a column you want to rename. `new_nameX`:: -The new name of the column. +The new name of the column. If it conflicts with an existing column name, +the existing column is dropped. If multiple columns are renamed to the same +name, all but the rightmost column with the same new name are dropped. *Description* diff --git a/docs/reference/esql/processing-commands/sort.asciidoc b/docs/reference/esql/processing-commands/sort.asciidoc index fea7bfaf0c65f..e76b9c76ab273 100644 --- a/docs/reference/esql/processing-commands/sort.asciidoc +++ b/docs/reference/esql/processing-commands/sort.asciidoc @@ -2,6 +2,8 @@ [[esql-sort]] === `SORT` +The `SORT` processing command sorts a table on one or more columns. + **Syntax** [source,esql] diff --git a/docs/reference/esql/processing-commands/stats.asciidoc b/docs/reference/esql/processing-commands/stats.asciidoc index fe84c56bbfc19..7377522a93201 100644 --- a/docs/reference/esql/processing-commands/stats.asciidoc +++ b/docs/reference/esql/processing-commands/stats.asciidoc @@ -2,11 +2,14 @@ [[esql-stats-by]] === `STATS ... BY` +The `STATS ... BY` processing command groups rows according to a common value +and calculate one or more aggregated values over the grouped rows. + **Syntax** [source,esql] ---- -STATS [column1 =] expression1[, ..., [columnN =] expressionN] +STATS [column1 =] expression1[, ..., [columnN =] expressionN] [BY grouping_expression1[, ..., grouping_expressionN]] ---- @@ -15,12 +18,15 @@ STATS [column1 =] expression1[, ..., [columnN =] expressionN] `columnX`:: The name by which the aggregated value is returned. If omitted, the name is equal to the corresponding expression (`expressionX`). +If multiple columns have the same name, all but the rightmost column with this +name will be ignored. `expressionX`:: An expression that computes an aggregated value. `grouping_expressionX`:: An expression that outputs the values to group by. +If its name coincides with one of the computed columns, that column will be ignored. NOTE: Individual `null` values are skipped when computing aggregations. @@ -39,8 +45,8 @@ NOTE: `STATS` without any groups is much much faster than adding a group. NOTE: Grouping on a single expression is currently much more optimized than grouping on many expressions. In some tests we have seen grouping on a single `keyword` - column to be five times faster than grouping on two `keyword` columns. Do - not try to work around this by combining the two columns together with + column to be five times faster than grouping on two `keyword` columns. Do + not try to work around this by combining the two columns together with something like <> and then grouping - that is not going to be faster. @@ -80,14 +86,36 @@ include::{esql-specs}/stats.csv-spec[tag=statsCalcMultipleValues] include::{esql-specs}/stats.csv-spec[tag=statsCalcMultipleValues-result] |=== -It's also possible to group by multiple values (only supported for long and -keyword family fields): +[[esql-stats-mv-group]] +If the grouping key is multivalued then the input row is in all groups: + +[source.merge.styled,esql] +---- +include::{esql-specs}/stats.csv-spec[tag=mv-group] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/stats.csv-spec[tag=mv-group-result] +|=== + +It's also possible to group by multiple values: [source,esql] ---- include::{esql-specs}/stats.csv-spec[tag=statsGroupByMultipleValues] ---- +If the all grouping keys are multivalued then the input row is in all groups: + +[source.merge.styled,esql] +---- +include::{esql-specs}/stats.csv-spec[tag=multi-mv-group] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/stats.csv-spec[tag=multi-mv-group-result] +|=== + Both the aggregating functions and the grouping expressions accept other functions. This is useful for using `STATS...BY` on multivalue columns. For example, to calculate the average salary change, you can use `MV_AVG` to diff --git a/docs/reference/esql/processing-commands/where.asciidoc b/docs/reference/esql/processing-commands/where.asciidoc index 3076f92c40fc0..407df30c57215 100644 --- a/docs/reference/esql/processing-commands/where.asciidoc +++ b/docs/reference/esql/processing-commands/where.asciidoc @@ -2,6 +2,9 @@ [[esql-where]] === `WHERE` +The `WHERE` processing command produces a table that contains all the rows from +the input table for which the provided condition evaluates to `true`. + **Syntax** [source,esql] @@ -14,11 +17,6 @@ WHERE expression `expression`:: A boolean expression. -*Description* - -The `WHERE` processing command produces a table that contains all the rows from -the input table for which the provided condition evaluates to `true`. - *Examples* [source,esql] @@ -33,7 +31,7 @@ Which, if `still_hired` is a boolean field, can be simplified to: include::{esql-specs}/docs.csv-spec[tag=whereBoolean] ---- -Use date math to retrieve data from a specific time range. For example, to +Use date math to retrieve data from a specific time range. For example, to retrieve the last hour of logs: [source,esql] @@ -59,4 +57,4 @@ include::../functions/rlike.asciidoc[tag=body] include::../functions/in.asciidoc[tag=body] -For a complete list of all operators, refer to <>. \ No newline at end of file +For a complete list of all operators, refer to <>. diff --git a/docs/reference/esql/source-commands/from.asciidoc b/docs/reference/esql/source-commands/from.asciidoc index 9ab21e8996aa0..a39afddefe024 100644 --- a/docs/reference/esql/source-commands/from.asciidoc +++ b/docs/reference/esql/source-commands/from.asciidoc @@ -2,6 +2,9 @@ [[esql-from]] === `FROM` +The `FROM` source command returns a table with data from a data stream, index, +or alias. + **Syntax** [source,esql] @@ -58,24 +61,22 @@ today's index: FROM ---- -Use comma-separated lists or wildcards to query multiple data streams, indices, -or aliases: +Use comma-separated lists or wildcards to <>: [source,esql] ---- FROM employees-00001,other-employees-* ---- -Use the format `:` to query data streams and indices -on remote clusters: +Use the format `:` to <>: [source,esql] ---- FROM cluster_one:employees-00001,cluster_two:other-employees-* ---- -See <>. - Use the optional `METADATA` directive to enable <>: [source,esql] diff --git a/docs/reference/esql/source-commands/row.asciidoc b/docs/reference/esql/source-commands/row.asciidoc index adce844f365b8..28a4f29ae9a5b 100644 --- a/docs/reference/esql/source-commands/row.asciidoc +++ b/docs/reference/esql/source-commands/row.asciidoc @@ -2,6 +2,9 @@ [[esql-row]] === `ROW` +The `ROW` source command produces a row with one or more columns with values +that you specify. This can be useful for testing. + **Syntax** [source,esql] @@ -13,16 +16,12 @@ ROW column1 = value1[, ..., columnN = valueN] `columnX`:: The column name. +In case of duplicate column names, only the rightmost duplicate creates a column. `valueX`:: The value for the column. Can be a literal, an expression, or a <>. -*Description* - -The `ROW` source command produces a row with one or more columns with values -that you specify. This can be useful for testing. - *Examples* [source.merge.styled,esql] diff --git a/docs/reference/esql/source-commands/show.asciidoc b/docs/reference/esql/source-commands/show.asciidoc index 298ea5d8f92b9..7090ab790133f 100644 --- a/docs/reference/esql/source-commands/show.asciidoc +++ b/docs/reference/esql/source-commands/show.asciidoc @@ -2,6 +2,9 @@ [[esql-show]] === `SHOW` +The `SHOW` source command returns information about the deployment and +its capabilities. + **Syntax** [source,esql] @@ -14,15 +17,10 @@ SHOW item `item`:: Can only be `INFO`. -*Description* - -The `SHOW` source command returns information about the deployment and -its capabilities: - -* Use `SHOW INFO` to return the deployment's version, build date and hash. - *Examples* +Use `SHOW INFO` to return the deployment's version, build date and hash. + [source,esql] ---- SHOW INFO diff --git a/docs/reference/features/apis/reset-features-api.asciidoc b/docs/reference/features/apis/reset-features-api.asciidoc index 2d2c7da039ea1..36ff12cf0fc33 100644 --- a/docs/reference/features/apis/reset-features-api.asciidoc +++ b/docs/reference/features/apis/reset-features-api.asciidoc @@ -26,7 +26,7 @@ POST /_features/_reset Return a cluster to the same state as a new installation by resetting the feature state for all {es} features. This deletes all state information stored in system indices. -The response code is `HTTP 200` if state is successfully reset for all features, `HTTP 207` if there is a mixture of successes and failures, and `HTTP 500` if the reset operation fails for all features. +The response code is `HTTP 200` if state is successfully reset for all features, or `HTTP 500` if the reset operation failed for any feature. Note that select features might provide a way to reset particular system indices. Using this API resets _all_ features, both those that are built-in and implemented as plugins. diff --git a/docs/reference/geospatial-analysis.asciidoc b/docs/reference/geospatial-analysis.asciidoc index 7577bb222127f..6760040e14bc7 100644 --- a/docs/reference/geospatial-analysis.asciidoc +++ b/docs/reference/geospatial-analysis.asciidoc @@ -2,7 +2,7 @@ [[geospatial-analysis]] = Geospatial analysis -Did you know that {es} has geospatial capabilities? https://www.elastic.co/blog/geo-location-and-search[{es} and geo] go way back, to 2010. A lot has happened since then and today {es} provides robust geospatial capabilities with speed, all with a stack that scales automatically. +Did you know that {es} has geospatial capabilities? https://www.elastic.co/blog/geo-location-and-search[{es} and geo] go way back, to 2010. A lot has happened since then and today {es} provides robust geospatial capabilities with speed, all with a stack that scales automatically. Not sure where to get started with {es} and geo? Then, you have come to the right place. @@ -18,8 +18,10 @@ Have an index with lat/lon pairs but no geo_point mapping? Use <> lets you clean, transform, and augment your data before indexing. +Data is often messy and incomplete. <> lets you clean, transform, and augment your data before indexing. +* Use <> together with <> to index CSV files with geo data. + Kibana's {kibana-ref}/import-geospatial-data.html[Import CSV] feature can help with this. * Use <> to add geographical location of an IPv4 or IPv6 address. * Use <> to convert grid tiles or hexagonal cell ids to bounding boxes or polygons which describe their shape. * Use <> for reverse geocoding. For example, use {kibana-ref}/reverse-geocoding-tutorial.html[reverse geocoding] to visualize metropolitan areas by web traffic. @@ -30,6 +32,18 @@ Data is often messy and incomplete. <> lets you clean, <> answer location-driven questions. Find documents that intersect with, are within, are contained by, or do not intersect your query geometry. Combine geospatial queries with full text search queries for unparalleled searching experience. For example, "Show me all subscribers that live within 5 miles of our new gym location, that joined in the last year and have running mentioned in their profile". +[discrete] +[[esql-query]] +=== ES|QL + +<> has support for <> functions, enabling efficient index searching for documents that intersect with, are within, are contained by, or are disjoint from a query geometry. In addition, the `ST_DISTANCE` function calculates the distance between two points. + +* experimental:[] <> +* experimental:[] <> +* experimental:[] <> +* experimental:[] <> +* experimental:[] <> + [discrete] [[geospatial-aggregate]] === Aggregate @@ -42,12 +56,12 @@ Geospatial bucket aggregations: * <> groups geo_point and geo_shape values into buckets that represent a grid. * <> groups geo_point and geo_shape values into buckets that represent an H3 hexagonal cell. * <> groups geo_point and geo_shape values into buckets that represent a grid. Each cell corresponds to a {wikipedia}/Tiled_web_map[map tile] as used by many online map sites. - + Geospatial metric aggregations: * <> computes the geographic bounding box containing all values for a Geopoint or Geoshape field. * <> computes the weighted centroid from all coordinate values for geo fields. -* <> aggregates all geo_point values within a bucket into a LineString ordered by the chosen sort field. Use geo_line aggregation to create {kibana-ref}/asset-tracking-tutorial.html[vehicle tracks]. +* <> aggregates all geo_point values within a bucket into a LineString ordered by the chosen sort field. Use geo_line aggregation to create {kibana-ref}/asset-tracking-tutorial.html[vehicle tracks]. Combine aggregations to perform complex geospatial analysis. For example, to calculate the most recent GPS tracks per flight, use a <> to group documents into buckets per aircraft. Then use geo-line aggregation to compute a track for each aircraft. In another example, use geotile grid aggregation to group documents into a grid. Then use geo-centroid aggregation to find the weighted centroid of each grid cell. @@ -79,4 +93,4 @@ Put machine learning to work for you and find the data that should stand out wit Let your location data drive insights and action with {kibana-ref}/geo-alerting.html[geographic alerts]. Commonly referred to as geo-fencing, track moving objects as they enter or exit a boundary to receive notifications through common business systems (email, Slack, Teams, PagerDuty, and more). -Interested in learning more? Follow {kibana-ref}/asset-tracking-tutorial.html[step-by-step instructions] for setting up tracking containment alerts to monitor moving vehicles. \ No newline at end of file +Interested in learning more? Follow {kibana-ref}/asset-tracking-tutorial.html[step-by-step instructions] for setting up tracking containment alerts to monitor moving vehicles. diff --git a/docs/reference/how-to/size-your-shards.asciidoc b/docs/reference/how-to/size-your-shards.asciidoc index 56e5fbbf15c77..ba0c0ab8b0b15 100644 --- a/docs/reference/how-to/size-your-shards.asciidoc +++ b/docs/reference/how-to/size-your-shards.asciidoc @@ -501,6 +501,7 @@ POST _reindex Here’s how to resolve common shard-related errors. [discrete] +[[troubleshooting-max-shards-open]] ==== this action would add [x] total shards, but this cluster currently has [y]/[z] maximum shards open; The <> cluster @@ -544,3 +545,42 @@ PUT _cluster/settings } } ---- + +[discrete] +[[troubleshooting-max-docs-limit]] +==== Number of documents in the shard cannot exceed [2147483519] + +Each {es} shard is a separate Lucene index, so it shares Lucene's +https://github.com/apache/lucene/issues/5176[`MAX_DOC` limit] of having at most +2,147,483,519 (`(2^31)-129`) documents. This per-shard limit applies to the sum +of `docs.count` plus `docs.deleted` as reported by the <>. Exceeding this limit will result in errors like the following: + +[source,txt] +---- +Elasticsearch exception [type=illegal_argument_exception, reason=Number of documents in the shard cannot exceed [2147483519]] +---- + +TIP: This calculation may differ from the <> +calculation, because the Count API does not include nested documents and does +not count deleted documents. + +This limit is much higher than the <> of approximately 200M documents per shard. + +If you encounter this problem, try to mitigate it by using the +<> to merge away some deleted docs. For +example: + +[source,console] +---- +POST my-index-000001/_forcemerge?only_expunge_deletes=true +---- +// TEST[setup:my_index] + +This will launch an asynchronous task which can be monitored via the +<>. + +It may also be helpful to <>, +or to <> or <> the index into +one with a larger number of shards. diff --git a/docs/reference/indices/forcemerge.asciidoc b/docs/reference/indices/forcemerge.asciidoc index 1d473acbd5d48..6eacaac5e7b2a 100644 --- a/docs/reference/indices/forcemerge.asciidoc +++ b/docs/reference/indices/forcemerge.asciidoc @@ -89,8 +89,9 @@ one at a time. If you expand the `force_merge` threadpool on a node then it will force merge its shards in parallel. Force merge makes the storage for the shard being merged temporarily -increase, up to double its size in case `max_num_segments` parameter is set to -`1`, as all segments need to be rewritten into a new one. +increase, as it may require free space up to triple its size in case +`max_num_segments` parameter is set to `1`, to rewrite all segments into a new +one. [[forcemerge-api-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc index 02a57504da1cf..9c75820a8f92b 100644 --- a/docs/reference/inference/inference-apis.asciidoc +++ b/docs/reference/inference/inference-apis.asciidoc @@ -25,6 +25,7 @@ include::delete-inference.asciidoc[] include::get-inference.asciidoc[] include::post-inference.asciidoc[] include::put-inference.asciidoc[] +include::service-amazon-bedrock.asciidoc[] include::service-azure-ai-studio.asciidoc[] include::service-azure-openai.asciidoc[] include::service-cohere.asciidoc[] diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc index 656feb54ffe42..948496c473a20 100644 --- a/docs/reference/inference/put-inference.asciidoc +++ b/docs/reference/inference/put-inference.asciidoc @@ -11,7 +11,6 @@ IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models. However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <>. - [discrete] [[put-inference-api-request]] ==== {api-request-title} @@ -25,7 +24,6 @@ However, if you do not plan to use the {infer} APIs to use these models or if yo * Requires the `manage_inference` <> (the built-in `inference_admin` role grants this privilege) - [discrete] [[put-inference-api-desc]] ==== {api-description-title} @@ -34,6 +32,7 @@ The create {infer} API enables you to create an {infer} endpoint and configure a The following services are available through the {infer} API, click the links to review the configuration details of the services: +* <> * <> * <> * <> diff --git a/docs/reference/inference/service-amazon-bedrock.asciidoc b/docs/reference/inference/service-amazon-bedrock.asciidoc new file mode 100644 index 0000000000000..4ffa368613a0e --- /dev/null +++ b/docs/reference/inference/service-amazon-bedrock.asciidoc @@ -0,0 +1,175 @@ +[[infer-service-amazon-bedrock]] +=== Amazon Bedrock {infer} service + +Creates an {infer} endpoint to perform an {infer} task with the `amazonbedrock` service. + +[discrete] +[[infer-service-amazon-bedrock-api-request]] +==== {api-request-title} + +`PUT /_inference//` + +[discrete] +[[infer-service-amazon-bedrock-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +include::inference-shared.asciidoc[tag=inference-id] + +``:: +(Required, string) +include::inference-shared.asciidoc[tag=task-type] ++ +-- +Available task types: + +* `completion`, +* `text_embedding`. +-- + +[discrete] +[[infer-service-amazon-bedrock-api-request-body]] +==== {api-request-body-title} + +`service`:: +(Required, string) The type of service supported for the specified task type. +In this case, +`amazonbedrock`. + +`service_settings`:: +(Required, object) +include::inference-shared.asciidoc[tag=service-settings] ++ +-- +These settings are specific to the `amazonbedrock` service. +-- + +`access_key`::: +(Required, string) +A valid AWS access key that has permissions to use Amazon Bedrock and access to models for inference requests. + +`secret_key`::: +(Required, string) +A valid AWS secret key that is paired with the `access_key`. +To create or manage access and secret keys, see https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html[Managing access keys for IAM users] in the AWS documentation. + +IMPORTANT: You need to provide the access and secret keys only once, during the {infer} model creation. +The <> does not retrieve your access or secret keys. +After creating the {infer} model, you cannot change the associated key pairs. +If you want to use a different access and secret key pair, delete the {infer} model and recreate it with the same name and the updated keys. + +`provider`::: +(Required, string) +The model provider for your deployment. +Note that some providers may support only certain task types. +Supported providers include: + +* `amazontitan` - available for `text_embedding` and `completion` task types +* `anthropic` - available for `completion` task type only +* `ai21labs` - available for `completion` task type only +* `cohere` - available for `text_embedding` and `completion` task types +* `meta` - available for `completion` task type only +* `mistral` - available for `completion` task type only + +`model`::: +(Required, string) +The base model ID or an ARN to a custom model based on a foundational model. +The base model IDs can be found in the https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html[Amazon Bedrock model IDs] documentation. +Note that the model ID must be available for the provider chosen, and your IAM user must have access to the model. + +`region`::: +(Required, string) +The region that your model or ARN is deployed in. +The list of available regions per model can be found in the https://docs.aws.amazon.com/bedrock/latest/userguide/models-regions.html[Model support by AWS region] documentation. + +`rate_limit`::: +(Optional, object) +By default, the `amazonbedrock` service sets the number of requests allowed per minute to `240`. +This helps to minimize the number of rate limit errors returned from Amazon Bedrock. +To modify this, set the `requests_per_minute` setting of this object in your service settings: ++ +-- +include::inference-shared.asciidoc[tag=request-per-minute-example] +-- + +`task_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=task-settings] ++ +.`task_settings` for the `completion` task type +[%collapsible%closed] +===== + +`max_new_tokens`::: +(Optional, integer) +Sets the maximum number for the output tokens to be generated. +Defaults to 64. + +`temperature`::: +(Optional, float) +A number between 0.0 and 1.0 that controls the apparent creativity of the results. At temperature 0.0 the model is most deterministic, at temperature 1.0 most random. +Should not be used if `top_p` or `top_k` is specified. + +`top_p`::: +(Optional, float) +Alternative to `temperature`. A number in the range of 0.0 to 1.0, to eliminate low-probability tokens. Top-p uses nucleus sampling to select top tokens whose sum of likelihoods does not exceed a certain value, ensuring both variety and coherence. +Should not be used if `temperature` is specified. + +`top_k`::: +(Optional, float) +Only available for `anthropic`, `cohere`, and `mistral` providers. +Alternative to `temperature`. Limits samples to the top-K most likely words, balancing coherence and variability. +Should not be used if `temperature` is specified. + +===== ++ +.`task_settings` for the `text_embedding` task type +[%collapsible%closed] +===== + +There are no `task_settings` available for the `text_embedding` task type. + +===== + +[discrete] +[[inference-example-amazonbedrock]] +==== Amazon Bedrock service example + +The following example shows how to create an {infer} endpoint called `amazon_bedrock_embeddings` to perform a `text_embedding` task type. + +Choose chat completion and embeddings models that you have access to from the https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html[Amazon Bedrock base models]. + +[source,console] +------------------------------------------------------------ +PUT _inference/text_embedding/amazon_bedrock_embeddings +{ + "service": "amazonbedrock", + "service_settings": { + "access_key": "", + "secret_key": "", + "region": "us-east-1", + "provider": "amazontitan", + "model": "amazon.titan-embed-text-v2:0" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] + +The next example shows how to create an {infer} endpoint called `amazon_bedrock_completion` to perform a `completion` task type. + +[source,console] +------------------------------------------------------------ +PUT _inference/completion/amazon_bedrock_completion +{ + "service": "amazonbedrock", + "service_settings": { + "access_key": "", + "secret_key": "", + "region": "us-east-1", + "provider": "amazontitan", + "model": "amazon.titan-text-premier-v1:0" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] diff --git a/docs/reference/inference/service-anthropic.asciidoc b/docs/reference/inference/service-anthropic.asciidoc new file mode 100644 index 0000000000000..41419db7a6069 --- /dev/null +++ b/docs/reference/inference/service-anthropic.asciidoc @@ -0,0 +1,124 @@ +[[infer-service-anthropic]] +=== Anthropic {infer} service + +Creates an {infer} endpoint to perform an {infer} task with the `anthropic` service. + + +[discrete] +[[infer-service-anthropic-api-request]] +==== {api-request-title} + +`PUT /_inference//` + +[discrete] +[[infer-service-anthropic-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +include::inference-shared.asciidoc[tag=inference-id] + +``:: +(Required, string) +include::inference-shared.asciidoc[tag=task-type] ++ +-- +Available task types: + +* `completion` +-- + +[discrete] +[[infer-service-anthropic-api-request-body]] +==== {api-request-body-title} + +`service`:: +(Required, string) +The type of service supported for the specified task type. In this case, +`anthropic`. + +`service_settings`:: +(Required, object) +include::inference-shared.asciidoc[tag=service-settings] ++ +-- +These settings are specific to the `anthropic` service. +-- + +`api_key`::: +(Required, string) +A valid API key for the Anthropic API. + +`model_id`::: +(Required, string) +The name of the model to use for the {infer} task. +You can find the supported models at https://docs.anthropic.com/en/docs/about-claude/models#model-names[Anthropic models]. + +`rate_limit`::: +(Optional, object) +By default, the `anthropic` service sets the number of requests allowed per minute to `50`. +This helps to minimize the number of rate limit errors returned from Anthropic. +To modify this, set the `requests_per_minute` setting of this object in your service settings: ++ +-- +include::inference-shared.asciidoc[tag=request-per-minute-example] +-- + +`task_settings`:: +(Required, object) +include::inference-shared.asciidoc[tag=task-settings] ++ +.`task_settings` for the `completion` task type +[%collapsible%closed] +===== +`max_tokens`::: +(Required, integer) +The maximum number of tokens to generate before stopping. + +`temperature`::: +(Optional, float) +The amount of randomness injected into the response. ++ +For more details about the supported range, see the https://docs.anthropic.com/en/api/messages[Anthropic messages API]. + +`top_k`::: +(Optional, integer) +Specifies to only sample from the top K options for each subsequent token. ++ +Recommended for advanced use cases only. You usually only need to use `temperature`. ++ +For more details, see the https://docs.anthropic.com/en/api/messages[Anthropic messages API]. + +`top_p`::: +(Optional, float) +Specifies to use Anthropic's nucleus sampling. ++ +In nucleus sampling, Anthropic computes the cumulative distribution over all the options for each subsequent token in decreasing probability order and cut it off once it reaches a particular probability specified by `top_p`. You should either alter `temperature` or `top_p`, but not both. ++ +Recommended for advanced use cases only. You usually only need to use `temperature`. ++ +For more details, see the https://docs.anthropic.com/en/api/messages[Anthropic messages API]. +===== + +[discrete] +[[inference-example-anthropic]] +==== Anthropic service example + +The following example shows how to create an {infer} endpoint called +`anthropic_completion` to perform a `completion` task type. + +[source,console] +------------------------------------------------------------ +PUT _inference/completion/anthropic_completion +{ + "service": "anthropic", + "service_settings": { + "api_key": "", + "model_id": "" + }, + "task_settings": { + "max_tokens": 1024 + } +} +------------------------------------------------------------ +// TEST[skip:TBD] diff --git a/docs/reference/inference/service-elasticsearch.asciidoc b/docs/reference/inference/service-elasticsearch.asciidoc index 3b9b5b1928d7b..50b97b3506ee8 100644 --- a/docs/reference/inference/service-elasticsearch.asciidoc +++ b/docs/reference/inference/service-elasticsearch.asciidoc @@ -35,7 +35,7 @@ Available task types: `service`:: (Required, string) -The type of service supported for the specified task type. In this case, +The type of service supported for the specified task type. In this case, `elasticsearch`. `service_settings`:: @@ -58,7 +58,7 @@ The total number of allocations this model is assigned across machine learning n `num_threads`::: (Required, integer) -Sets the number of threads used by each model allocation during inference. This generally increases the speed per inference request. The inference process is a compute-bound process; `threads_per_allocations` must not exceed the number of available allocated processors per node. +Sets the number of threads used by each model allocation during inference. This generally increases the speed per inference request. The inference process is a compute-bound process; `threads_per_allocations` must not exceed the number of available allocated processors per node. Must be a power of 2. Max allowed value is 32. `task_settings`:: @@ -98,6 +98,14 @@ PUT _inference/text_embedding/my-e5-model Valid values are `.multilingual-e5-small` and `.multilingual-e5-small_linux-x86_64`. For further details, refer to the {ml-docs}/ml-nlp-e5.html[E5 model documentation]. +[NOTE] +==== +You might see a 502 bad gateway error in the response when using the {kib} Console. +This error usually just reflects a timeout, while the model downloads in the background. +You can check the download progress in the {ml-app} UI. +If using the Python client, you can set the `timeout` parameter to a higher value. +==== + [discrete] [[inference-example-eland]] ==== Models uploaded by Eland via the elasticsearch service @@ -119,4 +127,4 @@ PUT _inference/text_embedding/my-msmarco-minilm-model ------------------------------------------------------------ // TEST[skip:TBD] <1> The `model_id` must be the ID of a text embedding model which has already been -{ml-docs}/ml-nlp-import-model.html#ml-nlp-import-script[uploaded through Eland]. \ No newline at end of file +{ml-docs}/ml-nlp-import-model.html#ml-nlp-import-script[uploaded through Eland]. diff --git a/docs/reference/inference/service-elser.asciidoc b/docs/reference/inference/service-elser.asciidoc index 829ff4968c5be..dff531f2a414b 100644 --- a/docs/reference/inference/service-elser.asciidoc +++ b/docs/reference/inference/service-elser.asciidoc @@ -34,7 +34,7 @@ Available task types: `service`:: (Required, string) -The type of service supported for the specified task type. In this case, +The type of service supported for the specified task type. In this case, `elser`. `service_settings`:: @@ -51,7 +51,7 @@ The total number of allocations this model is assigned across machine learning n `num_threads`::: (Required, integer) -Sets the number of threads used by each model allocation during inference. This generally increases the speed per inference request. The inference process is a compute-bound process; `threads_per_allocations` must not exceed the number of available allocated processors per node. +Sets the number of threads used by each model allocation during inference. This generally increases the speed per inference request. The inference process is a compute-bound process; `threads_per_allocations` must not exceed the number of available allocated processors per node. Must be a power of 2. Max allowed value is 32. @@ -92,4 +92,12 @@ Example response: "task_settings": {} } ------------------------------------------------------------ -// NOTCONSOLE \ No newline at end of file +// NOTCONSOLE + +[NOTE] +==== +You might see a 502 bad gateway error in the response when using the {kib} Console. +This error usually just reflects a timeout, while the model downloads in the background. +You can check the download progress in the {ml-app} UI. +If using the Python client, you can set the `timeout` parameter to a higher value. +==== diff --git a/docs/reference/ingest/apis/enrich/enrich-stats.asciidoc b/docs/reference/ingest/apis/enrich/enrich-stats.asciidoc index ad1ca62e37bbf..85273ee584b9a 100644 --- a/docs/reference/ingest/apis/enrich/enrich-stats.asciidoc +++ b/docs/reference/ingest/apis/enrich/enrich-stats.asciidoc @@ -121,6 +121,10 @@ The amount of time in milliseconds spent fetching data from the cache on success `misses_time_in_millis`:: (Long) The amount of time in milliseconds spent fetching data from the enrich index and updating the cache, on cache misses only. + +`size_in_bytes`:: +(Long) +An _approximation_ of the size in bytes that the enrich cache takes up on the heap. -- [[enrich-stats-api-example]] @@ -172,7 +176,8 @@ The API returns the following response: "misses": 0, "evictions": 0, "hits_time_in_millis": 0, - "misses_time_in_millis": 0 + "misses_time_in_millis": 0, + "size_in_bytes": 0 } ] } @@ -187,3 +192,4 @@ The API returns the following response: // TESTRESPONSE[s/"evictions": 0/"evictions" : $body.cache_stats.0.evictions/] // TESTRESPONSE[s/"hits_time_in_millis": 0/"hits_time_in_millis" : $body.cache_stats.0.hits_time_in_millis/] // TESTRESPONSE[s/"misses_time_in_millis": 0/"misses_time_in_millis" : $body.cache_stats.0.misses_time_in_millis/] +// TESTRESPONSE[s/"size_in_bytes": 0/"size_in_bytes" : $body.cache_stats.0.size_in_bytes/] diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc index f2f0b3ae8bb23..0cd9ee0578b70 100644 --- a/docs/reference/mapping/types/dense-vector.asciidoc +++ b/docs/reference/mapping/types/dense-vector.asciidoc @@ -448,3 +448,63 @@ POST /my-bit-vectors/_search?filter_path=hits.hits } ---- +==== Updatable field type + +To better accommodate scaling and performance needs, updating the `type` setting in `index_options` is possible with the <>, according to the following graph (jumps allowed): + +[source,txt] +---- +flat --> int8_flat --> int4_flat --> hnsw --> int8_hnsw --> int4_hnsw +---- + +For updating all HNSW types (`hnsw`, `int8_hnsw`, `int4_hnsw`) the number of connections `m` must either stay the same or increase. For scalar quantized formats (`int8_flat`, `int4_flat`, `int8_hnsw`, `int4_hnsw`) the `confidence_interval` must always be consistent (once defined, it cannot change). + +Updating `type` in `index_options` will fail in all other scenarios. + +Switching `types` won't re-index vectors that have already been indexed (they will keep using their original `type`), vectors being indexed after the change will use the new `type` instead. + +For example, it's possible to define a dense vector field that utilizes the `flat` type (raw float32 arrays) for a first batch of data to be indexed. + +[source,console] +-------------------------------------------------- +PUT my-index-000001 +{ + "mappings": { + "properties": { + "text_embedding": { + "type": "dense_vector", + "dims": 384, + "index_options": { + "type": "flat" + } + } + } + } +} +-------------------------------------------------- + +Changing the `type` to `int4_hnsw` makes sure vectors indexed after the change will use an int4 scalar quantized representation and HNSW (e.g., for KNN queries). +That includes new segments created by <> previously created segments. + +[source,console] +-------------------------------------------------- +PUT /my-index-000001/_mapping +{ + "properties": { + "text_embedding": { + "type": "dense_vector", + "dims": 384, + "index_options": { + "type": "int4_hnsw" + } + } + } +} +-------------------------------------------------- +// TEST[setup:my_index] + +Vectors indexed before this change will keep using the `flat` type (raw float32 representation and brute force search for KNN queries). + +In order to have all the vectors updated to the new type, either reindexing or force merging should be used. + +For debugging purposes, it's possible to inspect how many segments (and docs) exist for each `type` with the <>. diff --git a/docs/reference/mapping/types/rank-features.asciidoc b/docs/reference/mapping/types/rank-features.asciidoc index b54e99ede3fae..25d5278ca220d 100644 --- a/docs/reference/mapping/types/rank-features.asciidoc +++ b/docs/reference/mapping/types/rank-features.asciidoc @@ -70,6 +70,15 @@ GET my-index-000001/_search } } } + +GET my-index-000001/_search +{ + "query": { <6> + "term": { + "topics": "economics" + } + } +} -------------------------------------------------- <1> Rank features fields must use the `rank_features` field type @@ -77,6 +86,7 @@ GET my-index-000001/_search <3> Rank features fields must be a hash with string keys and strictly positive numeric values <4> This query ranks documents by how much they are about the "politics" topic. <5> This query ranks documents inversely to the number of "1star" reviews they received. +<6> This query returns documents that store the "economics" feature in the "topics" field. NOTE: `rank_features` fields only support single-valued features and strictly diff --git a/docs/reference/ml/anomaly-detection/functions/ml-geo-functions.asciidoc b/docs/reference/ml/anomaly-detection/functions/ml-geo-functions.asciidoc index 5c061daa1cd44..63a0f047db647 100644 --- a/docs/reference/ml/anomaly-detection/functions/ml-geo-functions.asciidoc +++ b/docs/reference/ml/anomaly-detection/functions/ml-geo-functions.asciidoc @@ -52,6 +52,12 @@ detects anomalies where the geographic location of a credit card transaction is unusual for a particular customer’s credit card. An anomaly might indicate fraud. +A "typical" value indicates a centroid of a cluster of previously observed +locations that is closest to the "actual" location at that time. For example, +there may be one centroid near the person's home that is associated with the +cluster of local grocery stores and restaurants, and another centroid near the +person's work associated with the cluster of lunch and coffee places. + IMPORTANT: The `field_name` that you supply must be a single string that contains two comma-separated numbers of the form `latitude,longitude`, a `geo_point` field, a `geo_shape` field that contains point values, or a diff --git a/docs/reference/modules/cluster/misc.asciidoc b/docs/reference/modules/cluster/misc.asciidoc index 3da5df4f16414..75eaca88c66b1 100644 --- a/docs/reference/modules/cluster/misc.asciidoc +++ b/docs/reference/modules/cluster/misc.asciidoc @@ -11,12 +11,12 @@ An entire cluster may be set to read-only with the following setting: (<>) Make the whole cluster read only (indices do not accept write operations), metadata is not allowed to be modified (create or delete - indices). + indices). Defaults to `false`. `cluster.blocks.read_only_allow_delete`:: (<>) Identical to `cluster.blocks.read_only` but allows to delete indices - to free up resources. + to free up resources. Defaults to `false`. WARNING: Don't rely on this setting to prevent changes to your cluster. Any user with access to the <> diff --git a/docs/reference/modules/cluster/shards_allocation.asciidoc b/docs/reference/modules/cluster/shards_allocation.asciidoc index 1e425c77d1264..dc53837125ee9 100644 --- a/docs/reference/modules/cluster/shards_allocation.asciidoc +++ b/docs/reference/modules/cluster/shards_allocation.asciidoc @@ -98,9 +98,9 @@ the cluster: Specify when shard rebalancing is allowed: -* `always` - Always allow rebalancing. +* `always` - (default) Always allow rebalancing. * `indices_primaries_active` - Only when all primaries in the cluster are allocated. -* `indices_all_active` - (default) Only when all shards (primaries and replicas) in the cluster are allocated. +* `indices_all_active` - Only when all shards (primaries and replicas) in the cluster are allocated. -- `cluster.routing.rebalance.enable`:: diff --git a/docs/reference/modules/indices/search-settings.asciidoc b/docs/reference/modules/indices/search-settings.asciidoc index e43ec076578d4..003974815c4bd 100644 --- a/docs/reference/modules/indices/search-settings.asciidoc +++ b/docs/reference/modules/indices/search-settings.asciidoc @@ -33,6 +33,39 @@ a single response. Defaults to 65,536. + Requests that attempt to return more than this limit will return an error. +[[search-settings-only-allowed-scripts]] +`search.aggs.only_allowed_metric_scripts`:: +(<>, boolean) +Configures whether only explicitly allowed scripts can be used in +<>. +Defaults to `false`. ++ +Requests using scripts not contained in either +<> +or +<> +will return an error. + +[[search-settings-allowed-inline-scripts]] +`search.aggs.allowed_inline_metric_scripts`:: +(<>, list of strings) +List of inline scripts that can be used in scripted metrics aggregations when +<> +is set to `true`. +Defaults to an empty list. ++ +Requests using other inline scripts will return an error. + +[[search-settings-allowed-stored-scripts]] +`search.aggs.allowed_stored_metric_scripts`:: +(<>, list of strings) +List of ids of stored scripts that can be used in scripted metrics aggregations when +<> +is set to `true`. +Defaults to an empty list. ++ +Requests using other stored scripts will return an error. + [[indices-query-bool-max-nested-depth]] `indices.query.bool.max_nested_depth`:: (<>, integer) Maximum nested depth of queries. Defaults to `30`. diff --git a/docs/reference/query-dsl.asciidoc b/docs/reference/query-dsl.asciidoc index 4d5504e5fe7ae..2f8f07f21f648 100644 --- a/docs/reference/query-dsl.asciidoc +++ b/docs/reference/query-dsl.asciidoc @@ -72,14 +72,12 @@ include::query-dsl/match-all-query.asciidoc[] include::query-dsl/span-queries.asciidoc[] +include::query-dsl/vector-queries.asciidoc[] + include::query-dsl/special-queries.asciidoc[] include::query-dsl/term-level-queries.asciidoc[] -include::query-dsl/text-expansion-query.asciidoc[] - -include::query-dsl/sparse-vector-query.asciidoc[] - include::query-dsl/minimum-should-match.asciidoc[] include::query-dsl/multi-term-rewrite.asciidoc[] diff --git a/docs/reference/query-dsl/sparse-vector-query.asciidoc b/docs/reference/query-dsl/sparse-vector-query.asciidoc index 80616ff174e36..08dd7ab7f4470 100644 --- a/docs/reference/query-dsl/sparse-vector-query.asciidoc +++ b/docs/reference/query-dsl/sparse-vector-query.asciidoc @@ -1,5 +1,5 @@ [[query-dsl-sparse-vector-query]] -== Sparse vector query +=== Sparse vector query ++++ Sparse vector @@ -19,7 +19,7 @@ For example, a stored vector `{"feature_0": 0.12, "feature_1": 1.2, "feature_2": [discrete] [[sparse-vector-query-ex-request]] -=== Example request using an {nlp} model +==== Example request using an {nlp} model [source,console] ---- @@ -37,7 +37,7 @@ GET _search // TEST[skip: Requires inference] [discrete] -=== Example request using precomputed vectors +==== Example request using precomputed vectors [source,console] ---- @@ -55,7 +55,7 @@ GET _search [discrete] [[sparse-vector-field-params]] -=== Top level parameters for `sparse_vector` +==== Top level parameters for `sparse_vector` `field`:: (Required, string) The name of the field that contains the token-weight pairs to be searched against. @@ -120,7 +120,7 @@ NOTE: The default values for `tokens_freq_ratio_threshold` and `tokens_weight_th [discrete] [[sparse-vector-query-example]] -=== Example ELSER query +==== Example ELSER query The following is an example of the `sparse_vector` query that references the ELSER model to perform semantic search. For a more detailed description of how to perform semantic search by using ELSER and the `sparse_vector` query, refer to <>. @@ -241,7 +241,7 @@ GET my-index/_search [discrete] [[sparse-vector-query-with-pruning-config-and-rescore-example]] -=== Example ELSER query with pruning configuration and rescore +==== Example ELSER query with pruning configuration and rescore The following is an extension to the above example that adds a preview:[] pruning configuration to the `sparse_vector` query. The pruning configuration identifies non-significant tokens to prune from the query in order to improve query performance. diff --git a/docs/reference/query-dsl/special-queries.asciidoc b/docs/reference/query-dsl/special-queries.asciidoc index 90cd9a696a6d9..a6d35d4f9b707 100644 --- a/docs/reference/query-dsl/special-queries.asciidoc +++ b/docs/reference/query-dsl/special-queries.asciidoc @@ -17,10 +17,6 @@ or collection of documents. This query finds queries that are stored as documents that match with the specified document. -<>:: -A query that finds the _k_ nearest vectors to a query -vector, as measured by a similarity metric. - <>:: A query that computes scores based on the values of numeric features and is able to efficiently skip non-competitive hits. @@ -32,9 +28,6 @@ This query allows a script to act as a filter. Also see the <>:: A query that allows to modify the score of a sub-query with a script. -<>:: -A query that allows you to perform semantic search. - <>:: A query that accepts other queries as json or yaml string. @@ -50,20 +43,14 @@ include::mlt-query.asciidoc[] include::percolate-query.asciidoc[] -include::knn-query.asciidoc[] - include::rank-feature-query.asciidoc[] include::script-query.asciidoc[] include::script-score-query.asciidoc[] -include::semantic-query.asciidoc[] - include::wrapper-query.asciidoc[] include::pinned-query.asciidoc[] include::rule-query.asciidoc[] - -include::weighted-tokens-query.asciidoc[] diff --git a/docs/reference/query-dsl/text-expansion-query.asciidoc b/docs/reference/query-dsl/text-expansion-query.asciidoc index 1c51429b5aa22..8faecad1dbdb9 100644 --- a/docs/reference/query-dsl/text-expansion-query.asciidoc +++ b/docs/reference/query-dsl/text-expansion-query.asciidoc @@ -1,5 +1,5 @@ [[query-dsl-text-expansion-query]] -== Text expansion query +=== Text expansion query ++++ Text expansion @@ -12,7 +12,7 @@ The text expansion query uses a {nlp} model to convert the query text into a lis [discrete] [[text-expansion-query-ex-request]] -=== Example request +==== Example request [source,console] ---- @@ -32,14 +32,14 @@ GET _search [discrete] [[text-expansion-query-params]] -=== Top level parameters for `text_expansion` +==== Top level parameters for `text_expansion` ``::: (Required, object) The name of the field that contains the token-weight pairs the NLP model created based on the input text. [discrete] [[text-expansion-rank-feature-field-params]] -=== Top level parameters for `` +==== Top level parameters for `` `model_id`:::: (Required, string) The ID of the model to use to convert the query text into token-weight pairs. @@ -84,7 +84,7 @@ NOTE: The default values for `tokens_freq_ratio_threshold` and `tokens_weight_th [discrete] [[text-expansion-query-example]] -=== Example ELSER query +==== Example ELSER query The following is an example of the `text_expansion` query that references the ELSER model to perform semantic search. For a more detailed description of how to perform semantic search by using ELSER and the `text_expansion` query, refer to <>. @@ -208,7 +208,7 @@ GET my-index/_search [discrete] [[text-expansion-query-with-pruning-config-and-rescore-example]] -=== Example ELSER query with pruning configuration and rescore +==== Example ELSER query with pruning configuration and rescore The following is an extension to the above example that adds a preview:[] pruning configuration to the `text_expansion` query. The pruning configuration identifies non-significant tokens to prune from the query in order to improve query performance. diff --git a/docs/reference/query-dsl/vector-queries.asciidoc b/docs/reference/query-dsl/vector-queries.asciidoc new file mode 100644 index 0000000000000..fe9f380eeb621 --- /dev/null +++ b/docs/reference/query-dsl/vector-queries.asciidoc @@ -0,0 +1,37 @@ +[[vector-queries]] +== Vector queries + +Vector queries are specialized queries that work on vector fields to efficiently perform <>. + +<>:: +A query that finds the _k_ nearest vectors to a query vector for <> fields, as measured by a similarity metric. + +<>:: +A query used to search <> field types. + +<>:: +A query that allows you to perform semantic search on <> fields. + +[discrete] +=== Deprecated vector queries + +The following queries have been deprecated and will be removed in the near future. +Use the <> query instead. + +<>:: +A query that allows you to perform sparse vector search on <> or <> fields. + +<>:: +Allows to perform text expansion queries optimizing for performance. + +include::knn-query.asciidoc[] + +include::sparse-vector-query.asciidoc[] + +include::semantic-query.asciidoc[] + +include::text-expansion-query.asciidoc[] + +include::weighted-tokens-query.asciidoc[] + + diff --git a/docs/reference/query-rules/apis/index.asciidoc b/docs/reference/query-rules/apis/index.asciidoc index f7303647f8515..53d5fc3dc4eee 100644 --- a/docs/reference/query-rules/apis/index.asciidoc +++ b/docs/reference/query-rules/apis/index.asciidoc @@ -1,8 +1,6 @@ [[query-rules-apis]] == Query rules APIs -preview::[] - ++++ Query rules APIs ++++ diff --git a/docs/reference/query-rules/apis/put-query-rule.asciidoc b/docs/reference/query-rules/apis/put-query-rule.asciidoc index 2b9a6ba892b84..9737673be009c 100644 --- a/docs/reference/query-rules/apis/put-query-rule.asciidoc +++ b/docs/reference/query-rules/apis/put-query-rule.asciidoc @@ -70,10 +70,10 @@ Matches all queries, regardless of input. -- - `metadata` (Optional, string) The metadata field to match against. This metadata will be used to match against `match_criteria` sent in the <>. -Required for all criteria types except `global`. +Required for all criteria types except `always`. - `values` (Optional, array of strings) The values to match against the metadata field. Only one value must match for the criteria to be met. -Required for all criteria types except `global`. +Required for all criteria types except `always`. `actions`:: (Required, object) The actions to take when the rule is matched. diff --git a/docs/reference/query-rules/apis/put-query-ruleset.asciidoc b/docs/reference/query-rules/apis/put-query-ruleset.asciidoc index 012060e1004ae..c164e9e140a4e 100644 --- a/docs/reference/query-rules/apis/put-query-ruleset.asciidoc +++ b/docs/reference/query-rules/apis/put-query-ruleset.asciidoc @@ -78,10 +78,10 @@ Matches all queries, regardless of input. -- - `metadata` (Optional, string) The metadata field to match against. This metadata will be used to match against `match_criteria` sent in the <>. -Required for all criteria types except `global`. +Required for all criteria types except `always`. - `values` (Optional, array of strings) The values to match against the metadata field. Only one value must match for the criteria to be met. -Required for all criteria types except `global`. +Required for all criteria types except `always`. Actions depend on the rule type. For `pinned` rules, actions follow the format specified by the <>. diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc index 20889df0c58eb..6a03ed03f2610 100644 --- a/docs/reference/release-notes.asciidoc +++ b/docs/reference/release-notes.asciidoc @@ -8,6 +8,7 @@ This section summarizes the changes in each release. * <> * <> +* <> * <> * <> * <> @@ -72,6 +73,7 @@ This section summarizes the changes in each release. include::release-notes/8.16.0.asciidoc[] include::release-notes/8.15.0.asciidoc[] +include::release-notes/8.14.3.asciidoc[] include::release-notes/8.14.2.asciidoc[] include::release-notes/8.14.1.asciidoc[] include::release-notes/8.14.0.asciidoc[] diff --git a/docs/reference/release-notes/8.12.0.asciidoc b/docs/reference/release-notes/8.12.0.asciidoc index 4c0fc50584b9f..bfa99401f41a2 100644 --- a/docs/reference/release-notes/8.12.0.asciidoc +++ b/docs/reference/release-notes/8.12.0.asciidoc @@ -14,6 +14,13 @@ there are deleted documents in the segments, quantiles may fail to build and pre This issue is fixed in 8.12.1. +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, +information about the new functionality of these upgraded nodes may not be registered properly with the master node. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. +If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. +To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes +are upgraded. This issue is fixed in 8.15.0. + [[breaking-8.12.0]] [float] === Breaking changes diff --git a/docs/reference/release-notes/8.12.1.asciidoc b/docs/reference/release-notes/8.12.1.asciidoc index 9aa9a11b3bf02..8ebe5cbac3852 100644 --- a/docs/reference/release-notes/8.12.1.asciidoc +++ b/docs/reference/release-notes/8.12.1.asciidoc @@ -3,6 +3,16 @@ Also see <>. +[[known-issues-8.12.1]] +[float] +=== Known issues +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, +information about the new functionality of these upgraded nodes may not be registered properly with the master node. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. +If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. +To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes +are upgraded. This issue is fixed in 8.15.0. + [[bug-8.12.1]] [float] === Bug fixes diff --git a/docs/reference/release-notes/8.12.2.asciidoc b/docs/reference/release-notes/8.12.2.asciidoc index 2be8449b6c1df..44202ee8226eb 100644 --- a/docs/reference/release-notes/8.12.2.asciidoc +++ b/docs/reference/release-notes/8.12.2.asciidoc @@ -3,6 +3,16 @@ Also see <>. +[[known-issues-8.12.2]] +[float] +=== Known issues +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, +information about the new functionality of these upgraded nodes may not be registered properly with the master node. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. +If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. +To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes +are upgraded. This issue is fixed in 8.15.0. + [[bug-8.12.2]] [float] === Bug fixes diff --git a/docs/reference/release-notes/8.13.0.asciidoc b/docs/reference/release-notes/8.13.0.asciidoc index 4bb2913f07be7..75e2341f33766 100644 --- a/docs/reference/release-notes/8.13.0.asciidoc +++ b/docs/reference/release-notes/8.13.0.asciidoc @@ -21,12 +21,18 @@ This affects clusters running version 8.10 or later, with an active downsampling https://www.elastic.co/guide/en/elasticsearch/reference/current/downsampling-ilm.html[configuration] or a configuration that was activated at some point since upgrading to version 8.10 or later. -* When upgrading clusters from version 8.12.2 or earlier, if your cluster contains non-master-eligible nodes, +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.13.0 not being accessible on the upgraded cluster. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. +are upgraded. This issue is fixed in 8.15.0. + +* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. +In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` +process to reclaim memory. This can cause inference requests to fail. +Elasticsearch will automatically restart the `pytorch_inference` process +after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) [[breaking-8.13.0]] [float] @@ -464,5 +470,3 @@ Search:: * Upgrade to Lucene 9.9.0 {es-pull}102782[#102782] * Upgrade to Lucene 9.9.1 {es-pull}103387[#103387] * Upgrade to Lucene 9.9.2 {es-pull}104753[#104753] - - diff --git a/docs/reference/release-notes/8.13.1.asciidoc b/docs/reference/release-notes/8.13.1.asciidoc index 572f9fe1172a9..c654af3dd5cc0 100644 --- a/docs/reference/release-notes/8.13.1.asciidoc +++ b/docs/reference/release-notes/8.13.1.asciidoc @@ -6,12 +6,18 @@ Also see <>. [[known-issues-8.13.1]] [float] === Known issues -* When upgrading clusters from version 8.12.2 or earlier, if your cluster contains non-master-eligible nodes, +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.13.0 not being accessible on the upgraded cluster. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. +are upgraded. This issue is fixed in 8.15.0. + +* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. +In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` +process to reclaim memory. This can cause inference requests to fail. +Elasticsearch will automatically restart the `pytorch_inference` process +after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) [[bug-8.13.1]] [float] @@ -45,5 +51,3 @@ Transform:: Transform:: * Raise loglevel of events related to transform lifecycle from DEBUG to INFO {es-pull}106602[#106602] - - diff --git a/docs/reference/release-notes/8.13.2.asciidoc b/docs/reference/release-notes/8.13.2.asciidoc index 20ae7abbb5769..f4540343ca9ea 100644 --- a/docs/reference/release-notes/8.13.2.asciidoc +++ b/docs/reference/release-notes/8.13.2.asciidoc @@ -6,12 +6,18 @@ Also see <>. [[known-issues-8.13.2]] [float] === Known issues -* When upgrading clusters from version 8.12.2 or earlier, if your cluster contains non-master-eligible nodes, +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.13.0 not being accessible on the upgraded cluster. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. +are upgraded. This issue is fixed in 8.15.0. + +* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. +In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` +process to reclaim memory. This can cause inference requests to fail. +Elasticsearch will automatically restart the `pytorch_inference` process +after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) [[bug-8.13.2]] [float] @@ -46,5 +52,3 @@ Packaging:: Security:: * Query API Key Information API support for the `typed_keys` request parameter {es-pull}106873[#106873] (issue: {es-issue}106817[#106817]) * Query API Keys support for both `aggs` and `aggregations` keywords {es-pull}107054[#107054] (issue: {es-issue}106839[#106839]) - - diff --git a/docs/reference/release-notes/8.13.3.asciidoc b/docs/reference/release-notes/8.13.3.asciidoc index ea51bd6f9b743..f1bb4211f4676 100644 --- a/docs/reference/release-notes/8.13.3.asciidoc +++ b/docs/reference/release-notes/8.13.3.asciidoc @@ -13,12 +13,18 @@ SQL:: [[known-issues-8.13.3]] [float] === Known issues -* When upgrading clusters from version 8.12.2 or earlier, if your cluster contains non-master-eligible nodes, +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.13.0 not being accessible on the upgraded cluster. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. +are upgraded. This issue is fixed in 8.15.0. + +* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. +In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` +process to reclaim memory. This can cause inference requests to fail. +Elasticsearch will automatically restart the `pytorch_inference` process +after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) [[bug-8.13.3]] [float] @@ -52,5 +58,3 @@ Search:: ES|QL:: * ESQL: Introduce language versioning to REST API {es-pull}106824[#106824] - - diff --git a/docs/reference/release-notes/8.13.4.asciidoc b/docs/reference/release-notes/8.13.4.asciidoc index b60c9f485bb31..446aae048945b 100644 --- a/docs/reference/release-notes/8.13.4.asciidoc +++ b/docs/reference/release-notes/8.13.4.asciidoc @@ -6,12 +6,18 @@ Also see <>. [[known-issues-8.13.4]] [float] === Known issues -* When upgrading clusters from version 8.12.2 or earlier, if your cluster contains non-master-eligible nodes, +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.13.0 not being accessible on the upgraded cluster. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. +are upgraded. This issue is fixed in 8.15.0. + +* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. +In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` +process to reclaim memory. This can cause inference requests to fail. +Elasticsearch will automatically restart the `pytorch_inference` process +after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) [[bug-8.13.4]] [float] @@ -28,5 +34,3 @@ Snapshot/Restore:: TSDB:: * Fix tsdb codec when doc-values spread in two blocks {es-pull}108276[#108276] - - diff --git a/docs/reference/release-notes/8.14.0.asciidoc b/docs/reference/release-notes/8.14.0.asciidoc index 5b92c49ced70a..c2fee6ecaa07a 100644 --- a/docs/reference/release-notes/8.14.0.asciidoc +++ b/docs/reference/release-notes/8.14.0.asciidoc @@ -15,12 +15,18 @@ Security:: [[known-issues-8.14.0]] [float] === Known issues -* When upgrading clusters from version 8.12.2 or earlier, if your cluster contains non-master-eligible nodes, +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.13.0 not being accessible on the upgraded cluster. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. +are upgraded. This issue is fixed in 8.15.0. + +* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. +In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` +process to reclaim memory. This can cause inference requests to fail. +Elasticsearch will automatically restart the `pytorch_inference` process +after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) [[bug-8.14.0]] [float] @@ -356,5 +362,3 @@ Network:: Packaging:: * Update bundled JDK to Java 22 (again) {es-pull}108654[#108654] - - diff --git a/docs/reference/release-notes/8.14.1.asciidoc b/docs/reference/release-notes/8.14.1.asciidoc index 1cab442eb9ac1..de3ecd210b488 100644 --- a/docs/reference/release-notes/8.14.1.asciidoc +++ b/docs/reference/release-notes/8.14.1.asciidoc @@ -7,12 +7,18 @@ Also see <>. [[known-issues-8.14.1]] [float] === Known issues -* When upgrading clusters from version 8.12.2 or earlier, if your cluster contains non-master-eligible nodes, +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.13.0 not being accessible on the upgraded cluster. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. +are upgraded. This issue is fixed in 8.15.0. + +* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. +In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` +process to reclaim memory. This can cause inference requests to fail. +Elasticsearch will automatically restart the `pytorch_inference` process +after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) [[bug-8.14.1]] [float] @@ -42,5 +48,3 @@ Vector Search:: Infra/Settings:: * Add remove index setting command {es-pull}109276[#109276] - - diff --git a/docs/reference/release-notes/8.14.2.asciidoc b/docs/reference/release-notes/8.14.2.asciidoc index d94067f030c61..f3f0651508dca 100644 --- a/docs/reference/release-notes/8.14.2.asciidoc +++ b/docs/reference/release-notes/8.14.2.asciidoc @@ -6,12 +6,18 @@ Also see <>. [[known-issues-8.14.2]] [float] === Known issues -* When upgrading clusters from version 8.12.2 or earlier, if your cluster contains non-master-eligible nodes, +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.13.0 not being accessible on the upgraded cluster. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. +are upgraded. This issue is fixed in 8.15.0. + +* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. +In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` +process to reclaim memory. This can cause inference requests to fail. +Elasticsearch will automatically restart the `pytorch_inference` process +after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) [[bug-8.14.2]] [float] diff --git a/docs/reference/release-notes/8.14.3.asciidoc b/docs/reference/release-notes/8.14.3.asciidoc new file mode 100644 index 0000000000000..17c53faa4a37f --- /dev/null +++ b/docs/reference/release-notes/8.14.3.asciidoc @@ -0,0 +1,32 @@ +[[release-notes-8.14.3]] +== {es} version 8.14.3 + +Also see <>. + +[[known-issues-8.14.3]] +[float] +=== Known issues +* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, +information about the new functionality of these upgraded nodes may not be registered properly with the master node. +This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. +If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. +To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes +are upgraded. This issue is fixed in 8.15.0. + +[[bug-8.14.3]] +[float] +=== Bug fixes + +Cluster Coordination:: +* Ensure tasks preserve versions in `MasterService` {es-pull}109850[#109850] + +ES|QL:: +* Introduce compute listener {es-pull}110400[#110400] + +Mapping:: +* Automatically adjust `ignore_malformed` only for the @timestamp {es-pull}109948[#109948] + +TSDB:: +* Disallow index.time_series.end_time setting from being set or updated in normal indices {es-pull}110268[#110268] (issue: {es-issue}110265[#110265]) + + diff --git a/docs/reference/release-notes/8.15.0.asciidoc b/docs/reference/release-notes/8.15.0.asciidoc index 97f4a51a1142f..c13c1c95c09ff 100644 --- a/docs/reference/release-notes/8.15.0.asciidoc +++ b/docs/reference/release-notes/8.15.0.asciidoc @@ -5,4 +5,12 @@ coming[8.15.0] Also see <>. +[[known-issues-8.15.0]] +[float] +=== Known issues +* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. +In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` +process to reclaim memory. This can cause inference requests to fail. +Elasticsearch will automatically restart the `pytorch_inference` process +after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) diff --git a/docs/reference/scripting/security.asciidoc b/docs/reference/scripting/security.asciidoc index 0f322d08726b9..249a705e92817 100644 --- a/docs/reference/scripting/security.asciidoc +++ b/docs/reference/scripting/security.asciidoc @@ -9,8 +9,8 @@ security in a defense in depth strategy for scripting. The second layer of security is the https://www.oracle.com/java/technologies/javase/seccodeguide.html[Java Security Manager]. As part of its startup sequence, {es} enables the Java Security Manager to limit the actions that -portions of the code can take. <> uses -the Java Security Manager as an additional layer of defense to prevent scripts +portions of the code can take. <> uses +the Java Security Manager as an additional layer of defense to prevent scripts from doing things like writing files and listening to sockets. {es} uses @@ -18,22 +18,28 @@ from doing things like writing files and listening to sockets. https://www.chromium.org/developers/design-documents/sandbox/osx-sandboxing-design[Seatbelt] in macOS, and https://msdn.microsoft.com/en-us/library/windows/desktop/ms684147[ActiveProcessLimit] -on Windows as additional security layers to prevent {es} from forking or +on Windows as additional security layers to prevent {es} from forking or running other processes. +Finally, scripts used in +<> +can be restricted to a defined list of scripts, or forbidden altogether. +This can prevent users from running particularly slow or resource intensive aggregation +queries. + You can modify the following script settings to restrict the type of scripts -that are allowed to run, and control the available +that are allowed to run, and control the available {painless}/painless-contexts.html[contexts] that scripts can run in. To -implement additional layers in your defense in depth strategy, follow the +implement additional layers in your defense in depth strategy, follow the <>. [[allowed-script-types-setting]] [discrete] === Allowed script types setting -{es} supports two script types: `inline` and `stored`. By default, {es} is -configured to run both types of scripts. To limit what type of scripts are run, -set `script.allowed_types` to `inline` or `stored`. To prevent any scripts from +{es} supports two script types: `inline` and `stored`. By default, {es} is +configured to run both types of scripts. To limit what type of scripts are run, +set `script.allowed_types` to `inline` or `stored`. To prevent any scripts from running, set `script.allowed_types` to `none`. IMPORTANT: If you use {kib}, set `script.allowed_types` to both or just `inline`. @@ -61,3 +67,48 @@ For example, to allow scripts to run only in `scoring` and `update` contexts: ---- script.allowed_contexts: score, update ---- + +[[allowed-script-in-aggs-settings]] +[discrete] +=== Allowed scripts in scripted metrics aggregations + +By default, all scripts are permitted in +<>. +To restrict the set of allowed scripts, set +<> +to `true` and provide the allowed scripts using +<> +and/or +<>. + +To disallow certain script types, omit the corresponding script list +(`search.aggs.allowed_inline_metric_scripts` or +`search.aggs.allowed_stored_metric_scripts`) or set it to an empty array. +When both script lists are not empty, the given stored scripts and the given inline scripts +will be allowed. + +The following example permits only 4 specific stored scripts to be used, and no inline scripts: + +[source,yaml] +---- +search.aggs.only_allowed_metric_scripts: true +search.aggs.allowed_inline_metric_scripts: [] +search.aggs.allowed_stored_metric_scripts: + - script_id_1 + - script_id_2 + - script_id_3 + - script_id_4 +---- + +Conversely, the next example allows specific inline scripts but no stored scripts: + +[source,yaml] +---- +search.aggs.only_allowed_metric_scripts: true +search.aggs.allowed_inline_metric_scripts: + - 'state.transactions = []' + - 'state.transactions.add(doc.some_field.value)' + - 'long sum = 0; for (t in state.transactions) { sum += t } return sum' + - 'long sum = 0; for (a in states) { sum += a } return sum' +search.aggs.allowed_stored_metric_scripts: [] +---- diff --git a/docs/reference/search/retriever.asciidoc b/docs/reference/search/retriever.asciidoc index 590df272cc89e..ed39ac786880b 100644 --- a/docs/reference/search/retriever.asciidoc +++ b/docs/reference/search/retriever.asciidoc @@ -28,6 +28,9 @@ A <> that replaces the functionality of a <> that produces top documents from <>. +`text_similarity_reranker`:: +A <> that enhances search results by re-ranking documents based on semantic similarity to a specified inference text, using a machine learning model. + [[standard-retriever]] ==== Standard Retriever @@ -201,6 +204,70 @@ GET /index/_search ---- // NOTCONSOLE +[[text-similarity-reranker-retriever]] +==== Text Similarity Re-ranker Retriever + +The `text_similarity_reranker` is a type of retriever that enhances search results by re-ranking documents based on semantic similarity to a specified inference text, using a machine learning model. + +===== Prerequisites + +To use `text_similarity_reranker` you must first set up a `rerank` task using the <>. +The `rerank` task should be set up with a machine learning model that can compute text similarity. +Currently you can integrate directly with the Cohere Rerank endpoint using the <> task, or upload a model to {es} <>. + +===== Parameters + +`field`:: +(Required, `string`) ++ +The document field to be used for text similarity comparisons. This field should contain the text that will be evaluated against the `inferenceText`. + +`inference_id`:: +(Required, `string`) ++ +Unique identifier of the inference endpoint created using the {infer} API. + +`inference_text`:: +(Required, `string`) ++ +The text snippet used as the basis for similarity comparison. + +`rank_window_size`:: +(Optional, `int`) ++ +The number of top documents to consider in the re-ranking process. Defaults to `10`. + +`min_score`:: +(Optional, `float`) ++ +Sets a minimum threshold score for including documents in the re-ranked results. Documents with similarity scores below this threshold will be excluded. Note that score calculations vary depending on the model used. + +===== Restrictions + +A text similarity re-ranker retriever is a compound retriever. Child retrievers may not use elements that are restricted by having a compound retriever as part of the retriever tree. + +===== Example + +[source,js] +---- +GET /index/_search +{ + "retriever": { + "text_similarity_reranker": { + "retriever": { + "standard": { ... } + } + }, + "field": "text", + "inference_id": "my-cohere-rerank-model", + "inference_text": "Most famous landmark in Paris", + "rank_window_size": 100, + "min_score": 0.5 + } +} +---- +// NOTCONSOLE + ==== Using `from` and `size` with a retriever tree The <> and <> diff --git a/docs/reference/search/search-your-data/retrievers-reranking/index.asciidoc b/docs/reference/search/search-your-data/retrievers-reranking/index.asciidoc new file mode 100644 index 0000000000000..87ed52e365370 --- /dev/null +++ b/docs/reference/search/search-your-data/retrievers-reranking/index.asciidoc @@ -0,0 +1,8 @@ +[[retrievers-reranking-overview]] +== Retrievers and reranking + +* <> +* <> + +include::retrievers-overview.asciidoc[] +include::semantic-reranking.asciidoc[] diff --git a/docs/reference/search/search-your-data/retrievers-overview.asciidoc b/docs/reference/search/search-your-data/retrievers-reranking/retrievers-overview.asciidoc similarity index 75% rename from docs/reference/search/search-your-data/retrievers-overview.asciidoc rename to docs/reference/search/search-your-data/retrievers-reranking/retrievers-overview.asciidoc index 92cd085583916..99659ae76e092 100644 --- a/docs/reference/search/search-your-data/retrievers-overview.asciidoc +++ b/docs/reference/search/search-your-data/retrievers-reranking/retrievers-overview.asciidoc @@ -1,7 +1,5 @@ [[retrievers-overview]] -== Retrievers - -// Will move to a top level "Retrievers and reranking" section once reranking is live +=== Retrievers preview::[] @@ -15,33 +13,32 @@ For implementation details, including notable restrictions, check out the [discrete] [[retrievers-overview-types]] -=== Retriever types +==== Retriever types Retrievers come in various types, each tailored for different search operations. The following retrievers are currently available: -* <>. -Returns top documents from a traditional https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl.html[query]. -Mimics a traditional query but in the context of a retriever framework. -This ensures backward compatibility as existing `_search` requests remain supported. -That way you can transition to the new abstraction at your own pace without mixing syntaxes. -* <>. -Returns top documents from a <>, in the context of a retriever framework. -* <>. -Combines and ranks multiple first-stage retrievers using the reciprocal rank fusion (RRF) algorithm. -Allows you to combine multiple result sets with different relevance indicators into a single result set. -An RRF retriever is a *compound retriever*, where its `filter` element is propagated to its sub retrievers. +* <>. Returns top documents from a +traditional https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl.html[query]. +Mimics a traditional query but in the context of a retriever framework. This +ensures backward compatibility as existing `_search` requests remain supported. +That way you can transition to the new abstraction at your own pace without +mixing syntaxes. +* <>. Returns top documents from a <>, +in the context of a retriever framework. +* <>. Combines and ranks multiple first-stage retrievers using +the reciprocal rank fusion (RRF) algorithm. Allows you to combine multiple result sets +with different relevance indicators into a single result set. +An RRF retriever is a *compound retriever*, where its `filter` element is +propagated to its sub retrievers. + Sub retrievers may not use elements that are restricted by having a compound retriever as part of the retriever tree. See the <> for detailed examples and information on how to use the RRF retriever. - -[NOTE] -==== -Stay tuned for more retriever types in future releases! -==== +* <>. Used for <>. +Requires first creating a `rerank` task using the <>. [discrete] -=== What makes retrievers useful? +==== What makes retrievers useful? Here's an overview of what makes retrievers useful and how they differ from regular queries. @@ -73,7 +70,7 @@ When using compound retrievers, only the query element is allowed, which enforce [discrete] [[retrievers-overview-example]] -=== Example +==== Example The following example demonstrates how using retrievers simplify the composability of queries for RRF ranking. @@ -154,25 +151,23 @@ GET example-index/_search [discrete] [[retrievers-overview-glossary]] -=== Glossary +==== Glossary Here are some important terms: -* *Retrieval Pipeline*. -Defines the entire retrieval and ranking logic to produce top hits. -* *Retriever Tree*. -A hierarchical structure that defines how retrievers interact. -* *First-stage Retriever*. -Returns an initial set of candidate documents. -* *Compound Retriever*. -Builds on one or more retrievers, enhancing document retrieval and ranking logic. -* *Combiners*. -Compound retrievers that merge top hits from multiple sub-retrievers. -//* NOT YET *Rerankers*. Special compound retrievers that reorder hits and may adjust the number of hits, with distinctions between first-stage and second-stage rerankers. +* *Retrieval Pipeline*. Defines the entire retrieval and ranking logic to +produce top hits. +* *Retriever Tree*. A hierarchical structure that defines how retrievers interact. +* *First-stage Retriever*. Returns an initial set of candidate documents. +* *Compound Retriever*. Builds on one or more retrievers, +enhancing document retrieval and ranking logic. +* *Combiners*. Compound retrievers that merge top hits +from multiple sub-retrievers. +* *Rerankers*. Special compound retrievers that reorder hits and may adjust the number of hits, with distinctions between first-stage and second-stage rerankers. [discrete] [[retrievers-overview-play-in-search]] -=== Retrievers in action +==== Retrievers in action The Search Playground builds Elasticsearch queries using the retriever abstraction. It automatically detects the fields and types in your index and builds a retriever tree based on your selections. @@ -180,7 +175,9 @@ It automatically detects the fields and types in your index and builds a retriev You can use the Playground to experiment with different retriever configurations and see how they affect search results. Refer to the {kibana-ref}/playground.html[Playground documentation] for more information. -// Content coming in https://github.com/elastic/kibana/pull/182692 - +[discrete] +[[retrievers-overview-api-reference]] +==== API reference +For implementation details, including notable restrictions, check out the <> in the Search API docs. \ No newline at end of file diff --git a/docs/reference/search/search-your-data/retrievers-reranking/semantic-reranking.asciidoc b/docs/reference/search/search-your-data/retrievers-reranking/semantic-reranking.asciidoc new file mode 100644 index 0000000000000..75c06aa953302 --- /dev/null +++ b/docs/reference/search/search-your-data/retrievers-reranking/semantic-reranking.asciidoc @@ -0,0 +1,151 @@ +[[semantic-reranking]] +=== Semantic reranking + +preview::[] + +[TIP] +==== +This overview focuses more on the high-level concepts and use cases for semantic reranking. For full implementation details on how to set up and use semantic reranking in {es}, see the <> in the Search API docs. +==== + +Rerankers improve the relevance of results from earlier-stage retrieval mechanisms. +_Semantic_ rerankers use machine learning models to reorder search results based on their semantic similarity to a query. + +First-stage retrievers and rankers must be very fast and efficient because they process either the entire corpus, or all matching documents. +In a multi-stage pipeline, you can progressively use more computationally intensive ranking functions and techniques, as they will operate on smaller result sets at each step. +This helps avoid query latency degradation and keeps costs manageable. + +Semantic reranking requires relatively large and complex machine learning models and operates in real-time in response to queries. +This technique makes sense on a small _top-k_ result set, as one the of the final steps in a pipeline. +This is a powerful technique for improving search relevance that works equally well with keyword, semantic, or hybrid retrieval algorithms. + +The next sections provide more details on the benefits, use cases, and model types used for semantic reranking. +The final sections include a practical, high-level overview of how to implement <> and links to the full reference documentation. + +[discrete] +[[semantic-reranking-use-cases]] +==== Use cases + +Semantic reranking enables a variety of use cases: + +* *Lexical (BM25) retrieval results reranking* +** Out-of-the-box semantic search by adding a simple API call to any lexical/BM25 retrieval pipeline. +** Adds semantic search capabilities on top of existing indices without reindexing, perfect for quick improvements. +** Ideal for environments with complex existing indices. + +* *Semantic retrieval results reranking* +** Improves results from semantic retrievers using ELSER sparse vector embeddings or dense vector embeddings by using more powerful models. +** Adds a refinement layer on top of hybrid retrieval with <>. + +* *General applications* +** Supports automatic and transparent chunking, eliminating the need for pre-chunking at index time. +** Provides explicit control over document relevance in retrieval-augmented generation (RAG) uses cases or other scenarios involving language model (LLM) inputs. + +Now that we've outlined the value of semantic reranking, we'll explore the specific models that power this process and how they differ. + +[discrete] +[[semantic-reranking-models]] +==== Cross-encoder and bi-encoder models + +At a high level, two model types are used for semantic reranking: cross-encoders and bi-encoders. + +NOTE: In this version, {es} *only supports cross-encoders* for semantic reranking. + +* A *cross-encoder model* can be thought of as a more powerful, all-in-one solution, because it generates query-aware document representations. +It takes the query and document texts as a single, concatenated input. +* A *bi-encoder model* takes as input either document or query text. +Documents and query embeddings are computed separately, so they aren't aware of each other. +** To compute a ranking score, an external operation is required. This typically involves computing dot-product or cosine similarity between the query and document embeddings. + +In brief, cross-encoders provide high accuracy but are more resource-intensive. +Bi-encoders are faster and more cost-effective but less precise. + +In future versions, {es} will also support bi-encoders. +If you're interested in a more detailed analysis of the practical differences between cross-encoders and bi-encoders, untoggle the next section. + +.Comparisons between cross-encoder and bi-encoder +[%collapsible] +============== +The following is a non-exhaustive list of considerations when choosing between cross-encoders and bi-encoders for semantic reranking: + +* Because a cross-encoder model simultaneously processes both query and document texts, it can better infer their relevance, making it more effective as a reranker than a bi-encoder. +* Cross-encoder models are generally larger and more computationally intensive, resulting in higher latencies and increased computational costs. +* There are significantly fewer open-source cross-encoders, while bi-encoders offer a wide variety of sizes, languages, and other trade-offs. +* The effectiveness of cross-encoders can also improve the relevance of semantic retrievers. +For example, their ability to take word order into account can improve on dense or sparse embedding retrieval. +* When trained in tandem with specific retrievers (like lexical/BM25), cross-encoders can “correct” typical errors made by those retrievers. +* Cross-encoders output scores that are consistent across queries. +This enables you to maintain high relevance in result sets, by setting a minimum score threshold for all queries. +For example, this is important when using results in a RAG workflow or if you're otherwise feeding results to LLMs. +Note that similarity scores from bi-encoders/embedding similarities are _query-dependent_, meaning you cannot set universal cut-offs. +* Bi-encoders rerank using embeddings. You can improve your reranking latency by creating embeddings at ingest-time. These embeddings can be stored for reranking without being indexed for retrieval, reducing your memory footprint. +============== + +[discrete] +[[semantic-reranking-in-es]] +==== Semantic reranking in {es} + +In {es}, semantic rerankers are implemented using the {es} <> and a <>. + +To use semantic reranking in {es}, you need to: + +. Choose a reranking model. In addition to cross-encoder models running on {es} inference nodes, we also expose external models and services via the Inference API to semantic rerankers. +** This includes cross-encoder models running in https://huggingface.co/inference-endpoints[HuggingFace Inference Endpoints] and the https://cohere.com/rerank[Cohere Rerank API]. +. Create a `rerank` task using the <>. +The Inference API creates an inference endpoint and configures your chosen machine learning model to perform the reranking task. +. Define a `text_similarity_reranker` retriever in your search request. +The retriever syntax makes it simple to configure both the retrieval and reranking of search results in a single API call. + +.*Example search request* with semantic reranker +[%collapsible] +============== +The following example shows a search request that uses a semantic reranker to reorder the top-k documents based on their semantic similarity to the query. +[source,console] +---- +POST _search +{ + "retriever": { + "text_similarity_reranker": { + "retriever": { + "standard": { + "query": { + "match": { + "text": "How often does the moon hide the sun?" + } + } + } + }, + "field": "text", + "inference_id": "my-cohere-rerank-model", + "inference_text": "How often does the moon hide the sun?", + "rank_window_size": 100, + "min_score": 0.5 + } + } +} +---- +// TEST[skip:TBD] +============== + +[discrete] +[[semantic-reranking-types]] +==== Supported reranking types + +The following `text_similarity_reranker` model configuration options are available. + +*Text similarity with cross-encoder* + +This solution uses a hosted or 3rd party inference service which relies on a cross-encoder model. +The model receives the text fields from the _top-K_ documents, as well as the search query, and calculates scores directly, which are then used to rerank the documents. + +Used with the Cohere inference service rolled out in 8.13, turn on semantic reranking that works out of the box. +Check out our https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/integrations/cohere/cohere-elasticsearch.ipynb[Python notebook] for using Cohere with {es}. + +[discrete] +[[semantic-reranking-learn-more]] +==== Learn more + +* Read the <> for syntax and implementation details +* Learn more about the <> abstraction +* Learn more about the Elastic <> +* Check out our https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/integrations/cohere/cohere-elasticsearch.ipynb[Python notebook] for using Cohere with {es} \ No newline at end of file diff --git a/docs/reference/search/search-your-data/search-with-synonyms.asciidoc b/docs/reference/search/search-your-data/search-with-synonyms.asciidoc index 596af695b7910..61d3a1d8f925b 100644 --- a/docs/reference/search/search-your-data/search-with-synonyms.asciidoc +++ b/docs/reference/search/search-your-data/search-with-synonyms.asciidoc @@ -82,6 +82,19 @@ If an index is created referencing a nonexistent synonyms set, the index will re The only way to recover from this scenario is to ensure the synonyms set exists then either delete and re-create the index, or close and re-open the index. ====== +[WARNING] +==== +Invalid synonym rules can cause errors when applying analyzer changes. +For reloadable analyzers, this prevents reloading and applying changes. +You must correct errors in the synonym rules and reload the analyzer. + +An index with invalid synonym rules cannot be reopened, making it inoperable when: + +* A node containing the index starts +* The index is opened from a closed state +* A node restart occurs (which reopens the node assigned shards) +==== + {es} uses synonyms as part of the <>. You can use two types of <> to include synonyms: diff --git a/docs/reference/search/search-your-data/search-your-data.asciidoc b/docs/reference/search/search-your-data/search-your-data.asciidoc index e1c1618410f2f..a885df2f2179e 100644 --- a/docs/reference/search/search-your-data/search-your-data.asciidoc +++ b/docs/reference/search/search-your-data/search-your-data.asciidoc @@ -45,7 +45,7 @@ results directly in the Kibana Search UI. include::search-api.asciidoc[] include::knn-search.asciidoc[] include::semantic-search.asciidoc[] -include::retrievers-overview.asciidoc[] +include::retrievers-reranking/index.asciidoc[] include::learning-to-rank.asciidoc[] include::search-across-clusters.asciidoc[] include::search-with-synonyms.asciidoc[] diff --git a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc index 6ecfea0a02dbc..ae27b46d4b876 100644 --- a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc @@ -17,6 +17,7 @@ For a list of supported models available on HuggingFace, refer to Azure based examples use models available through https://ai.azure.com/explore/models?selectedTask=embeddings[Azure AI Studio] or https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models[Azure OpenAI]. Mistral examples use the `mistral-embed` model from https://docs.mistral.ai/getting-started/models/[the Mistral API]. +Amazon Bedrock examples use the `amazon.titan-embed-text-v1` model from https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html[the Amazon Bedrock base models]. Click the name of the service you want to use on any of the widgets below to review the corresponding instructions. diff --git a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc index c2dabedb0336c..2b8b6c9c25afe 100644 --- a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc @@ -24,7 +24,6 @@ This tutorial uses the <> for demonstra To use the `semantic_text` field type, you must have an {infer} endpoint deployed in your cluster using the <>. - [discrete] [[semantic-text-infer-endpoint]] ==== Create the {infer} endpoint @@ -48,6 +47,13 @@ be used and ELSER creates sparse vectors. The `inference_id` is `my-elser-endpoint`. <2> The `elser` service is used in this example. +[NOTE] +==== +You might see a 502 bad gateway error in the response when using the {kib} Console. +This error usually just reflects a timeout, while the model downloads in the background. +You can check the download progress in the {ml-app} UI. +If using the Python client, you can set the `timeout` parameter to a higher value. +==== [discrete] [[semantic-text-index-mapping]] diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc index 15985088a6ff7..501d645665a02 100644 --- a/docs/reference/search/search.asciidoc +++ b/docs/reference/search/search.asciidoc @@ -141,7 +141,7 @@ When unspecified, the pre-filter phase is executed if any of these conditions is - The primary sort of the query targets an indexed field. [[search-preference]] -tag::search-preference[] +// tag::search-preference[] `preference`:: (Optional, string) Nodes and shards used for the search. By default, {es} selects from eligible @@ -178,7 +178,7 @@ Any string that does not start with `_`. If the cluster state and selected shards do not change, searches using the same `` value are routed to the same shards in the same order. ==== -end::search-preference[] +// end::search-preference[] [[search-api-query-params-q]] diff --git a/docs/reference/security/authorization/privileges.asciidoc b/docs/reference/security/authorization/privileges.asciidoc index 44897baa8cb4a..145bd8ebc06bb 100644 --- a/docs/reference/security/authorization/privileges.asciidoc +++ b/docs/reference/security/authorization/privileges.asciidoc @@ -282,7 +282,7 @@ status of {Ilm} This privilege is not available in {serverless-full}. `read_pipeline`:: -Read-only access to ingest pipline (get, simulate). +Read-only access to ingest pipeline (get, simulate). `read_slm`:: All read-only {slm-init} actions, such as getting policies and checking the diff --git a/docs/reference/setup/logging-config.asciidoc b/docs/reference/setup/logging-config.asciidoc index 7b36b6382c9bf..e382bbdacb464 100644 --- a/docs/reference/setup/logging-config.asciidoc +++ b/docs/reference/setup/logging-config.asciidoc @@ -140,19 +140,41 @@ documentation]. [[configuring-logging-levels]] === Configuring logging levels -Each Java package in the {es-repo}[{es} source code] has a related logger. For -example, the `org.elasticsearch.discovery` package has -`logger.org.elasticsearch.discovery` for logs related to the -<> process. - -To get more or less verbose logs, use the <> to change the related logger's log level. Each logger -accepts Log4j 2's built-in log levels, from least to most verbose: `OFF`, -`FATAL`, `ERROR`, `WARN`, `INFO`, `DEBUG`, and `TRACE`. The default log level is -`INFO`. Messages logged at higher verbosity levels (`DEBUG` and `TRACE`) are -only intended for expert use. To prevent leaking sensitive information in logs, -{es} will reject setting certain loggers to higher verbosity levels unless -<> is enabled. +Log4J 2 log messages include a _level_ field, which is one of the following (in +order of increasing verbosity): + +* `FATAL` +* `ERROR` +* `WARN` +* `INFO` +* `DEBUG` +* `TRACE` + +By default {es} includes all messages at levels `INFO`, `WARN`, `ERROR` and +`FATAL` in its logs, but filters out messages at levels `DEBUG` and `TRACE`. +This is the recommended configuration. Do not filter out messages at `INFO` or +higher log levels or else you may not be able to understand your cluster's +behaviour or troubleshoot common problems. Do not enable logging at levels +`DEBUG` or `TRACE` unless you are following instructions elsewhere in this +manual which call for more detailed logging, or you are an expert user who will +be reading the {es} source code to determine the meaning of the logs. + +Messages are logged by a hierarchy of loggers which matches the hierarchy of +Java packages and classes in the {es-repo}[{es} source code]. Every logger has +a corresponding <> which can be used +to control the verbosity of its logs. The setting's name is the fully-qualified +name of the package or class, prefixed with `logger.`. + +You may set each logger's verbosity to the name of a log level, for instance +`DEBUG`, which means that messages from this logger at levels up to the +specified one will be included in the logs. You may also use the value `OFF` to +suppress all messages from the logger. + +For example, the `org.elasticsearch.discovery` package contains functionality +related to the <> process, and you can +control the verbosity of its logs with the `logger.org.elasticsearch.discovery` +setting. To enable `DEBUG` logging for this package, use the +<> as follows: [source,console] ---- @@ -164,8 +186,8 @@ PUT /_cluster/settings } ---- -To reset a logger's verbosity to its default level, set the logger setting to -`null`: +To reset this package's log verbosity to its default level, set the logger +setting to `null`: [source,console] ---- @@ -211,6 +233,14 @@ formatting the same information in different ways, renaming the logger or adjusting the log level for specific messages. Do not rely on the contents of the application logs remaining precisely the same between versions. +NOTE: To prevent leaking sensitive information in logs, {es} suppresses certain +log messages by default even at the highest verbosity levels. To disable this +protection on a node, set the Java system property +`es.insecure_network_trace_enabled` to `true`. This feature is primarily +intended for test systems which do not contain any sensitive information. If you +set this property on a system which contains sensitive information, you must +protect your logs from unauthorized access. + [discrete] [[deprecation-logging]] === Deprecation logging diff --git a/docs/reference/synonyms/apis/synonyms-apis.asciidoc b/docs/reference/synonyms/apis/synonyms-apis.asciidoc index c9de52939b2fe..dbbc26c36d3df 100644 --- a/docs/reference/synonyms/apis/synonyms-apis.asciidoc +++ b/docs/reference/synonyms/apis/synonyms-apis.asciidoc @@ -21,6 +21,23 @@ These filters are applied as part of the <> process by the << NOTE: Synonyms sets are limited to a maximum of 10,000 synonym rules per set. If you need to manage more synonym rules, you can create multiple synonyms sets. +WARNING: Synonyms sets must exist before they can be added to indices. +If an index is created referencing a nonexistent synonyms set, the index will remain in a partially created and inoperable state. +The only way to recover from this scenario is to ensure the synonyms set exists then either delete and re-create the index, or close and re-open the index. + +[WARNING] +==== +Invalid synonym rules can cause errors when applying analyzer changes. +For reloadable analyzers, this prevents reloading and applying changes. +You must correct errors in the synonym rules and reload the analyzer. + +An index with invalid synonym rules cannot be reopened, making it inoperable when: + +* A node containing the index starts +* The index is opened from a closed state +* A node restart occurs (which reopens the node assigned shards) +==== + [discrete] [[synonyms-sets-apis]] === Synonyms sets APIs diff --git a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc index c8a42c4d0585a..6039d1de5345b 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc @@ -37,6 +37,12 @@ id="infer-api-ingest-mistral"> Mistral +
+
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc index a239c79e5a6d1..f95c4a6dbc8c8 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc @@ -164,3 +164,29 @@ PUT _ingest/pipeline/mistral_embeddings and the `output_field` that will contain the {infer} results. // end::mistral[] + +// tag::amazon-bedrock[] + +[source,console] +-------------------------------------------------- +PUT _ingest/pipeline/amazon_bedrock_embeddings +{ + "processors": [ + { + "inference": { + "model_id": "amazon_bedrock_embeddings", <1> + "input_output": { <2> + "input_field": "content", + "output_field": "content_embedding" + } + } + } + ] +} +-------------------------------------------------- +<1> The name of the inference endpoint you created by using the +<>, it's referred to as `inference_id` in that step. +<2> Configuration object that defines the `input_field` for the {infer} process +and the `output_field` that will contain the {infer} results. + +// end::amazon-bedrock[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc index 80c7c7ef23ee3..66b0cde549545 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc @@ -37,6 +37,12 @@ id="infer-api-mapping-mistral"> Mistral +
+
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc index a1bce38a02ad2..72c648e63871d 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc @@ -207,3 +207,38 @@ the {infer} pipeline configuration in the next step. <6> The field type which is text in this example. // end::mistral[] + +// tag::amazon-bedrock[] + +[source,console] +-------------------------------------------------- +PUT amazon-bedrock-embeddings +{ + "mappings": { + "properties": { + "content_embedding": { <1> + "type": "dense_vector", <2> + "dims": 1024, <3> + "element_type": "float", + "similarity": "dot_product" <4> + }, + "content": { <5> + "type": "text" <6> + } + } + } +} +-------------------------------------------------- +<1> The name of the field to contain the generated tokens. It must be referenced +in the {infer} pipeline configuration in the next step. +<2> The field to contain the tokens is a `dense_vector` field. +<3> The output dimensions of the model. This value may be different depending on the underlying model used. +See the https://docs.aws.amazon.com/bedrock/latest/userguide/titan-multiemb-models.html[Amazon Titan model] or the https://docs.cohere.com/reference/embed[Cohere Embeddings model] documentation. +<4> For Amazon Bedrock embeddings, the `dot_product` function should be used to +calculate similarity for Amazon titan models, or `cosine` for Cohere models. +<5> The name of the field from which to create the dense vector representation. +In this example, the name of the field is `content`. It must be referenced in +the {infer} pipeline configuration in the next step. +<6> The field type which is text in this example. + +// end::amazon-bedrock[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc index 4face6a105819..9a8028e2b3c6c 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc @@ -37,6 +37,12 @@ id="infer-api-reindex-mistral"> Mistral +
+
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc index 927e47ea4d67c..995189f1309aa 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc @@ -154,3 +154,26 @@ number makes the update of the reindexing process quicker which enables you to follow the progress closely and detect errors early. // end::mistral[] + +// tag::amazon-bedrock[] + +[source,console] +---- +POST _reindex?wait_for_completion=false +{ + "source": { + "index": "test-data", + "size": 50 <1> + }, + "dest": { + "index": "amazon-bedrock-embeddings", + "pipeline": "amazon_bedrock_embeddings" + } +} +---- +// TEST[skip:TBD] +<1> The default batch size for reindexing is 1000. Reducing `size` to a smaller +number makes the update of the reindexing process quicker which enables you to +follow the progress closely and detect errors early. + +// end::amazon-bedrock[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc index 9981eb90d4929..cf2e4994279d9 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc @@ -37,6 +37,12 @@ id="infer-api-requirements-mistral"> Mistral +
+
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc index 435e53bbc0bc0..856e4d5f0fe47 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc @@ -39,3 +39,9 @@ You can apply for access to Azure OpenAI by completing the form at https://aka.m * An API key generated for your account // end::mistral[] + +// tag::amazon-bedrock[] +* An AWS Account with https://aws.amazon.com/bedrock/[Amazon Bedrock] access +* A pair of access and secret keys used to access Amazon Bedrock + +// end::amazon-bedrock[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc index 6a67b28f91601..52cf65c4a1509 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc @@ -37,6 +37,12 @@ id="infer-api-search-mistral"> Mistral +
+
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc index 523c2301e75ff..5e23afeb19a9f 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc @@ -405,3 +405,68 @@ query from the `mistral-embeddings` index sorted by their proximity to the query // NOTCONSOLE // end::mistral[] + +// tag::amazon-bedrock[] + +[source,console] +-------------------------------------------------- +GET amazon-bedrock-embeddings/_search +{ + "knn": { + "field": "content_embedding", + "query_vector_builder": { + "text_embedding": { + "model_id": "amazon_bedrock_embeddings", + "model_text": "Calculate fuel cost" + } + }, + "k": 10, + "num_candidates": 100 + }, + "_source": [ + "id", + "content" + ] +} +-------------------------------------------------- +// TEST[skip:TBD] + +As a result, you receive the top 10 documents that are closest in meaning to the +query from the `amazon-bedrock-embeddings` index sorted by their proximity to the query: + +[source,consol-result] +-------------------------------------------------- +"hits": [ + { + "_index": "amazon-bedrock-embeddings", + "_id": "DDd5OowBHxQKHyc3TDSC", + "_score": 0.83704096, + "_source": { + "id": 862114, + "body": "How to calculate fuel cost for a road trip. By Tara Baukus Mello • Bankrate.com. Dear Driving for Dollars, My family is considering taking a long road trip to finish off the end of the summer, but I'm a little worried about gas prices and our overall fuel cost.It doesn't seem easy to calculate since we'll be traveling through many states and we are considering several routes.y family is considering taking a long road trip to finish off the end of the summer, but I'm a little worried about gas prices and our overall fuel cost. It doesn't seem easy to calculate since we'll be traveling through many states and we are considering several routes." + } + }, + { + "_index": "amazon-bedrock-embeddings", + "_id": "ajd5OowBHxQKHyc3TDSC", + "_score": 0.8345704, + "_source": { + "id": 820622, + "body": "Home Heating Calculator. Typically, approximately 50% of the energy consumed in a home annually is for space heating. When deciding on a heating system, many factors will come into play: cost of fuel, installation cost, convenience and life style are all important.This calculator can help you estimate the cost of fuel for different heating appliances.hen deciding on a heating system, many factors will come into play: cost of fuel, installation cost, convenience and life style are all important. This calculator can help you estimate the cost of fuel for different heating appliances." + } + }, + { + "_index": "amazon-bedrock-embeddings", + "_id": "Djd5OowBHxQKHyc3TDSC", + "_score": 0.8327426, + "_source": { + "id": 8202683, + "body": "Fuel is another important cost. This cost will depend on your boat, how far you travel, and how fast you travel. A 33-foot sailboat traveling at 7 knots should be able to travel 300 miles on 50 gallons of diesel fuel.If you are paying $4 per gallon, the trip would cost you $200.Most boats have much larger gas tanks than cars.uel is another important cost. This cost will depend on your boat, how far you travel, and how fast you travel. A 33-foot sailboat traveling at 7 knots should be able to travel 300 miles on 50 gallons of diesel fuel." + } + }, + (...) + ] +-------------------------------------------------- +// NOTCONSOLE + +// end::amazon-bedrock[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc index 1f3ad645d7c29..d13301b64a871 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc @@ -37,6 +37,12 @@ id="infer-api-task-mistral"> Mistral +
+
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc index 18fa3ba541bff..c6ef2a46a8731 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc @@ -177,3 +177,29 @@ PUT _inference/text_embedding/mistral_embeddings <1> <3> The Mistral embeddings model name, for example `mistral-embed`. // end::mistral[] + +// tag::amazon-bedrock[] + +[source,console] +------------------------------------------------------------ +PUT _inference/text_embedding/amazon_bedrock_embeddings <1> +{ + "service": "amazonbedrock", + "service_settings": { + "access_key": "", <2> + "secret_key": "", <3> + "region": "", <4> + "provider": "", <5> + "model": "" <6> + } +} +------------------------------------------------------------ +// TEST[skip:TBD] +<1> The task type is `text_embedding` in the path and the `inference_id` which is the unique identifier of the {infer} endpoint is `amazon_bedrock_embeddings`. +<2> The access key can be found on your AWS IAM management page for the user account to access Amazon Bedrock. +<3> The secret key should be the paired key for the specified access key. +<4> Specify the region that your model is hosted in. +<5> Specify the model provider. +<6> The model ID or ARN of the model to use. + +// end::amazon-bedrock[] diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 5e26d96c4ca17..e5fd9f124b444 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -119,29 +119,39 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + + + + + + + + + + + @@ -271,11 +281,6 @@ - - - - - @@ -296,11 +301,6 @@ - - - - - @@ -341,14 +341,9 @@ - - - - - - - - + + + @@ -361,14 +356,9 @@ - - - - - - - - + + + @@ -386,14 +376,9 @@ - - - - - - - - + + + @@ -1471,19 +1456,19 @@ - - - + + + - - - + + + - - - + + + @@ -1501,6 +1486,11 @@ + + + + + diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index e6441136f3d4b..2c3521197d7c4 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 515ab9d5f1822..efe2ff3449216 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=f8b4f4772d302c8ff580bc40d0f56e715de69b163546944f787c87abf209c961 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-all.zip +distributionSha256Sum=258e722ec21e955201e31447b0aed14201765a3bfbae296a46cf60b70e66db70 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index b740cf13397ab..f5feea6d6b116 100755 --- a/gradlew +++ b/gradlew @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## # @@ -84,7 +86,8 @@ done # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) -APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum diff --git a/gradlew.bat b/gradlew.bat index 7101f8e4676fc..9b42019c7915b 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -13,6 +13,8 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem @if "%DEBUG%"=="" @echo off @rem ########################################################################## diff --git a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java index f3f53f1b3c5ea..3c01e490369de 100644 --- a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java +++ b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java @@ -203,7 +203,7 @@ public Map parse(String inputString) { DissectKey key = dissectPair.key(); byte[] delimiter = dissectPair.delimiter().getBytes(StandardCharsets.UTF_8); // start dissection after the first delimiter - int i = leadingDelimiter.length(); + int i = leadingDelimiter.getBytes(StandardCharsets.UTF_8).length; int valueStart = i; int lookAheadMatches; // start walking the input string byte by byte, look ahead for matches where needed diff --git a/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java b/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java index 431b26fc1155d..2893e419a84a3 100644 --- a/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java +++ b/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java @@ -211,6 +211,18 @@ public void testMatchUnicode() { assertMatch("%{a->}࿏%{b}", "⟳༒࿏࿏࿏࿏࿏༒⟲", Arrays.asList("a", "b"), Arrays.asList("⟳༒", "༒⟲")); assertMatch("%{*a}࿏%{&a}", "⟳༒࿏༒⟲", Arrays.asList("⟳༒"), Arrays.asList("༒⟲")); assertMatch("%{}࿏%{a}", "⟳༒࿏༒⟲", Arrays.asList("a"), Arrays.asList("༒⟲")); + assertMatch( + "Zürich, the %{adjective} city in Switzerland", + "Zürich, the largest city in Switzerland", + Arrays.asList("adjective"), + Arrays.asList("largest") + ); + assertMatch( + "Zürich, the %{one} city in Switzerland; Zürich, the %{two} city in Switzerland", + "Zürich, the largest city in Switzerland; Zürich, the LARGEST city in Switzerland", + Arrays.asList("one", "two"), + Arrays.asList("largest", "LARGEST") + ); } public void testMatchRemainder() { diff --git a/libs/native/jna/build.gradle b/libs/native/jna/build.gradle index e34f35318126a..679191afbc574 100644 --- a/libs/native/jna/build.gradle +++ b/libs/native/jna/build.gradle @@ -15,8 +15,7 @@ base { dependencies { compileOnly project(':libs:elasticsearch-core') compileOnly project(':libs:elasticsearch-native') - // TODO: this will become an implementation dep onces jna is removed from server - compileOnly "net.java.dev.jna:jna:${versions.jna}" + implementation "net.java.dev.jna:jna:${versions.jna}" testImplementation(project(":test:framework")) { exclude group: 'org.elasticsearch', module: 'elasticsearch-native' diff --git a/libs/preallocate/licenses/jna-LICENSE.txt b/libs/native/jna/licenses/jna-LICENSE.txt similarity index 100% rename from libs/preallocate/licenses/jna-LICENSE.txt rename to libs/native/jna/licenses/jna-LICENSE.txt diff --git a/libs/preallocate/licenses/jna-NOTICE.txt b/libs/native/jna/licenses/jna-NOTICE.txt similarity index 100% rename from libs/preallocate/licenses/jna-NOTICE.txt rename to libs/native/jna/licenses/jna-NOTICE.txt diff --git a/libs/native/jna/src/main/java/module-info.java b/libs/native/jna/src/main/java/module-info.java index 1b95ccc7cdda0..6e8b0847ce030 100644 --- a/libs/native/jna/src/main/java/module-info.java +++ b/libs/native/jna/src/main/java/module-info.java @@ -14,6 +14,7 @@ requires org.elasticsearch.nativeaccess; requires org.elasticsearch.logging; requires com.sun.jna; + requires java.desktop; exports org.elasticsearch.nativeaccess.jna to com.sun.jna; diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaKernel32Library.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaKernel32Library.java index 2c7ec70f36eb3..1403806c595a7 100644 --- a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaKernel32Library.java +++ b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaKernel32Library.java @@ -15,12 +15,14 @@ import com.sun.jna.Structure; import com.sun.jna.Structure.ByReference; import com.sun.jna.WString; +import com.sun.jna.ptr.IntByReference; import com.sun.jna.win32.StdCallLibrary; import org.elasticsearch.nativeaccess.WindowsFunctions.ConsoleCtrlHandler; import org.elasticsearch.nativeaccess.lib.Kernel32Library; import java.util.List; +import java.util.function.IntConsumer; class JnaKernel32Library implements Kernel32Library { private static class JnaHandle implements Handle { @@ -158,6 +160,8 @@ private interface NativeFunctions extends StdCallLibrary { boolean SetProcessWorkingSetSize(Pointer handle, SizeT minSize, SizeT maxSize); + int GetCompressedFileSizeW(WString lpFileName, IntByReference lpFileSizeHigh); + int GetShortPathNameW(WString lpszLongPath, char[] lpszShortPath, int cchBuffer); boolean SetConsoleCtrlHandler(StdCallLibrary.StdCallCallback handler, boolean add); @@ -232,6 +236,15 @@ public boolean SetProcessWorkingSetSize(Handle handle, long minSize, long maxSiz return functions.SetProcessWorkingSetSize(jnaHandle.pointer, new SizeT(minSize), new SizeT(maxSize)); } + @Override + public int GetCompressedFileSizeW(String lpFileName, IntConsumer lpFileSizeHigh) { + var wideFileName = new WString(lpFileName); + var fileSizeHigh = new IntByReference(); + int ret = functions.GetCompressedFileSizeW(wideFileName, fileSizeHigh); + lpFileSizeHigh.accept(fileSizeHigh.getValue()); + return ret; + } + @Override public int GetShortPathNameW(String lpszLongPath, char[] lpszShortPath, int cchBuffer) { var wideFileName = new WString(lpszLongPath); diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaLinuxCLibrary.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaLinuxCLibrary.java index 742c666d59c23..ca3137ab5df0e 100644 --- a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaLinuxCLibrary.java +++ b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaLinuxCLibrary.java @@ -60,6 +60,8 @@ private interface NativeFunctions extends Library { * this is the only way, DON'T use it on some other architecture unless you know wtf you are doing */ NativeLong syscall(NativeLong number, Object... args); + + int fallocate(int fd, int mode, long offset, long length); } private final NativeFunctions functions; @@ -91,4 +93,9 @@ public int prctl(int option, long arg2, long arg3, long arg4, long arg5) { public long syscall(long number, int operation, int flags, long address) { return functions.syscall(new NativeLong(number), operation, flags, address).longValue(); } + + @Override + public int fallocate(int fd, int mode, long offset, long length) { + return functions.fallocate(fd, mode, offset, length); + } } diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaPosixCLibrary.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaPosixCLibrary.java index 03a7b9c0869be..d984d239e0b39 100644 --- a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaPosixCLibrary.java +++ b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaPosixCLibrary.java @@ -9,8 +9,11 @@ package org.elasticsearch.nativeaccess.jna; import com.sun.jna.Library; +import com.sun.jna.Memory; import com.sun.jna.Native; +import com.sun.jna.NativeLibrary; import com.sun.jna.NativeLong; +import com.sun.jna.Pointer; import com.sun.jna.Structure; import org.elasticsearch.nativeaccess.lib.PosixCLibrary; @@ -51,37 +54,58 @@ public void rlim_max(long v) { } } - public static class JnaFStore extends Structure implements Structure.ByReference, FStore { + public static final class JnaStat64 implements Stat64 { + final Memory memory; + private final int stSizeOffset; + private final int stBlocksOffset; - public int fst_flags = 0; - public int fst_posmode = 0; - public NativeLong fst_offset = new NativeLong(0); - public NativeLong fst_length = new NativeLong(0); - public NativeLong fst_bytesalloc = new NativeLong(0); + JnaStat64(int sizeof, int stSizeOffset, int stBlocksOffset) { + this.memory = new Memory(sizeof); + this.stSizeOffset = stSizeOffset; + this.stBlocksOffset = stBlocksOffset; + } + + @Override + public long st_size() { + return memory.getLong(stSizeOffset); + } + + @Override + public long st_blocks() { + return memory.getLong(stBlocksOffset); + } + } + + public static class JnaFStore implements FStore { + final Memory memory; + + JnaFStore() { + this.memory = new Memory(32); + } @Override public void set_flags(int flags) { - this.fst_flags = flags; + memory.setInt(0, flags); } @Override public void set_posmode(int posmode) { - this.fst_posmode = posmode; + memory.setInt(4, posmode); } @Override public void set_offset(long offset) { - fst_offset.setValue(offset); + memory.setLong(8, offset); } @Override public void set_length(long length) { - fst_length.setValue(length); + memory.setLong(16, length); } @Override public long bytesalloc() { - return fst_bytesalloc.longValue(); + return memory.getLong(24); } } @@ -94,15 +118,48 @@ private interface NativeFunctions extends Library { int mlockall(int flags); - int fcntl(int fd, int cmd, JnaFStore fst); + int fcntl(int fd, int cmd, Object... args); + + int ftruncate(int fd, NativeLong length); + + int open(String filename, int flags, Object... mode); + + int close(int fd); String strerror(int errno); } + private interface FStat64Function extends Library { + int fstat64(int fd, Pointer stat); + } + + private interface FXStatFunction extends Library { + int __fxstat(int version, int fd, Pointer stat); + } + private final NativeFunctions functions; + private final FStat64Function fstat64; JnaPosixCLibrary() { this.functions = Native.load("c", NativeFunctions.class); + FStat64Function fstat64; + try { + // JNA lazily finds symbols, so even though we try to bind two different functions below, if fstat64 + // isn't found, we won't know until runtime when calling the function. To force resolution of the + // symbol we get a function object directly from the native library. We don't use it, we just want to + // see if it will throw UnsatisfiedLinkError + NativeLibrary.getInstance("c").getFunction("fstat64"); + fstat64 = Native.load("c", FStat64Function.class); + } catch (UnsatisfiedLinkError e) { + // fstat has a long history in linux from the 32-bit architecture days. On some modern linux systems, + // fstat64 doesn't exist as a symbol in glibc. Instead, the compiler replaces fstat64 calls with + // the internal __fxstat method. Here we fall back to __fxstat, and staticall bind the special + // "version" argument so that the call site looks the same as that of fstat64 + var fxstat = Native.load("c", FXStatFunction.class); + int version = System.getProperty("os.arch").equals("aarch64") ? 0 : 1; + fstat64 = (fd, stat) -> fxstat.__fxstat(version, fd, stat); + } + this.fstat64 = fstat64; } @Override @@ -115,6 +172,11 @@ public RLimit newRLimit() { return new JnaRLimit(); } + @Override + public Stat64 newStat64(int sizeof, int stSizeOffset, int stBlocksOffset) { + return new JnaStat64(sizeof, stSizeOffset, stBlocksOffset); + } + @Override public int getrlimit(int resource, RLimit rlimit) { assert rlimit instanceof JnaRLimit; @@ -143,7 +205,34 @@ public FStore newFStore() { public int fcntl(int fd, int cmd, FStore fst) { assert fst instanceof JnaFStore; var jnaFst = (JnaFStore) fst; - return functions.fcntl(fd, cmd, jnaFst); + return functions.fcntl(fd, cmd, jnaFst.memory); + } + + @Override + public int ftruncate(int fd, long length) { + return functions.ftruncate(fd, new NativeLong(length)); + } + + @Override + public int open(String pathname, int flags) { + return functions.open(pathname, flags); + } + + @Override + public int open(String pathname, int flags, int mode) { + return functions.open(pathname, flags, mode); + } + + @Override + public int close(int fd) { + return functions.close(fd); + } + + @Override + public int fstat64(int fd, Stat64 stats) { + assert stats instanceof JnaStat64; + var jnaStats = (JnaStat64) stats; + return fstat64.fstat64(fd, jnaStats.memory); } @Override diff --git a/libs/native/src/main/java/module-info.java b/libs/native/src/main/java/module-info.java index d895df1be1c56..226503b24832d 100644 --- a/libs/native/src/main/java/module-info.java +++ b/libs/native/src/main/java/module-info.java @@ -19,6 +19,7 @@ to org.elasticsearch.nativeaccess.jna, org.elasticsearch.server, + org.elasticsearch.blobcache, org.elasticsearch.simdvec, org.elasticsearch.systemd; // allows jna to implement a library provider, and ProviderLocator to load it diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/LinuxNativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/LinuxNativeAccess.java index c50e639c94d27..f6e6035a8aba6 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/LinuxNativeAccess.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/LinuxNativeAccess.java @@ -18,6 +18,8 @@ class LinuxNativeAccess extends PosixNativeAccess { + private static final int STATX_BLOCKS = 0x400; /* Want/got stx_blocks */ + /** the preferred method is seccomp(2), since we can apply to all threads of the process */ static final int SECCOMP_SET_MODE_FILTER = 1; // since Linux 3.17 static final int SECCOMP_FILTER_FLAG_TSYNC = 1; // since Linux 3.17 @@ -88,7 +90,7 @@ record Arch( private final Systemd systemd; LinuxNativeAccess(NativeLibraryProvider libraryProvider) { - super("Linux", libraryProvider, new PosixConstants(-1L, 9, 1, 8)); + super("Linux", libraryProvider, new PosixConstants(-1L, 9, 1, 8, 64, 144, 48, 64)); this.linuxLibc = libraryProvider.getLibrary(LinuxCLibrary.class); this.systemd = new Systemd(libraryProvider.getLibrary(SystemdLibrary.class)); } @@ -120,6 +122,16 @@ protected void logMemoryLimitInstructions() { logger.warn("If you are logged in interactively, you will have to re-login for the new limits to take effect."); } + @Override + protected boolean nativePreallocate(int fd, long currentSize, long newSize) { + final int rc = linuxLibc.fallocate(fd, 0, currentSize, newSize - currentSize); + if (rc != 0) { + logger.warn("fallocate failed: " + libc.strerror(libc.errno())); + return false; + } + return true; + } + /** * Installs exec system call filtering for Linux. *

diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/MacNativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/MacNativeAccess.java index c53b7ba6ac2f0..f277c69de3192 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/MacNativeAccess.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/MacNativeAccess.java @@ -22,6 +22,11 @@ class MacNativeAccess extends PosixNativeAccess { + private static final int F_PREALLOCATE = 42; + private static final int F_ALLOCATECONTIG = 0x2; // allocate contiguous space + private static final int F_ALLOCATEALL = 0x4; // allocate all the requested space or no space at all + private static final int F_PEOFPOSMODE = 3; // allocate from the physical end of the file + /** The only supported flag... */ static final int SANDBOX_NAMED = 1; /** Allow everything except process fork and execution */ @@ -30,7 +35,7 @@ class MacNativeAccess extends PosixNativeAccess { private final MacCLibrary macLibc; MacNativeAccess(NativeLibraryProvider libraryProvider) { - super("MacOS", libraryProvider, new PosixConstants(9223372036854775807L, 5, 1, 6)); + super("MacOS", libraryProvider, new PosixConstants(9223372036854775807L, 5, 1, 6, 512, 144, 96, 104)); this.macLibc = libraryProvider.getLibrary(MacCLibrary.class); } @@ -44,6 +49,31 @@ protected void logMemoryLimitInstructions() { // we don't have instructions for macos } + @Override + protected boolean nativePreallocate(int fd, long currentSize, long newSize) { + var fst = libc.newFStore(); + fst.set_flags(F_ALLOCATECONTIG); + fst.set_posmode(F_PEOFPOSMODE); + fst.set_offset(0); + fst.set_length(newSize); + // first, try allocating contiguously + if (libc.fcntl(fd, F_PREALLOCATE, fst) != 0) { + // TODO: log warning? + // that failed, so let us try allocating non-contiguously + fst.set_flags(F_ALLOCATEALL); + if (libc.fcntl(fd, F_PREALLOCATE, fst) != 0) { + // i'm afraid captain dale had to bail + logger.warn("Could not allocate non-contiguous size: " + libc.strerror(libc.errno())); + return false; + } + } + if (libc.ftruncate(fd, newSize) != 0) { + logger.warn("Could not truncate file: " + libc.strerror(libc.errno())); + return false; + } + return true; + } + /** * Installs exec system call filtering on MacOS. *

diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccess.java index 61935ac93c5a3..0534bc10e910a 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccess.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccess.java @@ -8,7 +8,9 @@ package org.elasticsearch.nativeaccess; +import java.nio.file.Path; import java.util.Optional; +import java.util.OptionalLong; /** * Provides access to native functionality needed by Elastisearch. @@ -62,6 +64,16 @@ static NativeAccess instance() { */ Zstd getZstd(); + /** + * Retrieves the actual number of bytes of disk storage used to store a specified file. + * + * @param path the path to the file + * @return an {@link OptionalLong} that contains the number of allocated bytes on disk for the file, or empty if the size is invalid + */ + OptionalLong allocatedSizeInBytes(Path path); + + void tryPreallocate(Path file, long size); + /** * Returns an accessor for native functions only available on Windows, or {@code null} if not on Windows. */ diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/NoopNativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/NoopNativeAccess.java index fc186cb03b0d9..ffe65548eeb44 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/NoopNativeAccess.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/NoopNativeAccess.java @@ -11,7 +11,9 @@ import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; +import java.nio.file.Path; import java.util.Optional; +import java.util.OptionalLong; class NoopNativeAccess implements NativeAccess { @@ -51,6 +53,17 @@ public ExecSandboxState getExecSandboxState() { return ExecSandboxState.NONE; } + @Override + public OptionalLong allocatedSizeInBytes(Path path) { + logger.warn("Cannot get allocated size of file [" + path + "] because native access is not available"); + return OptionalLong.empty(); + } + + @Override + public void tryPreallocate(Path file, long size) { + logger.warn("Cannot preallocate file size because native access is not available"); + } + @Override public Systemd systemd() { logger.warn("Cannot get systemd access because native access is not available"); diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixConstants.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixConstants.java index 4695ce9ad899c..e767e2b3713ec 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixConstants.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixConstants.java @@ -11,4 +11,13 @@ /** * Code constants on POSIX systems. */ -record PosixConstants(long RLIMIT_INFINITY, int RLIMIT_AS, int RLIMIT_FSIZE, int RLIMIT_MEMLOCK) {} +record PosixConstants( + long RLIMIT_INFINITY, + int RLIMIT_AS, + int RLIMIT_FSIZE, + int RLIMIT_MEMLOCK, + int O_CREAT, + int statStructSize, + int statStructSizeOffset, + int statStructBlocksOffset +) {} diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java index 8f53d1ec4da64..2ce09e567c284 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java @@ -12,12 +12,17 @@ import org.elasticsearch.nativeaccess.lib.PosixCLibrary; import org.elasticsearch.nativeaccess.lib.VectorLibrary; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Optional; +import java.util.OptionalLong; abstract class PosixNativeAccess extends AbstractNativeAccess { public static final int MCL_CURRENT = 1; public static final int ENOMEM = 12; + public static final int O_RDONLY = 0; + public static final int O_WRONLY = 1; protected final PosixCLibrary libc; protected final VectorSimilarityFunctions vectorDistance; @@ -121,6 +126,52 @@ public void tryLockMemory() { protected abstract void logMemoryLimitInstructions(); + @Override + public OptionalLong allocatedSizeInBytes(Path path) { + assert Files.isRegularFile(path) : path; + var stats = libc.newStat64(constants.statStructSize(), constants.statStructSizeOffset(), constants.statStructBlocksOffset()); + + int fd = libc.open(path.toAbsolutePath().toString(), O_RDONLY); + if (fd == -1) { + logger.warn("Could not open file [" + path + "] to get allocated size: " + libc.strerror(libc.errno())); + return OptionalLong.empty(); + } + + if (libc.fstat64(fd, stats) != 0) { + logger.warn("Could not get stats for file [" + path + "] to get allocated size: " + libc.strerror(libc.errno())); + return OptionalLong.empty(); + } + if (libc.close(fd) != 0) { + logger.warn("Failed to close file [" + path + "] after getting stats: " + libc.strerror(libc.errno())); + } + return OptionalLong.of(stats.st_blocks() * 512); + } + + @Override + public void tryPreallocate(Path file, long newSize) { + // get fd and current size, then pass to OS variant + int fd = libc.open(file.toAbsolutePath().toString(), O_WRONLY, constants.O_CREAT()); + if (fd == -1) { + logger.warn("Could not open file [" + file + "] to preallocate size: " + libc.strerror(libc.errno())); + return; + } + + var stats = libc.newStat64(constants.statStructSize(), constants.statStructSizeOffset(), constants.statStructBlocksOffset()); + if (libc.fstat64(fd, stats) != 0) { + logger.warn("Could not get stats for file [" + file + "] to preallocate size: " + libc.strerror(libc.errno())); + } else { + if (nativePreallocate(fd, stats.st_size(), newSize)) { + logger.debug("pre-allocated file [{}] to {} bytes", file, newSize); + } // OS specific preallocate logs its own errors + } + + if (libc.close(fd) != 0) { + logger.warn("Could not close file [" + file + "] after trying to preallocate size: " + libc.strerror(libc.errno())); + } + } + + protected abstract boolean nativePreallocate(int fd, long currentSize, long newSize); + @Override public Optional getVectorSimilarityFunctions() { return Optional.ofNullable(vectorDistance); diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/WindowsNativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/WindowsNativeAccess.java index a9ccd15330595..5b4a5abad3e0a 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/WindowsNativeAccess.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/WindowsNativeAccess.java @@ -12,7 +12,11 @@ import org.elasticsearch.nativeaccess.lib.Kernel32Library.Handle; import org.elasticsearch.nativeaccess.lib.NativeLibraryProvider; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; import static java.lang.management.ManagementFactory.getMemoryMXBean; @@ -27,6 +31,8 @@ class WindowsNativeAccess extends AbstractNativeAccess { public static final int PAGE_GUARD = 0x0100; public static final int MEM_COMMIT = 0x1000; + private static final int INVALID_FILE_SIZE = -1; + /** * Constant for JOBOBJECT_BASIC_LIMIT_INFORMATION in Query/Set InformationJobObject */ @@ -119,6 +125,37 @@ public void tryInstallExecSandbox() { logger.debug("Windows ActiveProcessLimit initialization successful"); } + @Override + public OptionalLong allocatedSizeInBytes(Path path) { + assert Files.isRegularFile(path) : path; + String fileName = "\\\\?\\" + path; + AtomicInteger lpFileSizeHigh = new AtomicInteger(); + + final int lpFileSizeLow = kernel.GetCompressedFileSizeW(fileName, lpFileSizeHigh::set); + if (lpFileSizeLow == INVALID_FILE_SIZE) { + logger.warn("Unable to get allocated size of file [{}]. Error code {}", path, kernel.GetLastError()); + return OptionalLong.empty(); + } + + // convert lpFileSizeLow to unsigned long and combine with signed/shifted lpFileSizeHigh + final long allocatedSize = (((long) lpFileSizeHigh.get()) << Integer.SIZE) | Integer.toUnsignedLong(lpFileSizeLow); + if (logger.isTraceEnabled()) { + logger.trace( + "executing native method GetCompressedFileSizeW returned [high={}, low={}, allocated={}] for file [{}]", + lpFileSizeHigh.get(), + lpFileSizeLow, + allocatedSize, + path + ); + } + return OptionalLong.of(allocatedSize); + } + + @Override + public void tryPreallocate(Path file, long size) { + logger.warn("Cannot preallocate file size because operation is not available on Windows"); + } + @Override public ProcessLimits getProcessLimits() { return new ProcessLimits(ProcessLimits.UNKNOWN, ProcessLimits.UNKNOWN, ProcessLimits.UNKNOWN); diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/Kernel32Library.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/Kernel32Library.java index dd786b56087e2..f35d9fde5950d 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/Kernel32Library.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/Kernel32Library.java @@ -10,6 +10,8 @@ import org.elasticsearch.nativeaccess.WindowsFunctions.ConsoleCtrlHandler; +import java.util.function.IntConsumer; + public non-sealed interface Kernel32Library extends NativeLibrary { interface Handle {} @@ -81,6 +83,17 @@ interface MemoryBasicInformation { */ boolean SetProcessWorkingSetSize(Handle handle, long minSize, long maxSize); + /** + * Retrieves the actual number of bytes of disk storage used to store a specified file. + * + * https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getcompressedfilesizew + * + * @param lpFileName the path string + * @param lpFileSizeHigh pointer to high-order DWORD for compressed file size (or null if not needed) + * @return the low-order DWORD for compressed file size + */ + int GetCompressedFileSizeW(String lpFileName, IntConsumer lpFileSizeHigh); + /** * Retrieves the short path form of the specified path. * diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LinuxCLibrary.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LinuxCLibrary.java index 2a7b10ff3588f..8a2917e136bde 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LinuxCLibrary.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LinuxCLibrary.java @@ -35,4 +35,6 @@ interface SockFProg { * this is the only way, DON'T use it on some other architecture unless you know wtf you are doing */ long syscall(long number, int operation, int flags, long address); + + int fallocate(int fd, int mode, long offset, long length); } diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/PosixCLibrary.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/PosixCLibrary.java index d8db5fa070126..0e7d07d0ad623 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/PosixCLibrary.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/PosixCLibrary.java @@ -55,6 +55,25 @@ interface RLimit { */ int mlockall(int flags); + /** corresponds to struct stat64 */ + interface Stat64 { + long st_size(); + + long st_blocks(); + } + + Stat64 newStat64(int sizeof, int stSizeOffset, int stBlocksOffset); + + int open(String pathname, int flags, int mode); + + int open(String pathname, int flags); + + int close(int fd); + + int fstat64(int fd, Stat64 stats); + + int ftruncate(int fd, long length); + interface FStore { void set_flags(int flags); /* IN: flags word */ diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkKernel32Library.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkKernel32Library.java index f5eb5238dad93..a3ddc0d59890d 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkKernel32Library.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkKernel32Library.java @@ -20,6 +20,7 @@ import java.lang.invoke.MethodHandle; import java.lang.invoke.VarHandle; import java.nio.charset.StandardCharsets; +import java.util.function.IntConsumer; import static java.lang.foreign.MemoryLayout.PathElement.groupElement; import static java.lang.foreign.MemoryLayout.paddingLayout; @@ -57,6 +58,10 @@ class JdkKernel32Library implements Kernel32Library { "SetProcessWorkingSetSize", FunctionDescriptor.of(ADDRESS, JAVA_LONG, JAVA_LONG) ); + private static final MethodHandle GetCompressedFileSizeW$mh = downcallHandleWithError( + "GetCompressedFileSizeW", + FunctionDescriptor.of(JAVA_INT, ADDRESS, ADDRESS) + ); private static final MethodHandle GetShortPathNameW$mh = downcallHandleWithError( "GetShortPathNameW", FunctionDescriptor.of(JAVA_INT, ADDRESS, ADDRESS, JAVA_INT) @@ -276,6 +281,20 @@ public boolean SetProcessWorkingSetSize(Handle process, long minSize, long maxSi } } + @Override + public int GetCompressedFileSizeW(String lpFileName, IntConsumer lpFileSizeHigh) { + try (Arena arena = Arena.ofConfined()) { + MemorySegment wideFileName = ArenaUtil.allocateFrom(arena, lpFileName + "\0", StandardCharsets.UTF_16LE); + MemorySegment fileSizeHigh = arena.allocate(JAVA_INT); + + int ret = (int) GetCompressedFileSizeW$mh.invokeExact(lastErrorState, wideFileName, fileSizeHigh); + lpFileSizeHigh.accept(fileSizeHigh.get(JAVA_INT, 0)); + return ret; + } catch (Throwable t) { + throw new AssertionError(t); + } + } + @Override public int GetShortPathNameW(String lpszLongPath, char[] lpszShortPath, int cchBuffer) { try (Arena arena = Arena.ofConfined()) { diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkLinuxCLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkLinuxCLibrary.java index 700941e7e1db0..a31f212eab382 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkLinuxCLibrary.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkLinuxCLibrary.java @@ -49,6 +49,10 @@ class JdkLinuxCLibrary implements LinuxCLibrary { CAPTURE_ERRNO_OPTION, Linker.Option.firstVariadicArg(1) ); + private static final MethodHandle fallocate$mh = downcallHandleWithErrno( + "fallocate", + FunctionDescriptor.of(JAVA_INT, JAVA_INT, JAVA_INT, JAVA_LONG, JAVA_LONG) + ); private static class JdkSockFProg implements SockFProg { private static final MemoryLayout layout = MemoryLayout.structLayout(JAVA_SHORT, paddingLayout(6), ADDRESS); @@ -100,4 +104,13 @@ public long syscall(long number, int operation, int flags, long address) { throw new AssertionError(t); } } + + @Override + public int fallocate(int fd, int mode, long offset, long length) { + try { + return (int) fallocate$mh.invokeExact(errnoState, fd, mode, offset, length); + } catch (Throwable t) { + throw new AssertionError(t); + } + } } diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkPosixCLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkPosixCLibrary.java index 1a65225873c1d..7affd0614461d 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkPosixCLibrary.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkPosixCLibrary.java @@ -19,6 +19,7 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.StructLayout; import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; import java.lang.invoke.VarHandle; import static java.lang.foreign.MemoryLayout.PathElement.groupElement; @@ -48,7 +49,46 @@ class JdkPosixCLibrary implements PosixCLibrary { FunctionDescriptor.of(JAVA_INT, JAVA_INT, ADDRESS) ); private static final MethodHandle mlockall$mh = downcallHandleWithErrno("mlockall", FunctionDescriptor.of(JAVA_INT, JAVA_INT)); - private static final MethodHandle fcntl$mh = downcallHandle("fcntl", FunctionDescriptor.of(JAVA_INT, JAVA_INT, JAVA_INT, ADDRESS)); + private static final MethodHandle fcntl$mh = downcallHandle( + "fcntl", + FunctionDescriptor.of(JAVA_INT, JAVA_INT, JAVA_INT, ADDRESS), + CAPTURE_ERRNO_OPTION, + Linker.Option.firstVariadicArg(2) + ); + private static final MethodHandle ftruncate$mh = downcallHandleWithErrno( + "ftruncate", + FunctionDescriptor.of(JAVA_INT, JAVA_INT, JAVA_LONG) + ); + private static final MethodHandle open$mh = downcallHandle( + "open", + FunctionDescriptor.of(JAVA_INT, ADDRESS, JAVA_INT), + CAPTURE_ERRNO_OPTION, + Linker.Option.firstVariadicArg(2) + ); + private static final MethodHandle openWithMode$mh = downcallHandle( + "open", + FunctionDescriptor.of(JAVA_INT, ADDRESS, JAVA_INT, JAVA_INT), + CAPTURE_ERRNO_OPTION, + Linker.Option.firstVariadicArg(2) + ); + private static final MethodHandle close$mh = downcallHandleWithErrno("close", FunctionDescriptor.of(JAVA_INT, JAVA_INT)); + private static final MethodHandle fstat$mh; + static { + MethodHandle fstat; + try { + fstat = downcallHandleWithErrno("fstat64", FunctionDescriptor.of(JAVA_INT, JAVA_INT, ADDRESS)); + } catch (LinkageError e) { + // Due to different sizes of the stat structure for 32 vs 64 bit machines, on some systems fstat actually points to + // an internal symbol. So we fall back to looking for that symbol. + int version = System.getProperty("os.arch").equals("aarch64") ? 0 : 1; + fstat = MethodHandles.insertArguments( + downcallHandleWithErrno("__fxstat", FunctionDescriptor.of(JAVA_INT, JAVA_INT, JAVA_INT, ADDRESS)), + 1, + version + ); + } + fstat$mh = fstat; + } static final MemorySegment errnoState = Arena.ofAuto().allocate(CAPTURE_ERRNO_LAYOUT); @@ -85,6 +125,16 @@ public RLimit newRLimit() { return new JdkRLimit(); } + @Override + public Stat64 newStat64(int sizeof, int stSizeOffset, int stBlocksOffset) { + return new JdkStat64(sizeof, stSizeOffset, stBlocksOffset); + } + + @Override + public FStore newFStore() { + return new JdkFStore(); + } + @Override public int getrlimit(int resource, RLimit rlimit) { assert rlimit instanceof JdkRLimit; @@ -116,11 +166,6 @@ public int mlockall(int flags) { } } - @Override - public FStore newFStore() { - return new JdkFStore(); - } - @Override public int fcntl(int fd, int cmd, FStore fst) { assert fst instanceof JdkFStore; @@ -132,6 +177,55 @@ public int fcntl(int fd, int cmd, FStore fst) { } } + @Override + public int ftruncate(int fd, long length) { + try { + return (int) ftruncate$mh.invokeExact(errnoState, fd, length); + } catch (Throwable t) { + throw new AssertionError(t); + } + } + + @Override + public int open(String pathname, int flags) { + try (Arena arena = Arena.ofConfined()) { + MemorySegment nativePathname = MemorySegmentUtil.allocateString(arena, pathname); + return (int) open$mh.invokeExact(errnoState, nativePathname, flags); + } catch (Throwable t) { + throw new AssertionError(t); + } + } + + @Override + public int open(String pathname, int flags, int mode) { + try (Arena arena = Arena.ofConfined()) { + MemorySegment nativePathname = MemorySegmentUtil.allocateString(arena, pathname); + return (int) openWithMode$mh.invokeExact(errnoState, nativePathname, flags, mode); + } catch (Throwable t) { + throw new AssertionError(t); + } + } + + @Override + public int close(int fd) { + try { + return (int) close$mh.invokeExact(errnoState, fd); + } catch (Throwable t) { + throw new AssertionError(t); + } + } + + @Override + public int fstat64(int fd, Stat64 stat64) { + assert stat64 instanceof JdkStat64; + var jdkStat = (JdkStat64) stat64; + try { + return (int) fstat$mh.invokeExact(errnoState, fd, jdkStat.segment); + } catch (Throwable t) { + throw new AssertionError(t); + } + } + static class JdkRLimit implements RLimit { private static final MemoryLayout layout = MemoryLayout.structLayout(JAVA_LONG, JAVA_LONG); private static final VarHandle rlim_cur$vh = varHandleWithoutOffset(layout, groupElement(0)); @@ -170,19 +264,41 @@ public String toString() { } } + private static class JdkStat64 implements Stat64 { + + private final MemorySegment segment; + private final int stSizeOffset; + private final int stBlocksOffset; + + JdkStat64(int sizeof, int stSizeOffset, int stBlocksOffset) { + this.segment = Arena.ofAuto().allocate(sizeof, 8); + this.stSizeOffset = stSizeOffset; + this.stBlocksOffset = stBlocksOffset; + } + + @Override + public long st_size() { + return segment.get(JAVA_LONG, stSizeOffset); + } + + @Override + public long st_blocks() { + return segment.get(JAVA_LONG, stBlocksOffset); + } + } + private static class JdkFStore implements FStore { private static final MemoryLayout layout = MemoryLayout.structLayout(JAVA_INT, JAVA_INT, JAVA_LONG, JAVA_LONG, JAVA_LONG); - private static final VarHandle st_flags$vh = layout.varHandle(groupElement(0)); - private static final VarHandle st_posmode$vh = layout.varHandle(groupElement(1)); - private static final VarHandle st_offset$vh = layout.varHandle(groupElement(2)); - private static final VarHandle st_length$vh = layout.varHandle(groupElement(3)); - private static final VarHandle st_bytesalloc$vh = layout.varHandle(groupElement(4)); + private static final VarHandle st_flags$vh = varHandleWithoutOffset(layout, groupElement(0)); + private static final VarHandle st_posmode$vh = varHandleWithoutOffset(layout, groupElement(1)); + private static final VarHandle st_offset$vh = varHandleWithoutOffset(layout, groupElement(2)); + private static final VarHandle st_length$vh = varHandleWithoutOffset(layout, groupElement(3)); + private static final VarHandle st_bytesalloc$vh = varHandleWithoutOffset(layout, groupElement(4)); private final MemorySegment segment; JdkFStore() { - var arena = Arena.ofAuto(); - this.segment = arena.allocate(layout); + this.segment = Arena.ofAuto().allocate(layout); } @Override @@ -197,7 +313,7 @@ public void set_posmode(int posmode) { @Override public void set_offset(long offset) { - st_offset$vh.get(segment, offset); + st_offset$vh.set(segment, offset); } @Override diff --git a/libs/native/src/test/java/org/elasticsearch/nativeaccess/PreallocateTests.java b/libs/native/src/test/java/org/elasticsearch/nativeaccess/PreallocateTests.java new file mode 100644 index 0000000000000..c5d427c3aa47b --- /dev/null +++ b/libs/native/src/test/java/org/elasticsearch/nativeaccess/PreallocateTests.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.nativeaccess; + +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.OptionalLong; + +import static org.hamcrest.Matchers.equalTo; + +public class PreallocateTests extends ESTestCase { + public void testPreallocate() throws IOException { + assumeFalse("no preallocate on windows", System.getProperty("os.name").startsWith("Windows")); + Path file = createTempFile(); + long size = 1024 * 1024; // 1 MB + var nativeAccess = NativeAccess.instance(); + nativeAccess.tryPreallocate(file, size); + OptionalLong foundSize = nativeAccess.allocatedSizeInBytes(file); + assertTrue(foundSize.isPresent()); + assertThat(foundSize.getAsLong(), equalTo(size)); + } +} diff --git a/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSystemPropertyTests.java b/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSystemPropertyTests.java index 71da1b4b403f8..9875878d8658a 100644 --- a/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSystemPropertyTests.java +++ b/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSystemPropertyTests.java @@ -8,8 +8,8 @@ package org.elasticsearch.nativeaccess; -import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase.WithoutSecurityManager; import org.elasticsearch.test.compiler.InMemoryJavaCompiler; import org.elasticsearch.test.jar.JarUtils; @@ -27,7 +27,7 @@ import static org.hamcrest.Matchers.equalTo; @WithoutSecurityManager -public class VectorSystemPropertyTests extends LuceneTestCase { +public class VectorSystemPropertyTests extends ESTestCase { static Path jarPath; diff --git a/libs/preallocate/build.gradle b/libs/preallocate/build.gradle deleted file mode 100644 index a490c7168516e..0000000000000 --- a/libs/preallocate/build.gradle +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -apply plugin: 'elasticsearch.build' - -dependencies { - implementation project(':libs:elasticsearch-core') - implementation project(':libs:elasticsearch-logging') - implementation "net.java.dev.jna:jna:${versions.jna}" -} - -tasks.named('forbiddenApisMain').configure { - replaceSignatureFiles 'jdk-signatures' -} diff --git a/libs/preallocate/src/main/java/module-info.java b/libs/preallocate/src/main/java/module-info.java deleted file mode 100644 index 89c85d95ab2f0..0000000000000 --- a/libs/preallocate/src/main/java/module-info.java +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -module org.elasticsearch.preallocate { - requires org.elasticsearch.base; - requires org.elasticsearch.logging; - requires com.sun.jna; - - exports org.elasticsearch.preallocate to org.elasticsearch.blobcache, com.sun.jna; - - provides org.elasticsearch.jdk.ModuleQualifiedExportsService with org.elasticsearch.preallocate.PreallocateModuleExportsService; -} diff --git a/libs/preallocate/src/main/java/org/elasticsearch/preallocate/AbstractPosixPreallocator.java b/libs/preallocate/src/main/java/org/elasticsearch/preallocate/AbstractPosixPreallocator.java deleted file mode 100644 index e841b38c0059e..0000000000000 --- a/libs/preallocate/src/main/java/org/elasticsearch/preallocate/AbstractPosixPreallocator.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.preallocate; - -import com.sun.jna.FunctionMapper; -import com.sun.jna.Library; -import com.sun.jna.Native; -import com.sun.jna.NativeLong; -import com.sun.jna.Platform; -import com.sun.jna.Structure; - -import java.io.IOException; -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.util.Locale; -import java.util.Map; - -abstract class AbstractPosixPreallocator implements Preallocator { - - /** - * Constants relating to posix libc. - * - * @param SIZEOF_STAT The size of the stat64 structure, ie sizeof(stat64_t), found by importing sys/stat.h - * @param STAT_ST_SIZE_OFFSET The offsite into stat64 at which st_size exists, ie offsetof(stat64_t, st_size), - * found by importing sys/stat.h - * @param O_CREAT The file mode for creating a file upon opening, found by importing fcntl.h - */ - protected record PosixConstants(int SIZEOF_STAT, int STAT_ST_SIZE_OFFSET, int O_CREAT) {} - - private static final int O_WRONLY = 1; - - static final class Stat64 extends Structure implements Structure.ByReference { - public byte[] _ignore1; - public NativeLong st_size = new NativeLong(0); - public byte[] _ignore2; - - Stat64(int sizeof, int stSizeOffset) { - this._ignore1 = new byte[stSizeOffset]; - this._ignore2 = new byte[sizeof - stSizeOffset - 8]; - } - } - - private interface NativeFunctions extends Library { - String strerror(int errno); - - int open(String filename, int flags, Object... mode); - - int close(int fd); - } - - private interface FStat64Function extends Library { - int fstat64(int fd, Stat64 stat); - } - - public static final boolean NATIVES_AVAILABLE; - private static final NativeFunctions functions; - private static final FStat64Function fstat64; - - static { - functions = AccessController.doPrivileged((PrivilegedAction) () -> { - try { - return Native.load(Platform.C_LIBRARY_NAME, NativeFunctions.class); - } catch (final UnsatisfiedLinkError e) { - return null; - } - }); - fstat64 = AccessController.doPrivileged((PrivilegedAction) () -> { - try { - return Native.load(Platform.C_LIBRARY_NAME, FStat64Function.class); - } catch (final UnsatisfiedLinkError e) { - try { - // on Linux fstat64 isn't available as a symbol, but instead uses a special __ name - var options = Map.of(Library.OPTION_FUNCTION_MAPPER, (FunctionMapper) (lib, method) -> "__fxstat64"); - return Native.load(Platform.C_LIBRARY_NAME, FStat64Function.class, options); - } catch (UnsatisfiedLinkError e2) { - return null; - } - } - }); - NATIVES_AVAILABLE = functions != null && fstat64 != null; - } - - private class PosixNativeFileHandle implements NativeFileHandle { - - private final int fd; - - PosixNativeFileHandle(int fd) { - this.fd = fd; - } - - @Override - public int fd() { - return fd; - } - - @Override - public long getSize() throws IOException { - var stat = new Stat64(constants.SIZEOF_STAT, constants.STAT_ST_SIZE_OFFSET); - if (fstat64.fstat64(fd, stat) == -1) { - throw newIOException("Could not get size of file"); - } - return stat.st_size.longValue(); - } - - @Override - public void close() throws IOException { - if (functions.close(fd) != 0) { - throw newIOException("Could not close file"); - } - } - } - - protected final PosixConstants constants; - - AbstractPosixPreallocator(PosixConstants constants) { - this.constants = constants; - } - - @Override - public boolean useNative() { - return false; - } - - @Override - public NativeFileHandle open(String path) throws IOException { - int fd = functions.open(path, O_WRONLY, constants.O_CREAT); - if (fd < 0) { - throw newIOException(String.format(Locale.ROOT, "Could not open file [%s] for preallocation", path)); - } - return new PosixNativeFileHandle(fd); - } - - @Override - public String error(int errno) { - return functions.strerror(errno); - } - - private static IOException newIOException(String prefix) { - int errno = Native.getLastError(); - return new IOException(String.format(Locale.ROOT, "%s(errno=%d): %s", prefix, errno, functions.strerror(errno))); - } -} diff --git a/libs/preallocate/src/main/java/org/elasticsearch/preallocate/LinuxPreallocator.java b/libs/preallocate/src/main/java/org/elasticsearch/preallocate/LinuxPreallocator.java deleted file mode 100644 index 25ad4a26fd03e..0000000000000 --- a/libs/preallocate/src/main/java/org/elasticsearch/preallocate/LinuxPreallocator.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.preallocate; - -import com.sun.jna.Native; -import com.sun.jna.Platform; - -import java.security.AccessController; -import java.security.PrivilegedAction; - -final class LinuxPreallocator extends AbstractPosixPreallocator { - - LinuxPreallocator() { - super(new PosixConstants(144, 48, 64)); - } - - @Override - public boolean useNative() { - return Natives.NATIVES_AVAILABLE && super.useNative(); - } - - @Override - public int preallocate(final int fd, final long currentSize, final long fileSize) { - final int rc = Natives.fallocate(fd, 0, currentSize, fileSize - currentSize); - return rc == 0 ? 0 : Native.getLastError(); - } - - private static class Natives { - - public static final boolean NATIVES_AVAILABLE; - - static { - NATIVES_AVAILABLE = AccessController.doPrivileged((PrivilegedAction) () -> { - try { - Native.register(Natives.class, Platform.C_LIBRARY_NAME); - } catch (final UnsatisfiedLinkError e) { - return false; - } - return true; - }); - } - - static native int fallocate(int fd, int mode, long offset, long length); - } - -} diff --git a/libs/preallocate/src/main/java/org/elasticsearch/preallocate/MacOsPreallocator.java b/libs/preallocate/src/main/java/org/elasticsearch/preallocate/MacOsPreallocator.java deleted file mode 100644 index 149cf80527bd0..0000000000000 --- a/libs/preallocate/src/main/java/org/elasticsearch/preallocate/MacOsPreallocator.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.preallocate; - -import com.sun.jna.Native; -import com.sun.jna.NativeLong; -import com.sun.jna.Platform; -import com.sun.jna.Structure; - -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.util.Arrays; -import java.util.List; - -final class MacOsPreallocator extends AbstractPosixPreallocator { - - MacOsPreallocator() { - super(new PosixConstants(144, 96, 512)); - } - - @Override - public boolean useNative() { - return Natives.NATIVES_AVAILABLE && super.useNative(); - } - - @Override - public int preallocate(final int fd, final long currentSize /* unused */ , final long fileSize) { - // the Structure.ByReference constructor requires access to declared members - final Natives.Fcntl.FStore fst = AccessController.doPrivileged((PrivilegedAction) Natives.Fcntl.FStore::new); - fst.fst_flags = Natives.Fcntl.F_ALLOCATECONTIG; - fst.fst_posmode = Natives.Fcntl.F_PEOFPOSMODE; - fst.fst_offset = new NativeLong(0); - fst.fst_length = new NativeLong(fileSize); - // first, try allocating contiguously - if (Natives.fcntl(fd, Natives.Fcntl.F_PREALLOCATE, fst) != 0) { - // that failed, so let us try allocating non-contiguously - fst.fst_flags = Natives.Fcntl.F_ALLOCATEALL; - if (Natives.fcntl(fd, Natives.Fcntl.F_PREALLOCATE, fst) != 0) { - // i'm afraid captain dale had to bail - return Native.getLastError(); - } - } - if (Natives.ftruncate(fd, new NativeLong(fileSize)) != 0) { - return Native.getLastError(); - } - return 0; - } - - private static class Natives { - - static boolean NATIVES_AVAILABLE; - - static { - NATIVES_AVAILABLE = AccessController.doPrivileged((PrivilegedAction) () -> { - try { - Native.register(Natives.class, Platform.C_LIBRARY_NAME); - } catch (final UnsatisfiedLinkError e) { - return false; - } - return true; - }); - } - - static class Fcntl { - private static final int F_PREALLOCATE = 42; - - // allocate flags; these might be unused, but are here for reference - @SuppressWarnings("unused") - private static final int F_ALLOCATECONTIG = 0x00000002; // allocate contiguous space - private static final int F_ALLOCATEALL = 0x00000004; // allocate all the requested space or no space at all - - // position modes; these might be unused, but are here for reference - private static final int F_PEOFPOSMODE = 3; // allocate from the physical end of the file - @SuppressWarnings("unused") - private static final int F_VOLPOSMODE = 4; // allocate from the volume offset - - public static final class FStore extends Structure implements Structure.ByReference { - public int fst_flags = 0; - public int fst_posmode = 0; - public NativeLong fst_offset = new NativeLong(0); - public NativeLong fst_length = new NativeLong(0); - @SuppressWarnings("unused") - public NativeLong fst_bytesalloc = new NativeLong(0); - - @Override - protected List getFieldOrder() { - return Arrays.asList("fst_flags", "fst_posmode", "fst_offset", "fst_length", "fst_bytesalloc"); - } - - } - } - - static native int fcntl(int fd, int cmd, Fcntl.FStore fst); - - static native int ftruncate(int fd, NativeLong length); - } - -} diff --git a/libs/preallocate/src/main/java/org/elasticsearch/preallocate/NoNativePreallocator.java b/libs/preallocate/src/main/java/org/elasticsearch/preallocate/NoNativePreallocator.java deleted file mode 100644 index 447b178ba41d9..0000000000000 --- a/libs/preallocate/src/main/java/org/elasticsearch/preallocate/NoNativePreallocator.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.preallocate; - -import java.io.IOException; - -final class NoNativePreallocator implements Preallocator { - - @Override - public boolean useNative() { - return false; - } - - @Override - public NativeFileHandle open(String path) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public int preallocate(final int fd, final long currentSize, final long fileSize) { - throw new UnsupportedOperationException(); - } - - @Override - public String error(final int errno) { - throw new UnsupportedOperationException(); - } - -} diff --git a/libs/preallocate/src/main/java/org/elasticsearch/preallocate/Preallocate.java b/libs/preallocate/src/main/java/org/elasticsearch/preallocate/Preallocate.java deleted file mode 100644 index 8f7214e0877ba..0000000000000 --- a/libs/preallocate/src/main/java/org/elasticsearch/preallocate/Preallocate.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.preallocate; - -import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.logging.LogManager; -import org.elasticsearch.logging.Logger; -import org.elasticsearch.preallocate.Preallocator.NativeFileHandle; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.lang.reflect.Field; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.PrivilegedExceptionAction; - -public class Preallocate { - - private static final Logger logger = LogManager.getLogger(Preallocate.class); - - private static final boolean IS_LINUX; - private static final boolean IS_MACOS; - static { - String osName = System.getProperty("os.name"); - IS_LINUX = osName.startsWith("Linux"); - IS_MACOS = osName.startsWith("Mac OS X"); - } - - public static void preallocate(final Path cacheFile, final long fileSize) throws IOException { - if (IS_LINUX) { - preallocate(cacheFile, fileSize, new LinuxPreallocator()); - } else if (IS_MACOS) { - preallocate(cacheFile, fileSize, new MacOsPreallocator()); - } else { - preallocate(cacheFile, fileSize, new NoNativePreallocator()); - } - } - - @SuppressForbidden(reason = "need access to toFile for RandomAccessFile") - private static void preallocate(final Path cacheFile, final long fileSize, final Preallocator prealloactor) throws IOException { - boolean success = false; - try { - if (prealloactor.useNative()) { - try (NativeFileHandle openFile = prealloactor.open(cacheFile.toAbsolutePath().toString())) { - long currentSize = openFile.getSize(); - if (currentSize < fileSize) { - logger.info("pre-allocating cache file [{}] ({} bytes) using native methods", cacheFile, fileSize); - final int errno = prealloactor.preallocate(openFile.fd(), currentSize, fileSize - currentSize); - if (errno == 0) { - success = true; - logger.debug("pre-allocated cache file [{}] using native methods", cacheFile); - } else { - logger.warn( - "failed to pre-allocate cache file [{}] using native methods, errno: [{}], error: [{}]", - cacheFile, - errno, - prealloactor.error(errno) - ); - } - } - } catch (final Exception e) { - logger.warn(() -> "failed to pre-allocate cache file [" + cacheFile + "] using native methods", e); - } - } - // even if allocation was successful above, verify again here - try (RandomAccessFile raf = new RandomAccessFile(cacheFile.toFile(), "rw")) { - if (raf.length() != fileSize) { - logger.info("pre-allocating cache file [{}] ({} bytes) using setLength method", cacheFile, fileSize); - raf.setLength(fileSize); - logger.debug("pre-allocated cache file [{}] using setLength method", cacheFile); - } - success = raf.length() == fileSize; - } catch (final Exception e) { - logger.warn(() -> "failed to pre-allocate cache file [" + cacheFile + "] using setLength method", e); - throw e; - } - } finally { - if (success == false) { - // if anything goes wrong, delete the potentially created file to not waste disk space - Files.deleteIfExists(cacheFile); - } - } - } - - @SuppressForbidden(reason = "need access to fd on FileOutputStream") - private static class FileDescriptorFieldAction implements PrivilegedExceptionAction { - - private final FileOutputStream fileOutputStream; - - private FileDescriptorFieldAction(FileOutputStream fileOutputStream) { - this.fileOutputStream = fileOutputStream; - } - - @Override - public Field run() throws IOException, NoSuchFieldException { - // accessDeclaredMembers - final Field f = fileOutputStream.getFD().getClass().getDeclaredField("fd"); - // suppressAccessChecks - f.setAccessible(true); - return f; - } - } - -} diff --git a/libs/preallocate/src/main/java/org/elasticsearch/preallocate/PreallocateModuleExportsService.java b/libs/preallocate/src/main/java/org/elasticsearch/preallocate/PreallocateModuleExportsService.java deleted file mode 100644 index dd0c4236f2c75..0000000000000 --- a/libs/preallocate/src/main/java/org/elasticsearch/preallocate/PreallocateModuleExportsService.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.preallocate; - -import org.elasticsearch.jdk.ModuleQualifiedExportsService; - -public class PreallocateModuleExportsService extends ModuleQualifiedExportsService { - - @Override - protected void addExports(String pkg, Module target) { - module.addExports(pkg, target); - } - - @Override - protected void addOpens(String pkg, Module target) { - module.addOpens(pkg, target); - } -} diff --git a/libs/preallocate/src/main/java/org/elasticsearch/preallocate/Preallocator.java b/libs/preallocate/src/main/java/org/elasticsearch/preallocate/Preallocator.java deleted file mode 100644 index b70b3ff03f4bd..0000000000000 --- a/libs/preallocate/src/main/java/org/elasticsearch/preallocate/Preallocator.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.preallocate; - -import java.io.IOException; - -/** - * Represents platform native methods for pre-allocating files. - */ -interface Preallocator { - - /** A handle for an open file */ - interface NativeFileHandle extends AutoCloseable { - /** A valid native file descriptor */ - int fd(); - - /** Retrieves the current size of the file */ - long getSize() throws IOException; - } - - /** - * Returns if native methods for pre-allocating files are available. - * - * @return true if native methods are available, otherwise false - */ - boolean useNative(); - - /** - * Open a file for preallocation. - * - * @param path The absolute path to the file to be opened - * @return a handle to the open file that may be used for preallocate - */ - NativeFileHandle open(String path) throws IOException; - - /** - * Pre-allocate a file of given current size to the specified size using the given file descriptor. - * - * @param fd the file descriptor - * @param currentSize the current size of the file - * @param fileSize the size to pre-allocate - * @return 0 upon success - */ - int preallocate(int fd, long currentSize, long fileSize); - - /** - * Provide a string representation of the given error number. - * - * @param errno the error number - * @return the error message - */ - String error(int errno); - -} diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogram.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogram.java index 1e3042f8cf1e4..b813c9ec50c83 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogram.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogram.java @@ -213,7 +213,7 @@ public InternalAutoDateHistogram(StreamInput in) throws IOException { bucketInnerInterval = 1; // Calculated on merge. } // we changed the order format in 8.13 for partial reduce, therefore we need to order them to perform merge sort - if (in.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.HISTOGRAM_AGGS_KEY_SORTED)) { + if (in.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.V_8_14_0)) { // list is mutable by #readCollectionAsList contract buckets.sort(Comparator.comparingLong(b -> b.key)); } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EdgeNGramTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EdgeNGramTokenFilterFactory.java index 62986f7732437..07f2dd83ec94b 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EdgeNGramTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EdgeNGramTokenFilterFactory.java @@ -11,6 +11,8 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter; import org.apache.lucene.analysis.reverse.ReverseStringFilter; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; @@ -19,6 +21,8 @@ public class EdgeNGramTokenFilterFactory extends AbstractTokenFilterFactory { + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(EdgeNGramTokenFilterFactory.class); + private final int minGram; private final int maxGram; @@ -33,6 +37,13 @@ public class EdgeNGramTokenFilterFactory extends AbstractTokenFilterFactory { super(name, settings); this.minGram = settings.getAsInt("min_gram", 1); this.maxGram = settings.getAsInt("max_gram", 2); + if (settings.get("side") != null) { + deprecationLogger.critical( + DeprecationCategory.ANALYSIS, + "edge_ngram_side_deprecated", + "The [side] parameter is deprecated and will be removed. Use a [reverse] before and after the [edge_ngram] instead." + ); + } this.side = parseSide(settings.get("side", "front")); this.preserveOriginal = settings.getAsBoolean(PRESERVE_ORIG_KEY, false); } diff --git a/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/40_token_filters.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/40_token_filters.yml index 2e6c445dc5e59..1469cc2fa2548 100644 --- a/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/40_token_filters.yml +++ b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/40_token_filters.yml @@ -548,7 +548,27 @@ - match: { tokens.1.token: foob } - match: { tokens.2.token: fooba } - match: { tokens.3.token: foobar } +--- +"edge_ngram side deprecated": + - requires: + cluster_features: "gte_v8.16.0" + reason: 'side parameter is deprecated since 8.16.0' + test_runner_features: warnings + - do: + warnings: + - "The [side] parameter is deprecated and will be removed. Use a [reverse] before and after the [edge_ngram] instead." + indices.create: + index: test + body: + settings: + analysis: + filter: + my_edge_ngram: + type: edge_ngram + min_gram: 3 + side: front + max_gram: 6 --- "kstem": - do: diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index f95d9a0b0431f..52ce2a7a33ea6 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -165,7 +165,7 @@ public void testLogsIndexModeDataStreamIndexing() throws IOException, ExecutionE client(), "logs-composable-template", LOGS_OR_STANDARD_MAPPING, - Map.of("index.mode", "logs"), + Map.of("index.mode", "logsdb"), List.of("logs-*-*") ); final String dataStreamName = generateDataStreamName("logs"); @@ -188,7 +188,7 @@ public void testIndexModeLogsAndStandardSwitching() throws IOException, Executio ); createDataStream(client(), dataStreamName); for (int i = 0; i < randomIntBetween(5, 10); i++) { - final IndexMode indexMode = i % 2 == 0 ? IndexMode.LOGS : IndexMode.STANDARD; + final IndexMode indexMode = i % 2 == 0 ? IndexMode.LOGSDB : IndexMode.STANDARD; indexModes.add(indexMode); updateComposableIndexTemplate( client(), @@ -206,7 +206,7 @@ public void testIndexModeLogsAndStandardSwitching() throws IOException, Executio public void testIndexModeLogsAndTimeSeriesSwitching() throws IOException, ExecutionException, InterruptedException { final String dataStreamName = generateDataStreamName("custom"); final List indexPatterns = List.of("custom-*-*"); - final Map logsSettings = Map.of("index.mode", "logs"); + final Map logsSettings = Map.of("index.mode", "logsdb"); final Map timeSeriesSettings = Map.of("index.mode", "time_series", "index.routing_path", "host.name"); putComposableIndexTemplate(client(), "custom-composable-template", LOGS_OR_STANDARD_MAPPING, logsSettings, indexPatterns); @@ -221,13 +221,13 @@ public void testIndexModeLogsAndTimeSeriesSwitching() throws IOException, Execut rolloverDataStream(dataStreamName); indexLogOrStandardDocuments(client(), randomIntBetween(10, 20), randomIntBetween(32, 64), dataStreamName); - assertDataStreamBackingIndicesModes(dataStreamName, List.of(IndexMode.LOGS, IndexMode.TIME_SERIES, IndexMode.LOGS)); + assertDataStreamBackingIndicesModes(dataStreamName, List.of(IndexMode.LOGSDB, IndexMode.TIME_SERIES, IndexMode.LOGSDB)); } public void testInvalidIndexModeTimeSeriesSwitchWithoutRoutingPath() throws IOException, ExecutionException, InterruptedException { final String dataStreamName = generateDataStreamName("custom"); final List indexPatterns = List.of("custom-*-*"); - final Map logsSettings = Map.of("index.mode", "logs"); + final Map logsSettings = Map.of("index.mode", "logsdb"); final Map timeSeriesSettings = Map.of("index.mode", "time_series"); putComposableIndexTemplate(client(), "custom-composable-template", LOGS_OR_STANDARD_MAPPING, logsSettings, indexPatterns); @@ -249,7 +249,7 @@ public void testInvalidIndexModeTimeSeriesSwitchWithoutRoutingPath() throws IOEx public void testInvalidIndexModeTimeSeriesSwitchWithoutDimensions() throws IOException, ExecutionException, InterruptedException { final String dataStreamName = generateDataStreamName("custom"); final List indexPatterns = List.of("custom-*-*"); - final Map logsSettings = Map.of("index.mode", "logs"); + final Map logsSettings = Map.of("index.mode", "logsdb"); final Map timeSeriesSettings = Map.of("index.mode", "time_series", "index.routing_path", "host.name"); putComposableIndexTemplate(client(), "custom-composable-template", LOGS_OR_STANDARD_MAPPING, logsSettings, indexPatterns); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java index d3ec5b29ff5b9..780864db8b629 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java @@ -72,7 +72,7 @@ private static void waitForLogs(RestClient client) throws Exception { "template": { "settings": { "index": { - "mode": "logs" + "mode": "logsdb" } }, "mappings": { @@ -161,7 +161,7 @@ public void testLogsIndexing() throws IOException { randomIp(randomBoolean()) ) ); - assertDataStreamBackingIndexMode("logs", 0); + assertDataStreamBackingIndexMode("logsdb", 0); rolloverDataStream(client, DATA_STREAM_NAME); indexDocument( client, @@ -175,7 +175,7 @@ public void testLogsIndexing() throws IOException { randomIp(randomBoolean()) ) ); - assertDataStreamBackingIndexMode("logs", 1); + assertDataStreamBackingIndexMode("logsdb", 1); } public void testLogsStandardIndexModeSwitch() throws IOException { @@ -193,7 +193,7 @@ public void testLogsStandardIndexModeSwitch() throws IOException { randomIp(randomBoolean()) ) ); - assertDataStreamBackingIndexMode("logs", 0); + assertDataStreamBackingIndexMode("logsdb", 0); putTemplate(client, "custom-template", STANDARD_TEMPLATE); rolloverDataStream(client, DATA_STREAM_NAME); @@ -225,7 +225,7 @@ public void testLogsStandardIndexModeSwitch() throws IOException { randomIp(randomBoolean()) ) ); - assertDataStreamBackingIndexMode("logs", 2); + assertDataStreamBackingIndexMode("logsdb", 2); } private void assertDataStreamBackingIndexMode(final String indexMode, int backingIndex) throws IOException { diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java new file mode 100644 index 0000000000000..d3a2867fe2ecd --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java @@ -0,0 +1,328 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.logsdb; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.junit.Before; +import org.junit.ClassRule; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +@SuppressWarnings("unchecked") +public class LogsIndexModeCustomSettingsIT extends LogsIndexModeRestTestIT { + @ClassRule() + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .module("constant-keyword") + .module("data-streams") + .module("mapper-extras") + .module("x-pack-aggregate-metric") + .module("x-pack-stack") + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .setting("cluster.logsdb.enabled", "true") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Before + public void setup() throws Exception { + client = client(); + waitForLogs(client); + } + + private RestClient client; + + public void testOverrideIndexSorting() throws IOException { + var indexSortOverrideTemplate = """ + { + "template": { + "settings": { + "index": { + "sort.field": ["cluster", "custom.timestamp"], + "sort.order":["desc", "asc"] + } + }, + "mappings": { + "properties": { + "cluster": { + "type": "keyword" + }, + "custom": { + "properties": { + "timestamp": { + "type": "date" + } + } + } + } + } + } + }"""; + + assertOK(putComponentTemplate(client, "logs@custom", indexSortOverrideTemplate)); + assertOK(createDataStream(client, "logs-custom-dev")); + + var indexSortField = (List) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.sort.field"); + assertThat(indexSortField, equalTo(List.of("cluster", "custom.timestamp"))); + + var indexSortOrder = (List) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.sort.order"); + assertThat(indexSortOrder, equalTo(List.of("desc", "asc"))); + + // @timestamp is a default mapping and should still be present + var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0)); + String type = (String) subObject("properties").andThen(subObject("@timestamp")).apply(mapping).get("type"); + assertThat(type, equalTo("date")); + } + + public void testConfigureStoredSource() throws IOException { + var storedSourceMapping = """ + { + "template": { + "mappings": { + "_source": { + "mode": "stored" + } + } + } + }"""; + + Exception e = assertThrows(ResponseException.class, () -> putComponentTemplate(client, "logs@custom", storedSourceMapping)); + assertThat( + e.getMessage(), + containsString("updating component template [logs@custom] results in invalid composable template [logs]") + ); + assertThat(e.getMessage(), containsString("Indices with with index mode [logsdb] only support synthetic source")); + + assertOK(createDataStream(client, "logs-custom-dev")); + + var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0)); + String sourceMode = (String) subObject("_source").apply(mapping).get("mode"); + assertThat(sourceMode, equalTo("synthetic")); + } + + public void testOverrideIndexCodec() throws IOException { + var indexCodecOverrideTemplate = """ + { + "template": { + "settings": { + "index": { + "codec": "default" + } + } + } + }"""; + + assertOK(putComponentTemplate(client, "logs@custom", indexCodecOverrideTemplate)); + assertOK(createDataStream(client, "logs-custom-dev")); + + var indexCodec = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.codec"); + assertThat(indexCodec, equalTo("default")); + } + + public void testOverrideTimestampField() throws IOException { + var timestampMappingOverrideTemplate = """ + { + "template": { + "mappings": { + "properties": { + "@timestamp": { + "type": "date_nanos" + } + } + } + } + }"""; + + assertOK(putComponentTemplate(client, "logs@custom", timestampMappingOverrideTemplate)); + assertOK(createDataStream(client, "logs-custom-dev")); + + var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0)); + String type = (String) subObject("properties").andThen(subObject("@timestamp")).apply(mapping).get("type"); + assertThat(type, equalTo("date_nanos")); + } + + public void testOverrideHostNameField() throws IOException { + var timestampMappingOverrideTemplate = """ + { + "template": { + "mappings": { + "properties": { + "host": { + "properties": { + "name": { + "type": "keyword", + "index": false, + "ignore_above": 10 + } + } + } + } + } + } + }"""; + + assertOK(putComponentTemplate(client, "logs@custom", timestampMappingOverrideTemplate)); + assertOK(createDataStream(client, "logs-custom-dev")); + + var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0)); + var hostNameFieldParameters = subObject("properties").andThen(subObject("host")) + .andThen(subObject("properties")) + .andThen(subObject("name")) + .apply(mapping); + assertThat((String) hostNameFieldParameters.get("type"), equalTo("keyword")); + assertThat((boolean) hostNameFieldParameters.get("index"), equalTo(false)); + assertThat((int) hostNameFieldParameters.get("ignore_above"), equalTo(10)); + } + + public void testOverrideIndexSortingWithCustomTimestampField() throws IOException { + var timestampMappingAndIndexSortOverrideTemplate = """ + { + "template": { + "settings": { + "index": { + "sort.field": ["@timestamp", "cluster"], + "sort.order":["asc", "asc"] + } + }, + "mappings": { + "properties": { + "@timestamp": { + "type": "date_nanos" + }, + "cluster": { + "type": "keyword" + } + } + } + } + }"""; + + assertOK(putComponentTemplate(client, "logs@custom", timestampMappingAndIndexSortOverrideTemplate)); + assertOK(createDataStream(client, "logs-custom-dev")); + + var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0)); + String type = (String) subObject("properties").andThen(subObject("@timestamp")).apply(mapping).get("type"); + assertThat(type, equalTo("date_nanos")); + + var indexSortField = (List) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.sort.field"); + assertThat(indexSortField, equalTo(List.of("@timestamp", "cluster"))); + + var indexSortOrder = (List) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.sort.order"); + assertThat(indexSortOrder, equalTo(List.of("asc", "asc"))); + } + + public void testOverrideIgnoreMalformed() throws IOException { + var ignoreMalformedOverrideTemplate = """ + { + "template": { + "settings": { + "index": { + "mapping": { + "ignore_malformed": false + } + } + } + } + }"""; + + assertOK(putComponentTemplate(client, "logs@custom", ignoreMalformedOverrideTemplate)); + assertOK(createDataStream(client, "logs-custom-dev")); + + var ignoreMalformedIndexSetting = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + "index.mapping.ignore_malformed" + ); + assertThat(ignoreMalformedIndexSetting, equalTo("false")); + } + + public void testOverrideIgnoreDynamicBeyondLimit() throws IOException { + var ignoreMalformedOverrideTemplate = """ + { + "template": { + "settings": { + "index": { + "mapping": { + "total_fields": { + "ignore_dynamic_beyond_limit": false + } + } + } + } + } + }"""; + + assertOK(putComponentTemplate(client, "logs@custom", ignoreMalformedOverrideTemplate)); + assertOK(createDataStream(client, "logs-custom-dev")); + + var ignoreDynamicBeyondLimitIndexSetting = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + "index.mapping.total_fields.ignore_dynamic_beyond_limit" + ); + assertThat(ignoreDynamicBeyondLimitIndexSetting, equalTo("false")); + } + + public void testAddNonCompatibleMapping() throws IOException { + var nonCompatibleMappingAdditionTemplate = """ + { + "template": { + "mappings": { + "properties": { + "bomb": { + "type": "ip", + "doc_values": false + } + } + } + } + }"""; + + Exception e = assertThrows( + ResponseException.class, + () -> putComponentTemplate(client, "logs@custom", nonCompatibleMappingAdditionTemplate) + ); + assertThat( + e.getMessage(), + containsString("updating component template [logs@custom] results in invalid composable template [logs]") + ); + assertThat( + e.getMessage(), + containsString("field [bomb] of type [ip] doesn't support synthetic source because it doesn't have doc values") + ); + } + + private static Map getMapping(final RestClient client, final String indexName) throws IOException { + final Request request = new Request("GET", "/" + indexName + "/_mapping"); + + Map mappings = ((Map>) entityAsMap(client.performRequest(request)).get(indexName)).get( + "mappings" + ); + + return mappings; + } + + private Function> subObject(String key) { + return (mapAsObject) -> (Map) ((Map) mapAsObject).get(key); + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java index dcd2457b88f18..fada21224e3b2 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java @@ -50,7 +50,7 @@ public void setup() throws Exception { public void testLogsSettingsIndexModeDisabled() throws IOException { assertOK(createDataStream(client, "logs-custom-dev")); final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); - assertThat(indexMode, Matchers.not(equalTo(IndexMode.LOGS.getName()))); + assertThat(indexMode, Matchers.not(equalTo(IndexMode.LOGSDB.getName()))); } } diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java index 832267cebf97c..a4277748ea9bd 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java @@ -179,7 +179,7 @@ public void testCreateDataStream() throws IOException { assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); assertOK(createDataStream(client, "logs-custom-dev")); final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); - assertThat(indexMode, equalTo(IndexMode.LOGS.getName())); + assertThat(indexMode, equalTo(IndexMode.LOGSDB.getName())); } public void testBulkIndexing() throws IOException { diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/AbstractChallengeRestTest.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/AbstractChallengeRestTest.java new file mode 100644 index 0000000000000..8ee0e4d715c4c --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/AbstractChallengeRestTest.java @@ -0,0 +1,311 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.logsdb.qa; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.CheckedSupplier; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; + +import java.io.IOException; +import java.util.List; +import java.util.function.Supplier; + +public abstract class AbstractChallengeRestTest extends ESRestTestCase { + private final String baselineDataStreamName; + private final String contenderDataStreamName; + private final String baselineTemplateName; + private final String contenderTemplateName; + + private final int baselineTemplatePriority; + private final int contenderTemplatePriority; + private XContentBuilder baselineMappings; + private XContentBuilder contenderMappings; + private Settings.Builder baselineSettings; + private Settings.Builder contenderSettings; + private RestClient client; + + @ClassRule() + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .module("data-streams") + .module("x-pack-stack") + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .setting("cluster.logsdb.enabled", "true") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + public AbstractChallengeRestTest( + final String baselineDataStreamName, + final String contenderDataStreamName, + final String baselineTemplateName, + final String contenderTemplateName, + int baselineTemplatePriority, + int contenderTemplatePriority + ) { + this.baselineDataStreamName = baselineDataStreamName; + this.contenderDataStreamName = contenderDataStreamName; + this.baselineTemplateName = baselineTemplateName; + this.contenderTemplateName = contenderTemplateName; + this.baselineTemplatePriority = baselineTemplatePriority; + this.contenderTemplatePriority = contenderTemplatePriority; + } + + @Before + public void beforeTest() throws Exception { + beforeStart(); + client = client(); + this.baselineMappings = createBaselineMappings(); + this.contenderMappings = createContenderMappings(); + this.baselineSettings = createBaselineSettings(); + this.contenderSettings = createContenderSettings(); + createTemplates(); + createDataStreams(); + beforeEnd(); + } + + @After + public void afterTest() throws Exception { + afterStart(); + deleteDataStreams(); + deleteTemplates(); + afterEnd(); + } + + public void beforeStart() throws Exception {} + + public void beforeEnd() throws Exception {}; + + public void afterStart() throws Exception {} + + public void afterEnd() throws Exception {} + + private void createTemplates() throws IOException { + final Response createBaselineTemplateResponse = createTemplates( + getBaselineTemplateName(), + getBaselineDataStreamName() + "*", + baselineSettings, + baselineMappings, + getBaselineTemplatePriority() + ); + assert createBaselineTemplateResponse.getStatusLine().getStatusCode() == RestStatus.OK.getStatus(); + + final Response createContenderTemplateResponse = createTemplates( + getContenderTemplateName(), + getContenderDataStreamName() + "*", + contenderSettings, + contenderMappings, + getContenderTemplatePriority() + ); + assert createContenderTemplateResponse.getStatusLine().getStatusCode() == RestStatus.OK.getStatus(); + } + + private void createDataStreams() throws IOException { + final Response craeteBaselineDataStreamResponse = client.performRequest( + new Request("PUT", "_data_stream/" + getBaselineDataStreamName()) + ); + assert craeteBaselineDataStreamResponse.getStatusLine().getStatusCode() == RestStatus.OK.getStatus(); + + final Response createContenderDataStreamResponse = client.performRequest( + new Request("PUT", "_data_stream/" + getContenderDataStreamName()) + ); + assert createContenderDataStreamResponse.getStatusLine().getStatusCode() == RestStatus.OK.getStatus(); + } + + private Response createTemplates( + final String templateName, + final String pattern, + final Settings.Builder settings, + final XContentBuilder mappings, + int priority + ) throws IOException { + final String template = """ + { + "index_patterns": [ "%s" ], + "template": { + "settings":%s, + "mappings": %s + }, + "data_stream": {}, + "priority": %d + } + """; + final Request request = new Request("PUT", "/_index_template/" + templateName); + final String jsonSettings = settings.build().toString(); + final String jsonMappings = Strings.toString(mappings); + request.setJsonEntity(Strings.format(template, pattern, jsonSettings, jsonMappings, priority)); + return client.performRequest(request); + } + + private void deleteDataStreams() throws IOException { + final Response deleteBaselineDataStream = client.performRequest( + new Request("DELETE", "/_data_stream/" + getBaselineDataStreamName()) + ); + assert deleteBaselineDataStream.getStatusLine().getStatusCode() == RestStatus.OK.getStatus(); + + final Response deleteContenderDataStream = client.performRequest( + new Request("DELETE", "/_data_stream/" + getContenderDataStreamName()) + ); + assert deleteContenderDataStream.getStatusLine().getStatusCode() == RestStatus.OK.getStatus(); + } + + private void deleteTemplates() throws IOException { + final Response deleteBaselineTemplate = client.performRequest( + new Request("DELETE", "/_index_template/" + getBaselineTemplateName()) + ); + assert deleteBaselineTemplate.getStatusLine().getStatusCode() == RestStatus.OK.getStatus(); + + final Response deleteContenderTemplate = client.performRequest( + new Request("DELETE", "/_index_template/" + getContenderTemplateName()) + ); + assert deleteContenderTemplate.getStatusLine().getStatusCode() == RestStatus.OK.getStatus(); + } + + private Settings.Builder createSettings(final CheckedConsumer settingsConsumer) throws IOException { + final Settings.Builder settings = Settings.builder(); + settingsConsumer.accept(settings); + return settings; + } + + private Settings.Builder createBaselineSettings() throws IOException { + return createSettings(this::baselineSettings); + } + + private Settings.Builder createContenderSettings() throws IOException { + return createSettings(this::contenderSettings); + } + + private XContentBuilder createMappings(final CheckedConsumer builderConsumer) throws IOException { + final XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builderConsumer.accept(builder); + builder.endObject(); + return builder; + } + + private XContentBuilder createBaselineMappings() throws IOException { + return createMappings(this::baselineMappings); + } + + private XContentBuilder createContenderMappings() throws IOException { + return createMappings(this::contenderMappings); + } + + public abstract void baselineMappings(XContentBuilder builder) throws IOException; + + public abstract void contenderMappings(XContentBuilder builder) throws IOException; + + public void baselineSettings(Settings.Builder builder) {} + + public void contenderSettings(Settings.Builder builder) {} + + private Response indexDocuments( + final String dataStreamName, + final CheckedSupplier, IOException> documentsSupplier + ) throws IOException { + final StringBuilder sb = new StringBuilder(); + for (var document : documentsSupplier.get()) { + sb.append("{ \"create\": {} }").append("\n"); + sb.append(Strings.toString(document)).append("\n"); + } + var request = new Request("POST", "/" + dataStreamName + "/_bulk"); + request.setJsonEntity(sb.toString()); + request.addParameter("refresh", "true"); + return client.performRequest(request); + } + + public Response indexBaselineDocuments(final CheckedSupplier, IOException> documentsSupplier) throws IOException { + return indexDocuments(getBaselineDataStreamName(), documentsSupplier); + } + + public Response indexContenderDocuments(final CheckedSupplier, IOException> documentsSupplier) + throws IOException { + return indexDocuments(getContenderDataStreamName(), documentsSupplier); + } + + public Tuple indexDocuments( + final CheckedSupplier, IOException> baselineSupplier, + final CheckedSupplier, IOException> contenderSupplier + ) throws IOException { + return new Tuple<>(indexBaselineDocuments(baselineSupplier), indexContenderDocuments(contenderSupplier)); + } + + public Response queryBaseline(final SearchSourceBuilder search) throws IOException { + return query(search, this::getBaselineDataStreamName); + } + + public Response queryContender(final SearchSourceBuilder search) throws IOException { + return query(search, this::getContenderDataStreamName); + } + + private Response query(final SearchSourceBuilder search, final Supplier dataStreamNameSupplier) throws IOException { + final Request request = new Request("GET", "/" + dataStreamNameSupplier.get() + "/_search"); + request.setJsonEntity(Strings.toString(search)); + return client.performRequest(request); + } + + public String getBaselineDataStreamName() { + return baselineDataStreamName; + } + + public int getBaselineTemplatePriority() { + return baselineTemplatePriority; + } + + public int getContenderTemplatePriority() { + return contenderTemplatePriority; + } + + public String getContenderDataStreamName() { + return contenderDataStreamName; + } + + public String getBaselineTemplateName() { + return baselineTemplateName; + } + + public String getContenderTemplateName() { + return contenderTemplateName; + } + + public XContentBuilder getBaselineMappings() { + return baselineMappings; + } + + public XContentBuilder getContenderMappings() { + return contenderMappings; + } + + public Settings.Builder getBaselineSettings() { + return baselineSettings; + } + + public Settings.Builder getContenderSettings() { + return contenderSettings; + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java new file mode 100644 index 0000000000000..63db21e45ae9f --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java @@ -0,0 +1,297 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.logsdb.qa; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.FormatNames; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.datastreams.logsdb.qa.matchers.MatchResult; +import org.elasticsearch.datastreams.logsdb.qa.matchers.Matcher; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.hamcrest.Matchers; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class StandardVersusLogsIndexModeChallengeRestIT extends AbstractChallengeRestTest { + + public StandardVersusLogsIndexModeChallengeRestIT() { + super("standard-apache-baseline", "logs-apache-contender", "baseline-template", "contender-template", 101, 101); + } + + @Override + public void baselineMappings(XContentBuilder builder) throws IOException { + if (randomBoolean()) { + builder.startObject("properties") + + .startObject("@timestamp") + .field("type", "date") + .endObject() + + .startObject("host.name") + .field("type", "keyword") + .field("ignore_above", randomIntBetween(1000, 1200)) + .endObject() + + .startObject("message") + .field("type", "keyword") + .field("ignore_above", randomIntBetween(1000, 1200)) + .endObject() + + .startObject("method") + .field("type", "keyword") + .field("ignore_above", randomIntBetween(1000, 1200)) + .endObject() + + .startObject("memory_usage_bytes") + .field("type", "long") + .field("ignore_malformed", randomBoolean()) + .endObject() + + .endObject(); + } else { + builder.startObject("properties") + + .startObject("host.name") + .field("type", "keyword") + .field("ignore_above", randomIntBetween(1000, 1200)) + .endObject() + + .endObject(); + } + } + + @Override + public void contenderMappings(XContentBuilder builder) throws IOException { + builder.field("subobjects", false); + if (randomBoolean()) { + builder.startObject("properties") + + .startObject("@timestamp") + .field("type", "date") + .endObject() + + .startObject("host.name") + .field("type", "keyword") + .field("ignore_above", randomIntBetween(1000, 1200)) + .endObject() + + .startObject("message") + .field("type", "keyword") + .field("ignore_above", randomIntBetween(1000, 1200)) + .endObject() + + .startObject("method") + .field("type", "keyword") + .field("ignore_above", randomIntBetween(1000, 1200)) + .endObject() + + .startObject("memory_usage_bytes") + .field("type", "long") + .field("ignore_malformed", randomBoolean()) + .endObject() + + .endObject(); + } + } + + private static void settings(final Settings.Builder settings) { + if (randomBoolean()) { + settings.put("index.number_of_shards", randomIntBetween(2, 5)); + } + if (randomBoolean()) { + settings.put("index.number_of_replicas", randomIntBetween(1, 3)); + } + } + + @Override + public void contenderSettings(Settings.Builder builder) { + builder.put("index.mode", "logsdb"); + settings(builder); + } + + @Override + public void baselineSettings(Settings.Builder builder) { + settings(builder); + } + + @Override + public void beforeStart() throws Exception { + waitForLogs(client()); + } + + protected static void waitForLogs(RestClient client) throws Exception { + assertBusy(() -> { + try { + final Request request = new Request("GET", "_index_template/logs"); + assertOK(client.performRequest(request)); + } catch (ResponseException e) { + fail(e.getMessage()); + } + }); + } + + @SuppressWarnings("unchecked") + public void testMatchAllQuery() throws IOException { + final List documents = new ArrayList<>(); + int numberOfDocuments = ESTestCase.randomIntBetween(100, 200); + for (int i = 0; i < numberOfDocuments; i++) { + documents.add(generateDocument(Instant.now().plus(i, ChronoUnit.SECONDS))); + } + + assertDocumentIndexing(documents); + + final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()) + .size(numberOfDocuments); + + final MatchResult matchResult = Matcher.mappings(getContenderMappings(), getBaselineMappings()) + .settings(getContenderSettings(), getBaselineSettings()) + .expected(getQueryHits(queryBaseline(searchSourceBuilder))) + .ignoringSort(true) + .isEqualTo(getQueryHits(queryContender(searchSourceBuilder))); + assertTrue(matchResult.getMessage(), matchResult.isMatch()); + } + + public void testTermsQuery() throws IOException { + final List documents = new ArrayList<>(); + int numberOfDocuments = randomIntBetween(100, 200); + for (int i = 0; i < numberOfDocuments; i++) { + documents.add(generateDocument(Instant.now().plus(i, ChronoUnit.SECONDS))); + } + + assertDocumentIndexing(documents); + + final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.termQuery("method", "put")) + .size(numberOfDocuments); + + final MatchResult matchResult = Matcher.mappings(getContenderMappings(), getBaselineMappings()) + .settings(getContenderSettings(), getBaselineSettings()) + .expected(getQueryHits(queryBaseline(searchSourceBuilder))) + .ignoringSort(true) + .isEqualTo(getQueryHits(queryContender(searchSourceBuilder))); + assertTrue(matchResult.getMessage(), matchResult.isMatch()); + } + + public void testHistogramAggregation() throws IOException { + final List documents = new ArrayList<>(); + int numberOfDocuments = randomIntBetween(100, 200); + for (int i = 0; i < numberOfDocuments; i++) { + documents.add(generateDocument(Instant.now().plus(i, ChronoUnit.SECONDS))); + } + + assertDocumentIndexing(documents); + + final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()) + .size(numberOfDocuments) + .aggregation(new HistogramAggregationBuilder("agg").field("memory_usage_bytes").interval(100.0D)); + + final MatchResult matchResult = Matcher.mappings(getContenderMappings(), getBaselineMappings()) + .settings(getContenderSettings(), getBaselineSettings()) + .expected(getAggregationBuckets(queryBaseline(searchSourceBuilder), "agg")) + .ignoringSort(true) + .isEqualTo(getAggregationBuckets(queryContender(searchSourceBuilder), "agg")); + assertTrue(matchResult.getMessage(), matchResult.isMatch()); + } + + public void testTermsAggregation() throws IOException { + final List documents = new ArrayList<>(); + int numberOfDocuments = randomIntBetween(100, 200); + for (int i = 0; i < numberOfDocuments; i++) { + documents.add(generateDocument(Instant.now().plus(i, ChronoUnit.SECONDS))); + } + + assertDocumentIndexing(documents); + + final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()) + .size(0) + .aggregation(new TermsAggregationBuilder("agg").field("host.name")); + + final MatchResult matchResult = Matcher.mappings(getContenderMappings(), getBaselineMappings()) + .settings(getContenderSettings(), getBaselineSettings()) + .expected(getAggregationBuckets(queryBaseline(searchSourceBuilder), "agg")) + .ignoringSort(true) + .isEqualTo(getAggregationBuckets(queryContender(searchSourceBuilder), "agg")); + assertTrue(matchResult.getMessage(), matchResult.isMatch()); + } + + public void testDateHistogramAggregation() throws IOException { + final List documents = new ArrayList<>(); + int numberOfDocuments = randomIntBetween(100, 200); + for (int i = 0; i < numberOfDocuments; i++) { + documents.add(generateDocument(Instant.now().plus(i, ChronoUnit.SECONDS))); + } + + assertDocumentIndexing(documents); + + final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()) + .aggregation(AggregationBuilders.dateHistogram("agg").field("@timestamp").calendarInterval(DateHistogramInterval.SECOND)) + .size(0); + + final MatchResult matchResult = Matcher.mappings(getContenderMappings(), getBaselineMappings()) + .settings(getContenderSettings(), getBaselineSettings()) + .expected(getAggregationBuckets(queryBaseline(searchSourceBuilder), "agg")) + .ignoringSort(true) + .isEqualTo(getAggregationBuckets(queryContender(searchSourceBuilder), "agg")); + assertTrue(matchResult.getMessage(), matchResult.isMatch()); + } + + private static XContentBuilder generateDocument(final Instant timestamp) throws IOException { + return XContentFactory.jsonBuilder() + .startObject() + .field("@timestamp", DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(timestamp)) + .field("host.name", randomFrom("foo", "bar", "baz")) + .field("message", randomFrom("a message", "another message", "still another message", "one more message")) + .field("method", randomFrom("put", "post", "get")) + .field("memory_usage_bytes", randomLongBetween(1000, 2000)) + .endObject(); + } + + @SuppressWarnings("unchecked") + private static List> getQueryHits(final Response response) throws IOException { + final Map map = XContentHelper.convertToMap(XContentType.JSON.xContent(), response.getEntity().getContent(), true); + final Map hitsMap = (Map) map.get("hits"); + final List> hitsList = (List>) hitsMap.get("hits"); + return hitsList.stream().map(hit -> (Map) hit.get("_source")).toList(); + } + + @SuppressWarnings("unchecked") + private static List> getAggregationBuckets(final Response response, final String aggName) throws IOException { + final Map map = XContentHelper.convertToMap(XContentType.JSON.xContent(), response.getEntity().getContent(), true); + final Map aggs = (Map) map.get("aggregations"); + final Map agg = (Map) aggs.get(aggName); + return (List>) agg.get("buckets"); + } + + private void assertDocumentIndexing(List documents) throws IOException { + final Tuple tuple = indexDocuments(() -> documents, () -> documents); + assertThat(tuple.v1().getStatusLine().getStatusCode(), Matchers.equalTo(RestStatus.OK.getStatus())); + assertThat(tuple.v2().getStatusLine().getStatusCode(), Matchers.equalTo(RestStatus.OK.getStatus())); + } + +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ArrayEqualMatcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ArrayEqualMatcher.java new file mode 100644 index 0000000000000..25e6dc8ef31c9 --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ArrayEqualMatcher.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.logsdb.qa.matchers; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.util.Arrays; +import java.util.List; + +class ArrayEqualMatcher extends EqualMatcher { + ArrayEqualMatcher( + final XContentBuilder actualMappings, + final Settings.Builder actualSettings, + final XContentBuilder expectedMappings, + final Settings.Builder expectedSettings, + final Object[] actual, + final Object[] expected, + boolean ignoringSort + ) { + super(actualMappings, actualSettings, expectedMappings, expectedSettings, actual, expected, ignoringSort); + } + + @Override + public MatchResult match() { + return matchArraysEqual(actual, expected, ignoringSort); + } + + private MatchResult matchArraysEqual(final Object[] actualArray, final Object[] expectedArray, boolean ignoreSorting) { + if (actualArray.length != expectedArray.length) { + return MatchResult.noMatch( + formatErrorMessage(actualMappings, actualSettings, expectedMappings, expectedSettings, "Array lengths do no match") + ); + } + if (ignoreSorting) { + return matchArraysEqualIgnoringSorting(actualArray, expectedArray) + ? MatchResult.match() + : MatchResult.noMatch( + formatErrorMessage( + actualMappings, + actualSettings, + expectedMappings, + expectedSettings, + "Arrays do not match when ignoreing sort order" + ) + ); + } else { + return matchArraysEqualExact(actualArray, expectedArray) + ? MatchResult.match() + : MatchResult.noMatch( + formatErrorMessage(actualMappings, actualSettings, expectedMappings, expectedSettings, "Arrays do not match exactly") + ); + } + } + + private static boolean matchArraysEqualIgnoringSorting(final Object[] actualArray, final Object[] expectedArray) { + final List actualList = Arrays.asList(actualArray); + final List expectedList = Arrays.asList(expectedArray); + return actualList.containsAll(expectedList) && expectedList.containsAll(actualList); + } + + private static boolean matchArraysEqualExact(T[] actualArray, T[] expectedArray) { + for (int i = 0; i < actualArray.length; i++) { + boolean isEqual = actualArray[i].equals(expectedArray[i]); + if (isEqual == false) { + return false; + } + } + return true; + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/EqualMatcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/EqualMatcher.java new file mode 100644 index 0000000000000..b6c5704e0677d --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/EqualMatcher.java @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.logsdb.qa.matchers; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.util.List; + +class EqualMatcher extends Matcher { + protected final XContentBuilder actualMappings; + protected final Settings.Builder actualSettings; + protected final XContentBuilder expectedMappings; + protected final Settings.Builder expectedSettings; + protected final T actual; + protected final T expected; + protected final boolean ignoringSort; + + EqualMatcher( + XContentBuilder actualMappings, + Settings.Builder actualSettings, + XContentBuilder expectedMappings, + Settings.Builder expectedSettings, + T actual, + T expected, + boolean ignoringSort + ) { + this.actualMappings = actualMappings; + this.actualSettings = actualSettings; + this.expectedMappings = expectedMappings; + this.expectedSettings = expectedSettings; + this.actual = actual; + this.expected = expected; + this.ignoringSort = ignoringSort; + } + + public MatchResult match() { + if (actual == null) { + if (expected == null) { + + return MatchResult.noMatch( + formatErrorMessage( + actualMappings, + actualSettings, + expectedMappings, + expectedSettings, + "Both 'actual' and 'expected' are null" + ) + ); + } + return MatchResult.noMatch( + formatErrorMessage(actualMappings, actualSettings, expectedMappings, expectedSettings, "Expected is null but actual is not") + ); + } + if (expected == null) { + return MatchResult.noMatch( + formatErrorMessage(actualMappings, actualSettings, expectedMappings, expectedSettings, "Actual is null but expected is not") + ); + } + if (actual.getClass().equals(expected.getClass()) == false) { + return MatchResult.noMatch( + formatErrorMessage( + actualMappings, + actualSettings, + expectedMappings, + expectedSettings, + "Unable to match " + actual.getClass().getSimpleName() + " to " + expected.getClass().getSimpleName() + ) + ); + } + if (actual.getClass().isArray()) { + return new ArrayEqualMatcher( + actualMappings, + actualSettings, + expectedMappings, + expectedSettings, + (Object[]) actual, + (Object[]) expected, + ignoringSort + ).match(); + } + if (actual instanceof List act && expected instanceof List exp) { + return new ListEqualMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings, act, exp, ignoringSort).match(); + } + return new ObjectMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings, actual, expected).match(); + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ListEqualMatcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ListEqualMatcher.java new file mode 100644 index 0000000000000..56c24712f635c --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ListEqualMatcher.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.logsdb.qa.matchers; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.util.List; + +class ListEqualMatcher extends EqualMatcher> { + ListEqualMatcher( + final XContentBuilder actualMappings, + final Settings.Builder actualSettings, + final XContentBuilder expectedMappings, + final Settings.Builder expectedSettings, + final List actual, + final List expected, + final boolean ignoringSort + ) { + super(actualMappings, actualSettings, expectedMappings, expectedSettings, actual, expected, ignoringSort); + } + + @Override + @SuppressWarnings("unchecked") + public MatchResult match() { + return matchListEquals((List) actual, (List) expected, ignoringSort); + } + + private MatchResult matchListEquals(final List actualList, final List expectedList, boolean ignoreSorting) { + if (actualList.size() != expectedList.size()) { + return MatchResult.noMatch( + formatErrorMessage(actualMappings, actualSettings, expectedMappings, expectedSettings, "List lengths do no match") + ); + } + if (ignoreSorting) { + return matchListsEqualIgnoringSorting(actualList, expectedList) + ? MatchResult.match() + : MatchResult.noMatch( + formatErrorMessage( + actualMappings, + actualSettings, + expectedMappings, + expectedSettings, + "Lists do not match when ignoreing sort order" + ) + ); + } else { + return matchListsEqualExact(actualList, expectedList) + ? MatchResult.match() + : MatchResult.noMatch( + formatErrorMessage(actualMappings, actualSettings, expectedMappings, expectedSettings, "Lists do not match exactly") + ); + } + } + + private static boolean matchListsEqualIgnoringSorting(final List actualList, final List expectedList) { + return actualList.containsAll(expectedList) && expectedList.containsAll(actualList); + } + + private static boolean matchListsEqualExact(List actualList, List expectedList) { + for (int i = 0; i < actualList.size(); i++) { + boolean isEqual = actualList.get(i).equals(expectedList.get(i)); + if (isEqual == false) { + return false; + } + } + return true; + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/MatchResult.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/MatchResult.java new file mode 100644 index 0000000000000..07a57dcca3b71 --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/MatchResult.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.logsdb.qa.matchers; + +import java.util.Objects; + +public class MatchResult { + private final boolean isMatch; + private final String message; + + private MatchResult(boolean isMatch, String message) { + this.isMatch = isMatch; + this.message = message; + } + + public static MatchResult match() { + return new MatchResult(true, "Match successful"); + } + + public static MatchResult noMatch(final String reason) { + return new MatchResult(false, reason); + } + + public boolean isMatch() { + return isMatch; + } + + public String getMessage() { + return message; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MatchResult that = (MatchResult) o; + return isMatch == that.isMatch && Objects.equals(message, that.message); + } + + @Override + public int hashCode() { + return Objects.hash(isMatch, message); + } + + @Override + public String toString() { + return "MatchResult{" + "isMatch=" + isMatch + ", message='" + message + '\'' + '}'; + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/Matcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/Matcher.java new file mode 100644 index 0000000000000..7faa6cbff5456 --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/Matcher.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.logsdb.qa.matchers; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xcontent.XContentBuilder; + +/** + * A base class to be used for the matching logic when comparing query results. + */ +public abstract class Matcher { + + public static SettingsStep mappings(final XContentBuilder actualMappings, final XContentBuilder expectedMappings) { + return new Builder<>(expectedMappings, actualMappings); + } + + public interface SettingsStep { + ExpectedStep settings(Settings.Builder actualSettings, Settings.Builder expectedSettings); + } + + public interface ExpectedStep { + CompareStep expected(T expected); + } + + public interface CompareStep { + MatchResult isEqualTo(T actual); + + CompareStep ignoringSort(boolean ignoringSort); + } + + private static class Builder implements SettingsStep, CompareStep, ExpectedStep { + + private final XContentBuilder expectedMappings; + private final XContentBuilder actualMappings; + private Settings.Builder expectedSettings; + private Settings.Builder actualSettings; + private T expected; + private T actual; + private boolean ignoringSort; + + @Override + public ExpectedStep settings(Settings.Builder actualSettings, Settings.Builder expectedSettings) { + this.actualSettings = actualSettings; + this.expectedSettings = expectedSettings; + return this; + } + + private Builder( + final XContentBuilder actualMappings, + final XContentBuilder expectedMappings + + ) { + this.actualMappings = actualMappings; + this.expectedMappings = expectedMappings; + } + + @Override + public MatchResult isEqualTo(T actual) { + return new EqualMatcher<>(actualMappings, actualSettings, expectedMappings, expectedSettings, actual, expected, ignoringSort) + .match(); + } + + @Override + public CompareStep ignoringSort(boolean ignoringSort) { + this.ignoringSort = ignoringSort; + return this; + } + + @Override + public CompareStep expected(T expected) { + this.expected = expected; + return this; + } + } + + protected static String formatErrorMessage( + final XContentBuilder actualMappings, + final Settings.Builder actualSettings, + final XContentBuilder expectedMappings, + final Settings.Builder expectedSettings, + final String errorMessage + ) { + return "Error [" + + errorMessage + + "] " + + "actual mappings [" + + Strings.toString(actualMappings) + + "] " + + "actual settings [" + + Strings.toString(actualSettings.build()) + + "] " + + "expected mappings [" + + Strings.toString(expectedMappings) + + "] " + + "expected settings [" + + Strings.toString(expectedSettings.build()) + + "] "; + } + +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ObjectMatcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ObjectMatcher.java new file mode 100644 index 0000000000000..ebc50e8a7e89c --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ObjectMatcher.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.logsdb.qa.matchers; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xcontent.XContentBuilder; + +public class ObjectMatcher extends EqualMatcher { + ObjectMatcher( + final XContentBuilder actualMappings, + final Settings.Builder actualSettings, + final XContentBuilder expectedMappings, + final Settings.Builder expectedSettings, + final Object actual, + final Object expected + ) { + super(actualMappings, actualSettings, expectedMappings, expectedSettings, actual, expected, true); + } + + @Override + public MatchResult match() { + return actual.equals(expected) + ? MatchResult.match() + : MatchResult.noMatch( + formatErrorMessage(actualMappings, actualSettings, expectedMappings, expectedSettings, "Actual does not equal expected") + ); + } +} diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml index 04c70ee380d4f..54ce32eb13207 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml @@ -6,13 +6,13 @@ teardown: ignore: 404 - do: - indices.delete: - index: .fs-logs-foobar-* + indices.delete_index_template: + name: generic_logs_template ignore: 404 - do: - indices.delete_index_template: - name: generic_logs_template + indices.delete_data_stream: + name: destination-data-stream ignore: 404 - do: @@ -25,6 +25,11 @@ teardown: id: "failing_pipeline" ignore: 404 + - do: + ingest.delete_pipeline: + id: "reroute_pipeline" + ignore: 404 + --- "Redirect ingest failure in data stream to failure store": - requires: @@ -215,3 +220,103 @@ teardown: indices.delete: index: .fs-logs-foobar-* - is_true: acknowledged + +--- +"Ensure failure is redirected to correct failure store after a reroute processor": + - skip: + known_issues: + - cluster_feature: "gte_v8.15.0" + fixed_by: "gte_v8.16.0" + reason: "Failure store documents contained the original index name rather than the rerouted one before v8.16.0" + - requires: + test_runner_features: [allowed_warnings] + + - do: + ingest.put_pipeline: + id: "failing_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "fail": { + "message" : "error_message", + "tag": "foo-tag" + } + } + ] + } + - match: { acknowledged: true } + + - do: + allowed_warnings: + - "index template [destination_template] has index patterns [destination-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [destination_template] will take precedence during new index creation" + indices.put_index_template: + name: destination_template + body: + index_patterns: destination-data-stream + data_stream: + failure_store: true + template: + settings: + number_of_shards: 1 + number_of_replicas: 1 + index: + default_pipeline: "failing_pipeline" + + - do: + indices.create_data_stream: + name: destination-data-stream + + - do: + ingest.put_pipeline: + id: "reroute_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "reroute": { + "tag": "reroute-tag", + "destination": "destination-data-stream" + } + } + ] + } + - match: { acknowledged: true } + + - do: + allowed_warnings: + - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation" + indices.put_index_template: + name: generic_logs_template + body: + index_patterns: logs-* + data_stream: + failure_store: true + template: + settings: + number_of_shards: 1 + number_of_replicas: 1 + index: + default_pipeline: "reroute_pipeline" + + - do: + index: + index: logs-foobar + refresh: true + body: + '@timestamp': '2020-12-12' + foo: bar + + - do: + search: + index: .fs-logs-foobar-* + - length: { hits.hits: 0 } + + - do: + search: + index: .fs-destination-* + - length: { hits.hits: 1 } + - match: { hits.hits.0._index: "/\\.fs-destination-data-stream-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } + - match: { hits.hits.0._source.document.index: 'destination-data-stream' } diff --git a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverWithPipelinesIT.java b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverWithPipelinesIT.java index 7bb875f8b6f69..16a8013ae9c4a 100644 --- a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverWithPipelinesIT.java +++ b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverWithPipelinesIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.plugins.internal; +import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.common.bytes.BytesArray; @@ -89,13 +90,12 @@ public DocumentParsingProvider getDocumentParsingProvider() { // returns a static instance, because we want to assert that the wrapping is called only once return new DocumentParsingProvider() { @Override - public DocumentSizeObserver newFixedSizeDocumentObserver(long normalisedBytesParsed) { - providedFixedSize.set(normalisedBytesParsed); - return new TestDocumentSizeObserver(normalisedBytesParsed); - } - - @Override - public DocumentSizeObserver newDocumentSizeObserver() { + public DocumentSizeObserver newDocumentSizeObserver(DocWriteRequest request) { + if (request instanceof IndexRequest indexRequest && indexRequest.getNormalisedBytesParsed() > 0) { + long normalisedBytesParsed = indexRequest.getNormalisedBytesParsed(); + providedFixedSize.set(normalisedBytesParsed); + return new TestDocumentSizeObserver(normalisedBytesParsed); + } return new TestDocumentSizeObserver(0L); } @@ -137,6 +137,7 @@ public Map map() throws IOException { public long normalisedBytesParsed() { return mapCounter; } + } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java index 23bbaf62cec07..ae3de1f1c446f 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java @@ -30,6 +30,7 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentMap; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; @@ -45,7 +46,7 @@ public final class DateProcessor extends AbstractProcessor { private final String field; private final String targetField; private final List formats; - private final List, Function>> dateParsers; + private final List>> dateParsers; private final String outputFormat; DateProcessor( @@ -80,14 +81,12 @@ public final class DateProcessor extends AbstractProcessor { for (String format : formats) { DateFormat dateFormat = DateFormat.fromString(format); - dateParsers.add((params) -> { - var documentTimezone = timezone == null ? null : timezone.newInstance(params).execute(); - var documentLocale = locale == null ? null : locale.newInstance(params).execute(); - return Cache.INSTANCE.getOrCompute( + dateParsers.add( + (documentTimezone, documentLocale) -> Cache.INSTANCE.getOrCompute( new Cache.Key(format, documentTimezone, documentLocale), - () -> dateFormat.getFunction(format, newDateTimeZone(documentTimezone), newLocale(documentLocale)) - ); - }); + () -> dateFormat.getFunction(format, documentTimezone, documentLocale) + ) + ); } this.outputFormat = outputFormat; formatter = DateFormatter.forPattern(this.outputFormat); @@ -106,15 +105,27 @@ public IngestDocument execute(IngestDocument ingestDocument) { Object obj = ingestDocument.getFieldValue(field, Object.class); String value = null; if (obj != null) { - // Not use Objects.toString(...) here, because null gets changed to "null" which may confuse some date parsers + // Don't use Objects.toString(...) here, because null gets changed to "null" which may confuse some date parsers value = obj.toString(); } + // run (potential) mustache application just a single time for this document in order to + // extract the timezone and locale to use for date parsing + final ZoneId documentTimezone; + final Locale documentLocale; + final Map sourceAndMetadata = ingestDocument.getSourceAndMetadata(); + try { + documentTimezone = newDateTimeZone(timezone == null ? null : timezone.newInstance(sourceAndMetadata).execute()); + documentLocale = newLocale(locale == null ? null : locale.newInstance(sourceAndMetadata).execute()); + } catch (Exception e) { + throw new IllegalArgumentException("unable to parse date [" + value + "]", e); + } + ZonedDateTime dateTime = null; Exception lastException = null; - for (Function, Function> dateParser : dateParsers) { + for (BiFunction> dateParser : dateParsers) { try { - dateTime = dateParser.apply(ingestDocument.getSourceAndMetadata()).apply(value); + dateTime = dateParser.apply(documentTimezone, documentLocale).apply(value); break; } catch (Exception e) { // try the next parser and keep track of the exceptions @@ -255,6 +266,6 @@ Function getOrCompute(Key key, Supplier zonedDateTimeFunction1 = str -> ZonedDateTime.now(); Function zonedDateTimeFunction2 = str -> ZonedDateTime.now(); var cache = new DateProcessor.Cache(1); - var key1 = new DateProcessor.Cache.Key("format-1", ZoneId.systemDefault().toString(), Locale.ROOT.toString()); - var key2 = new DateProcessor.Cache.Key("format-2", ZoneId.systemDefault().toString(), Locale.ROOT.toString()); + var key1 = new DateProcessor.Cache.Key("format-1", ZoneId.systemDefault(), Locale.ROOT); + var key2 = new DateProcessor.Cache.Key("format-2", ZoneId.systemDefault(), Locale.ROOT); when(supplier1.get()).thenReturn(zonedDateTimeFunction1); when(supplier2.get()).thenReturn(zonedDateTimeFunction2); @@ -391,4 +393,36 @@ public void testCacheIsEvictedAfterReachMaxCapacity() { verify(supplier1, times(3)).get(); verify(supplier2, times(2)).get(); } + + public void testMustacheTemplateExecutesAtMostTwiceWithMultipleFormats() { + final TemplateScript.Factory factory = mock(TemplateScript.Factory.class); + final TemplateScript compiledScript = mock(TemplateScript.class); + when(factory.newInstance(any())).thenReturn(compiledScript); + when(compiledScript.execute()).thenReturn(null); + + final List matchFormats = List.of( + "dd/MM/yyyy", + "dd-MM-yyyy", + "uuuu-dd-MM", + "uuuu-MM-dd", + "TAI64N", + "epoch_millis", + "yyyy dd MM" + ); + DateProcessor dateProcessor = new DateProcessor( + randomAlphaOfLength(10), + null, + factory, + factory, + "date_as_string", + matchFormats, + "date_as_date" + ); + + Map document = new HashMap<>(); + document.put("date_as_string", "2010 12 06"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + dateProcessor.execute(ingestDocument); + verify(compiledScript, atMost(2)).execute(); + } } diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java new file mode 100644 index 0000000000000..cc757c413713d --- /dev/null +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java @@ -0,0 +1,196 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import fixture.geoip.EnterpriseGeoIpHttpFixture; + +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.ingest.PutPipelineRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.ingest.EnterpriseGeoIpTask; +import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration; +import org.elasticsearch.ingest.geoip.direct.PutDatabaseConfigurationAction; +import org.elasticsearch.persistent.PersistentTasksService; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.reindex.ReindexPlugin; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.transport.RemoteTransportException; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.ClassRule; + +import java.io.IOException; +import java.util.Collection; +import java.util.Map; + +import static org.elasticsearch.ingest.EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER; +import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderTaskExecutor.MAXMIND_LICENSE_KEY_SETTING; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; + +public class EnterpriseGeoIpDownloaderIT extends ESIntegTestCase { + + private static final String DATABASE_TYPE = "GeoIP2-City"; + + @ClassRule + public static final EnterpriseGeoIpHttpFixture fixture = new EnterpriseGeoIpHttpFixture(DATABASE_TYPE); + + protected String getEndpoint() { + return fixture.getAddress(); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString(MAXMIND_LICENSE_KEY_SETTING.getKey(), "license_key"); + Settings.Builder builder = Settings.builder(); + builder.setSecureSettings(secureSettings) + .put(super.nodeSettings(nodeOrdinal, otherSettings)) + .put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), true); + // note: this is using the enterprise fixture for the regular downloader, too, as + // a slightly hacky way of making the regular downloader not actually download any files + builder.put(GeoIpDownloader.ENDPOINT_SETTING.getKey(), getEndpoint()); + return builder.build(); + } + + @SuppressWarnings("unchecked") + protected Collection> nodePlugins() { + // the reindex plugin is (somewhat surprisingly) necessary in order to be able to delete-by-query, + // which modules/ingest-geoip does to delete old chunks + return CollectionUtils.appendToCopyNoNullElements(super.nodePlugins(), IngestGeoIpPlugin.class, ReindexPlugin.class); + } + + @SuppressWarnings("unchecked") + public void testEnterpriseDownloaderTask() throws Exception { + /* + * This test starts the enterprise geoip downloader task, and creates a database configuration. Then it creates an ingest + * pipeline that references that database, and ingests a single document using that pipeline. It then asserts that the document + * was updated with information from the database. + * Note that the "enterprise database" is actually just a geolite database being loaded by the GeoIpHttpFixture. + */ + EnterpriseGeoIpDownloader.DEFAULT_MAXMIND_ENDPOINT = getEndpoint(); + final String pipelineName = "enterprise_geoip_pipeline"; + final String indexName = "enterprise_geoip_test_index"; + final String sourceField = "ip"; + final String targetField = "ip-city"; + + startEnterpriseGeoIpDownloaderTask(); + configureDatabase(DATABASE_TYPE); + createGeoIpPipeline(pipelineName, DATABASE_TYPE, sourceField, targetField); + + assertBusy(() -> { + /* + * We know that the .geoip_databases index has been populated, but we don't know for sure that the database has been pulled + * down and made available on all nodes. So we run this ingest-and-check step in an assertBusy. + */ + logger.info("Ingesting a test document"); + String documentId = ingestDocument(indexName, pipelineName, sourceField); + GetResponse getResponse = client().get(new GetRequest(indexName, documentId)).actionGet(); + Map returnedSource = getResponse.getSource(); + assertNotNull(returnedSource); + Object targetFieldValue = returnedSource.get(targetField); + assertNotNull(targetFieldValue); + assertThat(((Map) targetFieldValue).get("organization_name"), equalTo("Bredband2 AB")); + }); + } + + private void startEnterpriseGeoIpDownloaderTask() { + PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); + persistentTasksService.sendStartRequest( + ENTERPRISE_GEOIP_DOWNLOADER, + ENTERPRISE_GEOIP_DOWNLOADER, + new EnterpriseGeoIpTask.EnterpriseGeoIpTaskParams(), + TimeValue.MAX_VALUE, + ActionListener.wrap(r -> logger.debug("Started enterprise geoip downloader task"), e -> { + Throwable t = e instanceof RemoteTransportException ? ExceptionsHelper.unwrapCause(e) : e; + if (t instanceof ResourceAlreadyExistsException == false) { + logger.error("failed to create enterprise geoip downloader task", e); + } + }) + ); + } + + private void configureDatabase(String databaseType) throws Exception { + admin().cluster() + .execute( + PutDatabaseConfigurationAction.INSTANCE, + new PutDatabaseConfigurationAction.Request( + TimeValue.MAX_VALUE, + TimeValue.MAX_VALUE, + new DatabaseConfiguration("test", databaseType, new DatabaseConfiguration.Maxmind("test_account")) + ) + ) + .actionGet(); + ensureGreen(GeoIpDownloader.DATABASES_INDEX); + assertBusy(() -> { + SearchResponse searchResponse = client().search(new SearchRequest(GeoIpDownloader.DATABASES_INDEX)).actionGet(); + try { + assertThat(searchResponse.getHits().getHits().length, equalTo(1)); + } finally { + searchResponse.decRef(); + } + }); + } + + private void createGeoIpPipeline(String pipelineName, String databaseType, String sourceField, String targetField) throws IOException { + final BytesReference bytes; + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + builder.startObject(); + { + builder.field("description", "test"); + builder.startArray("processors"); + { + builder.startObject(); + { + builder.startObject("geoip"); + { + builder.field("field", sourceField); + builder.field("target_field", targetField); + builder.field("database_file", databaseType + ".mmdb"); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endArray(); + } + builder.endObject(); + bytes = BytesReference.bytes(builder); + } + assertAcked(clusterAdmin().putPipeline(new PutPipelineRequest(pipelineName, bytes, XContentType.JSON)).actionGet()); + } + + private String ingestDocument(String indexName, String pipelineName, String sourceField) { + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add( + new IndexRequest(indexName).source("{\"" + sourceField + "\": \"89.160.20.128\"}", XContentType.JSON).setPipeline(pipelineName) + ); + BulkResponse response = client().bulk(bulkRequest).actionGet(); + BulkItemResponse[] bulkItemResponses = response.getItems(); + assertThat(bulkItemResponses.length, equalTo(1)); + assertThat(bulkItemResponses[0].status(), equalTo(RestStatus.CREATED)); + return bulkItemResponses[0].getId(); + } +} diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java index 9eab00fbadf20..f7ab384c69bf1 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java @@ -152,9 +152,9 @@ public void testInvalidTimestamp() throws Exception { updateClusterSettings(Settings.builder().put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), true)); assertBusy(() -> { GeoIpTaskState state = getGeoIpTaskState(); - assertEquals( - Set.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "MyCustomGeoLite2-City.mmdb"), - state.getDatabases().keySet() + assertThat( + state.getDatabases().keySet(), + containsInAnyOrder("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "MyCustomGeoLite2-City.mmdb") ); }, 2, TimeUnit.MINUTES); @@ -227,9 +227,9 @@ public void testGeoIpDatabasesDownload() throws Exception { updateClusterSettings(Settings.builder().put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), true)); assertBusy(() -> { GeoIpTaskState state = getGeoIpTaskState(); - assertEquals( - Set.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "MyCustomGeoLite2-City.mmdb"), - state.getDatabases().keySet() + assertThat( + state.getDatabases().keySet(), + containsInAnyOrder("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "MyCustomGeoLite2-City.mmdb") ); putGeoIpPipeline(); // This is to work around the race condition described in #92888 }, 2, TimeUnit.MINUTES); @@ -238,9 +238,9 @@ public void testGeoIpDatabasesDownload() throws Exception { assertBusy(() -> { try { GeoIpTaskState state = (GeoIpTaskState) getTask().getState(); - assertEquals( - Set.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "MyCustomGeoLite2-City.mmdb"), - state.getDatabases().keySet() + assertThat( + state.getDatabases().keySet(), + containsInAnyOrder("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "MyCustomGeoLite2-City.mmdb") ); GeoIpTaskState.Metadata metadata = state.getDatabases().get(id); int size = metadata.lastChunk() - metadata.firstChunk() + 1; @@ -301,9 +301,9 @@ public void testGeoIpDatabasesDownloadNoGeoipProcessors() throws Exception { assertNotNull(getTask().getState()); // removing all geoip processors should not result in the task being stopped assertBusy(() -> { GeoIpTaskState state = getGeoIpTaskState(); - assertEquals( - Set.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "MyCustomGeoLite2-City.mmdb"), - state.getDatabases().keySet() + assertThat( + state.getDatabases().keySet(), + containsInAnyOrder("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "MyCustomGeoLite2-City.mmdb") ); }); } @@ -337,9 +337,9 @@ public void testDoNotDownloadDatabaseOnPipelineCreation() throws Exception { assertAcked(indicesAdmin().prepareUpdateSettings(indexIdentifier).setSettings(indexSettings).get()); assertBusy(() -> { GeoIpTaskState state = getGeoIpTaskState(); - assertEquals( - Set.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "MyCustomGeoLite2-City.mmdb"), - state.getDatabases().keySet() + assertThat( + state.getDatabases().keySet(), + containsInAnyOrder("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "MyCustomGeoLite2-City.mmdb") ); }, 2, TimeUnit.MINUTES); diff --git a/modules/ingest-geoip/src/main/java/module-info.java b/modules/ingest-geoip/src/main/java/module-info.java index fa0b0266414f0..4d0acefcb6c9f 100644 --- a/modules/ingest-geoip/src/main/java/module-info.java +++ b/modules/ingest-geoip/src/main/java/module-info.java @@ -15,5 +15,6 @@ requires com.maxmind.geoip2; requires com.maxmind.db; + exports org.elasticsearch.ingest.geoip.direct to org.elasticsearch.server; exports org.elasticsearch.ingest.geoip.stats to org.elasticsearch.server; } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java index efae8fa0c50ca..dcb882ede230c 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java @@ -24,6 +24,7 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.Tuple; import org.elasticsearch.env.Environment; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.index.Index; @@ -52,7 +53,6 @@ import java.util.Collection; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -64,6 +64,7 @@ import java.util.zip.GZIPInputStream; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpTaskState.getEnterpriseGeoIpTaskState; import static org.elasticsearch.ingest.geoip.GeoIpTaskState.getGeoIpTaskState; /** @@ -183,13 +184,14 @@ public Boolean isValid(String databaseFile) { if (state == null) { return true; } + GeoIpTaskState.Metadata metadata = state.getDatabases().get(databaseFile); // we never remove metadata from cluster state, if metadata is null we deal with built-in database, which is always valid if (metadata == null) { return true; } - boolean valid = metadata.isValid(currentState.metadata().settings()); + boolean valid = metadata.isNewEnough(currentState.metadata().settings()); if (valid && metadata.isCloseToExpiration()) { HeaderWarning.addWarning( "database [{}] was not updated for over 25 days, geoip processor will stop working if there is no update for 30 days", @@ -269,20 +271,52 @@ void checkDatabases(ClusterState state) { } } - GeoIpTaskState taskState = getGeoIpTaskState(state); - if (taskState == null) { - // Note: an empty state will purge stale entries in databases map - taskState = GeoIpTaskState.EMPTY; + // we'll consult each of the geoip downloaders to build up a list of database metadatas to work with + List> validMetadatas = new ArrayList<>(); + + // process the geoip task state for the (ordinary) geoip downloader + { + GeoIpTaskState taskState = getGeoIpTaskState(state); + if (taskState == null) { + // Note: an empty state will purge stale entries in databases map + taskState = GeoIpTaskState.EMPTY; + } + validMetadatas.addAll( + taskState.getDatabases() + .entrySet() + .stream() + .filter(e -> e.getValue().isNewEnough(state.getMetadata().settings())) + .map(entry -> Tuple.tuple(entry.getKey(), entry.getValue())) + .toList() + ); + } + + // process the geoip task state for the enterprise geoip downloader + { + EnterpriseGeoIpTaskState taskState = getEnterpriseGeoIpTaskState(state); + if (taskState == null) { + // Note: an empty state will purge stale entries in databases map + taskState = EnterpriseGeoIpTaskState.EMPTY; + } + validMetadatas.addAll( + taskState.getDatabases() + .entrySet() + .stream() + .filter(e -> e.getValue().isNewEnough(state.getMetadata().settings())) + .map(entry -> Tuple.tuple(entry.getKey(), entry.getValue())) + .toList() + ); } - taskState.getDatabases().entrySet().stream().filter(e -> e.getValue().isValid(state.getMetadata().settings())).forEach(e -> { - String name = e.getKey(); - GeoIpTaskState.Metadata metadata = e.getValue(); + // run through all the valid metadatas, regardless of source, and retrieve them + validMetadatas.forEach(e -> { + String name = e.v1(); + GeoIpTaskState.Metadata metadata = e.v2(); DatabaseReaderLazyLoader reference = databases.get(name); String remoteMd5 = metadata.md5(); String localMd5 = reference != null ? reference.getMd5() : null; if (Objects.equals(localMd5, remoteMd5)) { - logger.debug("Current reference of [{}] is up to date [{}] with was recorded in CS [{}]", name, localMd5, remoteMd5); + logger.debug("[{}] is up to date [{}] with cluster state [{}]", name, localMd5, remoteMd5); return; } @@ -293,15 +327,14 @@ void checkDatabases(ClusterState state) { } }); + // TODO perhaps we need to handle the license flap persistent task state better than we do + // i think the ideal end state is that we *do not* drop the files that the enterprise downloader + // handled if they fall out -- which means we need to track that in the databases map itself + + // start with the list of all databases we currently know about in this service, + // then drop the ones that didn't check out as valid from the task states List staleEntries = new ArrayList<>(databases.keySet()); - staleEntries.removeAll( - taskState.getDatabases() - .entrySet() - .stream() - .filter(e -> e.getValue().isValid(state.getMetadata().settings())) - .map(Map.Entry::getKey) - .collect(Collectors.toSet()) - ); + staleEntries.removeAll(validMetadatas.stream().map(Tuple::v1).collect(Collectors.toSet())); removeStaleEntries(staleEntries); } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java new file mode 100644 index 0000000000000..9645e34751642 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java @@ -0,0 +1,474 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.flush.FlushRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.hash.MessageDigests; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.reindex.DeleteByQueryAction; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.ingest.geoip.GeoIpTaskState.Metadata; +import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration; +import org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationMetadata; +import org.elasticsearch.persistent.AllocatedPersistentTask; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata.PersistentTask; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.Scheduler; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.io.InputStream; +import java.net.PasswordAuthentication; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderTaskExecutor.MAXMIND_SETTINGS_PREFIX; + +/** + * Main component responsible for downloading new GeoIP databases. + * New databases are downloaded in chunks and stored in .geoip_databases index + * Downloads are verified against MD5 checksum provided by the server + * Current state of all stored databases is stored in cluster state in persistent task state + */ +public class EnterpriseGeoIpDownloader extends AllocatedPersistentTask { + + private static final Logger logger = LogManager.getLogger(EnterpriseGeoIpDownloader.class); + private static final Pattern CHECKSUM_PATTERN = Pattern.compile("(\\w{64})\\s\\s(.*)"); + + // for overriding in tests + static String DEFAULT_MAXMIND_ENDPOINT = System.getProperty( + MAXMIND_SETTINGS_PREFIX + "endpoint.default", + "https://download.maxmind.com/geoip/databases" + ); + // n.b. a future enhancement might be to allow for a MAXMIND_ENDPOINT_SETTING, but + // at the moment this is an unsupported system property for use in tests (only) + + static String downloadUrl(final String name, final String suffix) { + String endpointPattern = DEFAULT_MAXMIND_ENDPOINT; + if (endpointPattern.contains("%")) { + throw new IllegalArgumentException("Invalid endpoint [" + endpointPattern + "]"); + } + if (endpointPattern.endsWith("/") == false) { + endpointPattern += "/"; + } + endpointPattern += "%s/download?suffix=%s"; + + // at this point the pattern looks like this (in the default case): + // https://download.maxmind.com/geoip/databases/%s/download?suffix=%s + + return Strings.format(endpointPattern, name, suffix); + } + + static final String DATABASES_INDEX = ".geoip_databases"; + static final int MAX_CHUNK_SIZE = 1024 * 1024; + + private final Client client; + private final HttpClient httpClient; + private final ClusterService clusterService; + private final ThreadPool threadPool; + + // visible for testing + protected volatile EnterpriseGeoIpTaskState state; + private volatile Scheduler.ScheduledCancellable scheduled; + private final Supplier pollIntervalSupplier; + private final Function credentialsBuilder; + + EnterpriseGeoIpDownloader( + Client client, + HttpClient httpClient, + ClusterService clusterService, + ThreadPool threadPool, + long id, + String type, + String action, + String description, + TaskId parentTask, + Map headers, + Supplier pollIntervalSupplier, + Function credentialsBuilder + ) { + super(id, type, action, description, parentTask, headers); + this.client = client; + this.httpClient = httpClient; + this.clusterService = clusterService; + this.threadPool = threadPool; + this.pollIntervalSupplier = pollIntervalSupplier; + this.credentialsBuilder = credentialsBuilder; + } + + void setState(EnterpriseGeoIpTaskState state) { + // this is for injecting the state in GeoIpDownloaderTaskExecutor#nodeOperation just after the task instance has been created + // by the PersistentTasksNodeService -- since the GeoIpDownloader is newly created, the state will be null, and the passed-in + // state cannot be null + assert this.state == null + : "setState() cannot be called when state is already non-null. This most likely happened because setState() was called twice"; + assert state != null : "Should never call setState with a null state. Pass an EnterpriseGeoIpTaskState.EMPTY instead."; + this.state = state; + } + + // visible for testing + void updateDatabases() throws IOException { + var clusterState = clusterService.state(); + var geoipIndex = clusterState.getMetadata().getIndicesLookup().get(EnterpriseGeoIpDownloader.DATABASES_INDEX); + if (geoipIndex != null) { + logger.trace("the geoip index [{}] exists", EnterpriseGeoIpDownloader.DATABASES_INDEX); + if (clusterState.getRoutingTable().index(geoipIndex.getWriteIndex()).allPrimaryShardsActive() == false) { + logger.debug("not updating databases because not all primary shards of [{}] index are active yet", DATABASES_INDEX); + return; + } + var blockException = clusterState.blocks().indexBlockedException(ClusterBlockLevel.WRITE, geoipIndex.getWriteIndex().getName()); + if (blockException != null) { + throw blockException; + } + } + + logger.trace("Updating geoip databases"); + IngestGeoIpMetadata geoIpMeta = clusterState.metadata().custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY); + + // if there are entries in the cs that aren't in the persistent task state, + // then download those (only) + // --- + // if there are in the persistent task state, that aren't in the cluster state + // then nuke those (only) + // --- + // else, just download everything + boolean addedSomething = false; + { + Set existingDatabaseNames = state.getDatabases().keySet(); + for (Map.Entry entry : geoIpMeta.getDatabases().entrySet()) { + final String id = entry.getKey(); + DatabaseConfiguration database = entry.getValue().database(); + if (existingDatabaseNames.contains(database.name() + ".mmdb") == false) { + logger.debug("A new database appeared [{}]", database.name()); + + final String accountId = database.maxmind().accountId(); + try (HttpClient.PasswordAuthenticationHolder holder = credentialsBuilder.apply(accountId)) { + if (holder == null) { + logger.warn("No credentials found to download database [{}], skipping download...", id); + } else { + processDatabase(holder.get(), database); + addedSomething = true; + } + } + } + } + } + + boolean droppedSomething = false; + { + // rip anything out of the task state that doesn't match what's in the cluster state, + // that is, if there's no longer an entry for a database in the repository, + // then drop it from the task state, too + Set databases = geoIpMeta.getDatabases() + .values() + .stream() + .map(c -> c.database().name() + ".mmdb") + .collect(Collectors.toSet()); + EnterpriseGeoIpTaskState _state = state; + Collection> metas = _state.getDatabases() + .entrySet() + .stream() + .map(entry -> Tuple.tuple(entry.getKey(), entry.getValue())) + .toList(); + for (Tuple metaTuple : metas) { + String name = metaTuple.v1(); + Metadata meta = metaTuple.v2(); + if (databases.contains(name) == false) { + logger.debug("Dropping [{}], databases was {}", name, databases); + _state = _state.remove(name); + deleteOldChunks(name, meta.lastChunk() + 1); + droppedSomething = true; + } + } + if (droppedSomething) { + state = _state; + updateTaskState(); + } + } + + if (addedSomething == false && droppedSomething == false) { + RuntimeException accumulator = null; + for (Map.Entry entry : geoIpMeta.getDatabases().entrySet()) { + final String id = entry.getKey(); + DatabaseConfiguration database = entry.getValue().database(); + + final String accountId = database.maxmind().accountId(); + try (HttpClient.PasswordAuthenticationHolder holder = credentialsBuilder.apply(accountId)) { + if (holder == null) { + logger.warn("No credentials found to download database [{}], skipping download...", id); + } else { + processDatabase(holder.get(), database); + } + } catch (Exception e) { + accumulator = ExceptionsHelper.useOrSuppress(accumulator, ExceptionsHelper.convertToRuntime(e)); + } + } + if (accumulator != null) { + throw accumulator; + } + } + } + + /** + * This method fetches the sha256 file and tar.gz file for the given database from the Maxmind endpoint, then indexes that tar.gz + * file into the .geoip_databases Elasticsearch index, deleting any old versions of the database tar.gz from the index if they exist. + * If the computed sha256 does not match the expected sha256, an error will be logged and the database will not be put into the + * Elasticsearch index. + *

+ * As an implementation detail, this method retrieves the sha256 checksum of the database to download and then invokes + * {@link EnterpriseGeoIpDownloader#processDatabase(PasswordAuthentication, String, String, String)} with that checksum, deferring to + * that method to actually download and process the tar.gz itself. + * + * @param auth The credentials to use to download from the Maxmind endpoint + * @param database The database to be downloaded from Maxmind and indexed into an Elasticsearch index + * @throws IOException If there is an error fetching the sha256 file + */ + void processDatabase(PasswordAuthentication auth, DatabaseConfiguration database) throws IOException { + final String name = database.name(); + logger.debug("Processing database [{}] for configuration [{}]", name, database.id()); + + final String sha256Url = downloadUrl(name, "tar.gz.sha256"); + final String tgzUrl = downloadUrl(name, "tar.gz"); + + String result = new String(httpClient.getBytes(auth, sha256Url), StandardCharsets.UTF_8).trim(); // this throws if the auth is bad + var matcher = CHECKSUM_PATTERN.matcher(result); + boolean match = matcher.matches(); + if (match == false) { + throw new RuntimeException("Unexpected sha256 response from [" + sha256Url + "]"); + } + final String sha256 = matcher.group(1); + // the name that comes from the enterprise downloader cluster state doesn't include the .mmdb extension, + // but the downloading and indexing of database code expects it to be there, so we add it on here before further processing + processDatabase(auth, name + ".mmdb", sha256, tgzUrl); + } + + /** + * This method fetches the tar.gz file for the given database from the Maxmind endpoint, then indexes that tar.gz + * file into the .geoip_databases Elasticsearch index, deleting any old versions of the database tar.gz from the index if they exist. + * + * @param auth The credentials to use to download from the Maxmind endpoint + * The name of the database to be downloaded from Maxmind and indexed into an Elasticsearch index + * @param sha256 The sha256 to compare to the computed sha256 of the downloaded tar.gz file + * @param url The URL for the Maxmind endpoint from which the database's tar.gz will be downloaded + */ + private void processDatabase(PasswordAuthentication auth, String name, String sha256, String url) { + Metadata metadata = state.getDatabases().getOrDefault(name, Metadata.EMPTY); + if (Objects.equals(metadata.sha256(), sha256)) { + updateTimestamp(name, metadata); + return; + } + logger.debug("downloading geoip database [{}]", name); + long start = System.currentTimeMillis(); + try (InputStream is = httpClient.get(auth, url)) { + int firstChunk = metadata.lastChunk() + 1; // if there is no metadata, then Metadata.EMPTY + 1 = 0 + Tuple tuple = indexChunks(name, is, firstChunk, MessageDigests.sha256(), sha256, start); + int lastChunk = tuple.v1(); + String md5 = tuple.v2(); + if (lastChunk > firstChunk) { + state = state.put(name, new Metadata(start, firstChunk, lastChunk - 1, md5, start, sha256)); + updateTaskState(); + logger.info("successfully downloaded geoip database [{}]", name); + deleteOldChunks(name, firstChunk); + } + } catch (Exception e) { + logger.error(() -> "error downloading geoip database [" + name + "]", e); + } + } + + // visible for testing + void deleteOldChunks(String name, int firstChunk) { + BoolQueryBuilder queryBuilder = new BoolQueryBuilder().filter(new MatchQueryBuilder("name", name)) + .filter(new RangeQueryBuilder("chunk").to(firstChunk, false)); + DeleteByQueryRequest request = new DeleteByQueryRequest(); + request.indices(DATABASES_INDEX); + request.setQuery(queryBuilder); + client.execute( + DeleteByQueryAction.INSTANCE, + request, + ActionListener.wrap(r -> {}, e -> logger.warn("could not delete old chunks for geoip database [" + name + "]", e)) + ); + } + + // visible for testing + protected void updateTimestamp(String name, Metadata old) { + logger.debug("geoip database [{}] is up to date, updated timestamp", name); + state = state.put( + name, + new Metadata(old.lastUpdate(), old.firstChunk(), old.lastChunk(), old.md5(), System.currentTimeMillis(), old.sha256()) + ); + updateTaskState(); + } + + void updateTaskState() { + PlainActionFuture> future = new PlainActionFuture<>(); + updatePersistentTaskState(state, future); + state = ((EnterpriseGeoIpTaskState) future.actionGet().getState()); + } + + // visible for testing + Tuple indexChunks( + String name, + InputStream is, + int chunk, + @Nullable MessageDigest digest, + String expectedChecksum, + long timestamp + ) throws IOException { + MessageDigest md5 = MessageDigests.md5(); + for (byte[] buf = getChunk(is); buf.length != 0; buf = getChunk(is)) { + md5.update(buf); + if (digest != null) { + digest.update(buf); + } + IndexRequest indexRequest = new IndexRequest(DATABASES_INDEX).id(name + "_" + chunk + "_" + timestamp) + .create(true) + .source(XContentType.SMILE, "name", name, "chunk", chunk, "data", buf); + client.index(indexRequest).actionGet(); + chunk++; + } + + // May take some time before automatic flush kicks in: + // (otherwise the translog will contain large documents for some time without good reason) + FlushRequest flushRequest = new FlushRequest(DATABASES_INDEX); + client.admin().indices().flush(flushRequest).actionGet(); + // Ensure that the chunk documents are visible: + RefreshRequest refreshRequest = new RefreshRequest(DATABASES_INDEX); + client.admin().indices().refresh(refreshRequest).actionGet(); + + String actualMd5 = MessageDigests.toHexString(md5.digest()); + String actualChecksum = digest == null ? actualMd5 : MessageDigests.toHexString(digest.digest()); + if (Objects.equals(expectedChecksum, actualChecksum) == false) { + throw new IOException("checksum mismatch, expected [" + expectedChecksum + "], actual [" + actualChecksum + "]"); + } + return Tuple.tuple(chunk, actualMd5); + } + + // visible for testing + static byte[] getChunk(InputStream is) throws IOException { + byte[] buf = new byte[MAX_CHUNK_SIZE]; + int chunkSize = 0; + while (chunkSize < MAX_CHUNK_SIZE) { + int read = is.read(buf, chunkSize, MAX_CHUNK_SIZE - chunkSize); + if (read == -1) { + break; + } + chunkSize += read; + } + if (chunkSize < MAX_CHUNK_SIZE) { + buf = Arrays.copyOf(buf, chunkSize); + } + return buf; + } + + /** + * Downloads the geoip databases now, and schedules them to be downloaded again after pollInterval. + */ + synchronized void runDownloader() { + // by the time we reach here, the state will never be null + assert this.state != null : "this.setState() is null. You need to call setState() before calling runDownloader()"; + + // there's a race condition between here and requestReschedule. originally this scheduleNextRun call was at the end of this + // block, but remember that updateDatabases can take seconds to run (it's downloading bytes from the internet), and so during the + // very first run there would be no future run scheduled to reschedule in requestReschedule. which meant that if you went from zero + // to N(>=2) databases in quick succession, then all but the first database wouldn't necessarily get downloaded, because the + // requestReschedule call in the EnterpriseGeoIpDownloaderTaskExecutor's clusterChanged wouldn't have a scheduled future run to + // reschedule. scheduling the next run at the beginning of this run means that there's a much smaller window (milliseconds?, rather + // than seconds) in which such a race could occur. technically there's a window here, still, but i think it's _greatly_ reduced. + scheduleNextRun(pollIntervalSupplier.get()); + // TODO regardless of the above comment, i like the idea of checking the lowest last-checked time and then running the math to get + // to the next interval from then -- maybe that's a neat future enhancement to add + + if (isCancelled() || isCompleted()) { + return; + } + try { + updateDatabases(); // n.b. this downloads bytes from the internet, it can take a while + } catch (Exception e) { + logger.error("exception during geoip databases update", e); + } + try { + cleanDatabases(); + } catch (Exception e) { + logger.error("exception during geoip databases cleanup", e); + } + } + + /** + * This method requests that the downloader be rescheduled to run immediately (presumably because a dynamic property supplied by + * pollIntervalSupplier or eagerDownloadSupplier has changed, or a pipeline with a geoip processor has been added). This method does + * nothing if this task is cancelled, completed, or has not yet been scheduled to run for the first time. It cancels any existing + * scheduled run. + */ + public void requestReschedule() { + if (isCancelled() || isCompleted()) { + return; + } + if (scheduled != null && scheduled.cancel()) { + scheduleNextRun(TimeValue.ZERO); + } + } + + private void cleanDatabases() { + List> expiredDatabases = state.getDatabases() + .entrySet() + .stream() + .filter(e -> e.getValue().isNewEnough(clusterService.state().metadata().settings()) == false) + .map(entry -> Tuple.tuple(entry.getKey(), entry.getValue())) + .toList(); + expiredDatabases.forEach(e -> { + String name = e.v1(); + Metadata meta = e.v2(); + deleteOldChunks(name, meta.lastChunk() + 1); + state = state.put(name, new Metadata(meta.lastUpdate(), meta.firstChunk(), meta.lastChunk(), meta.md5(), meta.lastCheck() - 1)); + updateTaskState(); + }); + } + + @Override + protected void onCancelled() { + if (scheduled != null) { + scheduled.cancel(); + } + markAsCompleted(); + } + + private void scheduleNextRun(TimeValue time) { + if (threadPool.scheduler().isShutdown() == false) { + scheduled = threadPool.schedule(this::runDownloader, time, threadPool.generic()); + } + } + +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java new file mode 100644 index 0000000000000..8fc46fe157548 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java @@ -0,0 +1,257 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.SecureSetting; +import org.elasticsearch.common.settings.SecureSettings; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.ingest.EnterpriseGeoIpTask.EnterpriseGeoIpTaskParams; +import org.elasticsearch.ingest.IngestService; +import org.elasticsearch.persistent.AllocatedPersistentTask; +import org.elasticsearch.persistent.PersistentTaskState; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; +import org.elasticsearch.persistent.PersistentTasksExecutor; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.io.InputStream; +import java.security.GeneralSecurityException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import static org.elasticsearch.ingest.EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER; +import static org.elasticsearch.ingest.geoip.GeoIpDownloaderTaskExecutor.ENABLED_SETTING; +import static org.elasticsearch.ingest.geoip.GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING; + +public class EnterpriseGeoIpDownloaderTaskExecutor extends PersistentTasksExecutor + implements + ClusterStateListener { + private static final Logger logger = LogManager.getLogger(EnterpriseGeoIpDownloader.class); + + static final String MAXMIND_SETTINGS_PREFIX = "ingest.geoip.downloader.maxmind."; + + public static final Setting MAXMIND_LICENSE_KEY_SETTING = SecureSetting.secureString( + MAXMIND_SETTINGS_PREFIX + "license_key", + null + ); + + private final Client client; + private final HttpClient httpClient; + private final ClusterService clusterService; + private final ThreadPool threadPool; + private final Settings settings; + private volatile TimeValue pollInterval; + private final AtomicReference currentTask = new AtomicReference<>(); + + private volatile SecureSettings cachedSecureSettings; + + EnterpriseGeoIpDownloaderTaskExecutor(Client client, HttpClient httpClient, ClusterService clusterService, ThreadPool threadPool) { + super(ENTERPRISE_GEOIP_DOWNLOADER, threadPool.generic()); + this.client = new OriginSettingClient(client, IngestService.INGEST_ORIGIN); + this.httpClient = httpClient; + this.clusterService = clusterService; + this.threadPool = threadPool; + this.settings = clusterService.getSettings(); + this.pollInterval = POLL_INTERVAL_SETTING.get(settings); + + // do an initial load using the node settings + reload(clusterService.getSettings()); + } + + /** + * This method completes the initialization of the EnterpriseGeoIpDownloaderTaskExecutor by registering several listeners. + */ + public void init() { + clusterService.addListener(this); + clusterService.getClusterSettings().addSettingsUpdateConsumer(POLL_INTERVAL_SETTING, this::setPollInterval); + } + + private void setPollInterval(TimeValue pollInterval) { + if (Objects.equals(this.pollInterval, pollInterval) == false) { + this.pollInterval = pollInterval; + EnterpriseGeoIpDownloader currentDownloader = getCurrentTask(); + if (currentDownloader != null) { + currentDownloader.requestReschedule(); + } + } + } + + private HttpClient.PasswordAuthenticationHolder buildCredentials(final String username) { + final char[] passwordChars; + if (cachedSecureSettings.getSettingNames().contains(MAXMIND_LICENSE_KEY_SETTING.getKey())) { + passwordChars = cachedSecureSettings.getString(MAXMIND_LICENSE_KEY_SETTING.getKey()).getChars(); + } else { + passwordChars = null; + } + + // if the username is missing, empty, or blank, return null as 'no auth' + if (username == null || username.isEmpty() || username.isBlank()) { + return null; + } + + // likewise if the password chars array is missing or empty, return null as 'no auth' + if (passwordChars == null || passwordChars.length == 0) { + return null; + } + + return new HttpClient.PasswordAuthenticationHolder(username, passwordChars); + } + + @Override + protected EnterpriseGeoIpDownloader createTask( + long id, + String type, + String action, + TaskId parentTaskId, + PersistentTasksCustomMetadata.PersistentTask taskInProgress, + Map headers + ) { + return new EnterpriseGeoIpDownloader( + client, + httpClient, + clusterService, + threadPool, + id, + type, + action, + getDescription(taskInProgress), + parentTaskId, + headers, + () -> pollInterval, + this::buildCredentials + ); + } + + @Override + protected void nodeOperation(AllocatedPersistentTask task, EnterpriseGeoIpTaskParams params, PersistentTaskState state) { + EnterpriseGeoIpDownloader downloader = (EnterpriseGeoIpDownloader) task; + EnterpriseGeoIpTaskState geoIpTaskState = (state == null) ? EnterpriseGeoIpTaskState.EMPTY : (EnterpriseGeoIpTaskState) state; + downloader.setState(geoIpTaskState); + currentTask.set(downloader); + if (ENABLED_SETTING.get(clusterService.state().metadata().settings(), settings)) { + downloader.runDownloader(); + } + } + + public EnterpriseGeoIpDownloader getCurrentTask() { + return currentTask.get(); + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + EnterpriseGeoIpDownloader currentDownloader = getCurrentTask(); + if (currentDownloader != null) { + boolean hasGeoIpMetadataChanges = event.metadataChanged() + && event.changedCustomMetadataSet().contains(IngestGeoIpMetadata.TYPE); + if (hasGeoIpMetadataChanges) { + currentDownloader.requestReschedule(); // watching the cluster changed events to kick the thing off if it's not running + } + } + } + + public synchronized void reload(Settings settings) { + // `SecureSettings` are available here! cache them as they will be needed + // whenever dynamic cluster settings change and we have to rebuild the accounts + try { + this.cachedSecureSettings = extractSecureSettings(settings, List.of(MAXMIND_LICENSE_KEY_SETTING)); + } catch (GeneralSecurityException e) { + // rethrow as a runtime exception, there's logging higher up the call chain around ReloadablePlugin + throw new ElasticsearchException("Exception while reloading enterprise geoip download task executor", e); + } + } + + /** + * Extracts the {@link SecureSettings}` out of the passed in {@link Settings} object. The {@code Setting} argument has to have the + * {@code SecureSettings} open/available. Normally {@code SecureSettings} are available only under specific callstacks (eg. during node + * initialization or during a `reload` call). The returned copy can be reused freely as it will never be closed (this is a bit of + * cheating, but it is necessary in this specific circumstance). Only works for secure settings of type string (not file). + * + * @param source A {@code Settings} object with its {@code SecureSettings} open/available. + * @param securePluginSettings The list of settings to copy. + * @return A copy of the {@code SecureSettings} of the passed in {@code Settings} argument. + */ + private static SecureSettings extractSecureSettings(Settings source, List> securePluginSettings) + throws GeneralSecurityException { + // get the secure settings out + final SecureSettings sourceSecureSettings = Settings.builder().put(source, true).getSecureSettings(); + // filter and cache them... + final Map innerMap = new HashMap<>(); + if (sourceSecureSettings != null && securePluginSettings != null) { + for (final String settingKey : sourceSecureSettings.getSettingNames()) { + for (final Setting secureSetting : securePluginSettings) { + if (secureSetting.match(settingKey)) { + innerMap.put( + settingKey, + new SecureSettingValue( + sourceSecureSettings.getString(settingKey), + sourceSecureSettings.getSHA256Digest(settingKey) + ) + ); + } + } + } + } + return new SecureSettings() { + @Override + public boolean isLoaded() { + return true; + } + + @Override + public SecureString getString(String setting) { + return innerMap.get(setting).value(); + } + + @Override + public Set getSettingNames() { + return innerMap.keySet(); + } + + @Override + public InputStream getFile(String setting) { + throw new UnsupportedOperationException("A cached SecureSetting cannot be a file"); + } + + @Override + public byte[] getSHA256Digest(String setting) { + return innerMap.get(setting).sha256Digest(); + } + + @Override + public void close() throws IOException {} + + @Override + public void writeTo(StreamOutput out) throws IOException { + throw new UnsupportedOperationException("A cached SecureSetting cannot be serialized"); + } + }; + } + + /** + * A single-purpose record for the internal implementation of extractSecureSettings + */ + private record SecureSettingValue(SecureString value, byte[] sha256Digest) {} +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java new file mode 100644 index 0000000000000..57e944ef9b994 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.VersionedNamedWriteable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.ingest.EnterpriseGeoIpTask; +import org.elasticsearch.ingest.geoip.GeoIpTaskState.Metadata; +import org.elasticsearch.persistent.PersistentTaskState; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import static org.elasticsearch.ingest.geoip.GeoIpDownloader.GEOIP_DOWNLOADER; +import static org.elasticsearch.persistent.PersistentTasksCustomMetadata.getTaskWithId; +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; + +class EnterpriseGeoIpTaskState implements PersistentTaskState, VersionedNamedWriteable { + + private static final ParseField DATABASES = new ParseField("databases"); + + static final EnterpriseGeoIpTaskState EMPTY = new EnterpriseGeoIpTaskState(Map.of()); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + GEOIP_DOWNLOADER, + true, + args -> { + List> databases = (List>) args[0]; + return new EnterpriseGeoIpTaskState(databases.stream().collect(Collectors.toMap(Tuple::v1, Tuple::v2))); + } + ); + + static { + PARSER.declareNamedObjects(constructorArg(), (p, c, name) -> Tuple.tuple(name, Metadata.fromXContent(p)), DATABASES); + } + + public static EnterpriseGeoIpTaskState fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + private final Map databases; + + EnterpriseGeoIpTaskState(Map databases) { + this.databases = Map.copyOf(databases); + } + + EnterpriseGeoIpTaskState(StreamInput input) throws IOException { + databases = input.readImmutableMap( + in -> new Metadata(in.readLong(), in.readVInt(), in.readVInt(), in.readString(), in.readLong(), in.readOptionalString()) + ); + } + + public EnterpriseGeoIpTaskState put(String name, Metadata metadata) { + HashMap newDatabases = new HashMap<>(databases); + newDatabases.put(name, metadata); + return new EnterpriseGeoIpTaskState(newDatabases); + } + + public EnterpriseGeoIpTaskState remove(String name) { + HashMap newDatabases = new HashMap<>(databases); + newDatabases.remove(name); + return new EnterpriseGeoIpTaskState(newDatabases); + } + + public Map getDatabases() { + return databases; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EnterpriseGeoIpTaskState that = (EnterpriseGeoIpTaskState) o; + return databases.equals(that.databases); + } + + @Override + public int hashCode() { + return Objects.hash(databases); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.startObject("databases"); + for (Map.Entry e : databases.entrySet()) { + builder.field(e.getKey(), e.getValue()); + } + builder.endObject(); + } + builder.endObject(); + return builder; + } + + @Override + public String getWriteableName() { + return "enterprise-geoip-downloader"; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeMap(databases, (o, v) -> { + o.writeLong(v.lastUpdate()); + o.writeVInt(v.firstChunk()); + o.writeVInt(v.lastChunk()); + o.writeString(v.md5()); + o.writeLong(v.lastCheck()); + o.writeOptionalString(v.sha256()); + }); + } + + /** + * Retrieves the geoip downloader's task state from the cluster state. This may return null in some circumstances, + * for example if the geoip downloader task hasn't been created yet (which it wouldn't be if it's disabled). + * + * @param state the cluster state to read the task state from + * @return the geoip downloader's task state or null if there is not a state to read + */ + @Nullable + static EnterpriseGeoIpTaskState getEnterpriseGeoIpTaskState(ClusterState state) { + PersistentTasksCustomMetadata.PersistentTask task = getTaskWithId(state, EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER); + return (task == null) ? null : (EnterpriseGeoIpTaskState) task.getState(); + } + +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java index 13394a2a0c7cc..ee6f2f16f051b 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; @@ -318,14 +319,15 @@ public void requestReschedule() { } private void cleanDatabases() { - List> expiredDatabases = state.getDatabases() + List> expiredDatabases = state.getDatabases() .entrySet() .stream() - .filter(e -> e.getValue().isValid(clusterService.state().metadata().settings()) == false) + .filter(e -> e.getValue().isNewEnough(clusterService.state().metadata().settings()) == false) + .map(entry -> Tuple.tuple(entry.getKey(), entry.getValue())) .toList(); expiredDatabases.forEach(e -> { - String name = e.getKey(); - Metadata meta = e.getValue(); + String name = e.v1(); + Metadata meta = e.v2(); deleteOldChunks(name, meta.lastChunk() + 1); state = state.put(name, new Metadata(meta.lastUpdate(), meta.firstChunk(), meta.lastChunk(), meta.md5(), meta.lastCheck() - 1)); updateTaskState(); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java index 09ac488f96e2d..3f89bb1dd5c50 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java @@ -217,7 +217,7 @@ public void clusterChanged(ClusterChangedEvent event) { } boolean hasIndicesChanges = event.previousState().metadata().indices().equals(event.state().metadata().indices()) == false; - boolean hasIngestPipelineChanges = event.changedCustomMetadataSet().contains(IngestMetadata.TYPE); + boolean hasIngestPipelineChanges = event.metadataChanged() && event.changedCustomMetadataSet().contains(IngestMetadata.TYPE); if (hasIngestPipelineChanges || hasIndicesChanges) { boolean newAtLeastOneGeoipProcessor = hasAtLeastOneGeoipProcessor(event.state()); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java index a405d90b24dcc..56f96786d9b7f 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java @@ -42,6 +42,11 @@ class GeoIpTaskState implements PersistentTaskState, VersionedNamedWriteable { + private static boolean includeSha256(TransportVersion version) { + return version.isPatchFrom(TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER_BACKPORT_8_15) + || version.onOrAfter(TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER); + } + private static final ParseField DATABASES = new ParseField("databases"); static final GeoIpTaskState EMPTY = new GeoIpTaskState(Map.of()); @@ -71,7 +76,16 @@ public static GeoIpTaskState fromXContent(XContentParser parser) throws IOExcept } GeoIpTaskState(StreamInput input) throws IOException { - databases = input.readImmutableMap(in -> new Metadata(in.readLong(), in.readVInt(), in.readVInt(), in.readString(), in.readLong())); + databases = input.readImmutableMap( + in -> new Metadata( + in.readLong(), + in.readVInt(), + in.readVInt(), + in.readString(), + in.readLong(), + includeSha256(in.getTransportVersion()) ? input.readOptionalString() : null + ) + ); } public GeoIpTaskState put(String name, Metadata metadata) { @@ -129,16 +143,21 @@ public void writeTo(StreamOutput out) throws IOException { o.writeVInt(v.lastChunk); o.writeString(v.md5); o.writeLong(v.lastCheck); + if (includeSha256(o.getTransportVersion())) { + o.writeOptionalString(v.sha256); + } }); } - record Metadata(long lastUpdate, int firstChunk, int lastChunk, String md5, long lastCheck) implements ToXContentObject { + record Metadata(long lastUpdate, int firstChunk, int lastChunk, String md5, long lastCheck, @Nullable String sha256) + implements + ToXContentObject { /** * An empty Metadata object useful for getOrDefault -type calls. Crucially, the 'lastChunk' is -1, so it's safe to use * with logic that says the new firstChunk is the old lastChunk + 1. */ - static Metadata EMPTY = new Metadata(-1, -1, -1, "", -1); + static Metadata EMPTY = new Metadata(-1, -1, -1, "", -1, null); private static final String NAME = GEOIP_DOWNLOADER + "-metadata"; private static final ParseField LAST_CHECK = new ParseField("last_check"); @@ -146,6 +165,7 @@ record Metadata(long lastUpdate, int firstChunk, int lastChunk, String md5, long private static final ParseField FIRST_CHUNK = new ParseField("first_chunk"); private static final ParseField LAST_CHUNK = new ParseField("last_chunk"); private static final ParseField MD5 = new ParseField("md5"); + private static final ParseField SHA256 = new ParseField("sha256"); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( NAME, @@ -155,7 +175,8 @@ record Metadata(long lastUpdate, int firstChunk, int lastChunk, String md5, long (int) args[1], (int) args[2], (String) args[3], - (long) (args[4] == null ? args[0] : args[4]) + (long) (args[4] == null ? args[0] : args[4]), + (String) args[5] ) ); @@ -165,6 +186,7 @@ record Metadata(long lastUpdate, int firstChunk, int lastChunk, String md5, long PARSER.declareInt(constructorArg(), LAST_CHUNK); PARSER.declareString(constructorArg(), MD5); PARSER.declareLong(optionalConstructorArg(), LAST_CHECK); + PARSER.declareString(optionalConstructorArg(), SHA256); } public static Metadata fromXContent(XContentParser parser) { @@ -179,11 +201,15 @@ public static Metadata fromXContent(XContentParser parser) { Objects.requireNonNull(md5); } + Metadata(long lastUpdate, int firstChunk, int lastChunk, String md5, long lastCheck) { + this(lastUpdate, firstChunk, lastChunk, md5, lastCheck, null); + } + public boolean isCloseToExpiration() { return Instant.ofEpochMilli(lastCheck).isBefore(Instant.now().minus(25, ChronoUnit.DAYS)); } - public boolean isValid(Settings settings) { + public boolean isNewEnough(Settings settings) { TimeValue valid = settings.getAsTime("ingest.geoip.database_validity", TimeValue.timeValueDays(30)); return Instant.ofEpochMilli(lastCheck).isAfter(Instant.now().minus(valid.getMillis(), ChronoUnit.MILLIS)); } @@ -197,6 +223,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(FIRST_CHUNK.getPreferredName(), firstChunk); builder.field(LAST_CHUNK.getPreferredName(), lastChunk); builder.field(MD5.getPreferredName(), md5); + if (sha256 != null) { // only serialize if not null, for prettiness reasons + builder.field(SHA256.getPreferredName(), sha256); + } } builder.endObject(); return builder; diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/HttpClient.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/HttpClient.java index 8efc4dc2e74bd..2f6bd6ef20fd0 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/HttpClient.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/HttpClient.java @@ -24,6 +24,7 @@ import java.security.AccessController; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; +import java.util.Arrays; import java.util.Objects; import static java.net.HttpURLConnection.HTTP_MOVED_PERM; @@ -34,6 +35,31 @@ class HttpClient { + /** + * A PasswordAuthenticationHolder is just a wrapper around a PasswordAuthentication to implement AutoCloseable. + * This construction makes it possible to use a PasswordAuthentication in a try-with-resources statement, which + * makes it easier to ensure cleanup of the PasswordAuthentication is performed after it's finished being used. + */ + static final class PasswordAuthenticationHolder implements AutoCloseable { + private PasswordAuthentication auth; + + PasswordAuthenticationHolder(String username, char[] passwordChars) { + this.auth = new PasswordAuthentication(username, passwordChars); // clones the passed-in chars + } + + public PasswordAuthentication get() { + Objects.requireNonNull(auth); + return auth; + } + + @Override + public void close() { + final PasswordAuthentication clear = this.auth; + this.auth = null; // set to null and then clear it + Arrays.fill(clear.getPassword(), '\0'); // zero out the password chars + } + } + // a private sentinel value for representing the idea that there's no auth for some request. // this allows us to have a not-null requirement on the methods that do accept an auth. // if you don't want auth, then don't use those methods. ;) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java new file mode 100644 index 0000000000000..f5ac755b6b980 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java @@ -0,0 +1,157 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.cluster.DiffableUtils; +import org.elasticsearch.cluster.NamedDiff; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationMetadata; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Holds the ingest-geoip databases that are available in the cluster state. + */ +public final class IngestGeoIpMetadata implements Metadata.Custom { + + public static final String TYPE = "ingest_geoip"; + private static final ParseField DATABASES_FIELD = new ParseField("databases"); + + public static final IngestGeoIpMetadata EMPTY = new IngestGeoIpMetadata(Map.of()); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "ingest_geoip_metadata", + a -> new IngestGeoIpMetadata( + ((List) a[0]).stream().collect(Collectors.toMap((m) -> m.database().id(), Function.identity())) + ) + ); + static { + PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> DatabaseConfigurationMetadata.parse(p, n), v -> { + throw new IllegalArgumentException("ordered " + DATABASES_FIELD.getPreferredName() + " are not supported"); + }, DATABASES_FIELD); + } + + private final Map databases; + + public IngestGeoIpMetadata(Map databases) { + this.databases = Map.copyOf(databases); + } + + @Override + public String getWriteableName() { + return TYPE; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER; + } + + public Map getDatabases() { + return databases; + } + + public IngestGeoIpMetadata(StreamInput in) throws IOException { + this.databases = in.readMap(StreamInput::readString, DatabaseConfigurationMetadata::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeMap(databases, StreamOutput::writeWriteable); + } + + public static IngestGeoIpMetadata fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public Iterator toXContentChunked(ToXContent.Params ignored) { + return Iterators.concat(ChunkedToXContentHelper.xContentValuesMap(DATABASES_FIELD.getPreferredName(), databases)); + } + + @Override + public EnumSet context() { + return Metadata.ALL_CONTEXTS; + } + + @Override + public Diff diff(Metadata.Custom before) { + return new GeoIpMetadataDiff((IngestGeoIpMetadata) before, this); + } + + static class GeoIpMetadataDiff implements NamedDiff { + + final Diff> databases; + + GeoIpMetadataDiff(IngestGeoIpMetadata before, IngestGeoIpMetadata after) { + this.databases = DiffableUtils.diff(before.databases, after.databases, DiffableUtils.getStringKeySerializer()); + } + + GeoIpMetadataDiff(StreamInput in) throws IOException { + databases = DiffableUtils.readJdkMapDiff( + in, + DiffableUtils.getStringKeySerializer(), + DatabaseConfigurationMetadata::new, + DatabaseConfigurationMetadata::readDiffFrom + ); + } + + @Override + public Metadata.Custom apply(Metadata.Custom part) { + return new IngestGeoIpMetadata(databases.apply(((IngestGeoIpMetadata) part).databases)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + databases.writeTo(out); + } + + @Override + public String getWriteableName() { + return TYPE; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER; + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + IngestGeoIpMetadata that = (IngestGeoIpMetadata) o; + return Objects.equals(databases, that.databases); + } + + @Override + public int hashCode() { + return Objects.hash(databases); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java index 9d0f9848d97b6..e606688ad60a0 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java @@ -12,8 +12,10 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -25,8 +27,18 @@ import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.ingest.EnterpriseGeoIpTask.EnterpriseGeoIpTaskParams; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.ingest.Processor; +import org.elasticsearch.ingest.geoip.direct.DeleteDatabaseConfigurationAction; +import org.elasticsearch.ingest.geoip.direct.GetDatabaseConfigurationAction; +import org.elasticsearch.ingest.geoip.direct.PutDatabaseConfigurationAction; +import org.elasticsearch.ingest.geoip.direct.RestDeleteDatabaseConfigurationAction; +import org.elasticsearch.ingest.geoip.direct.RestGetDatabaseConfigurationAction; +import org.elasticsearch.ingest.geoip.direct.RestPutDatabaseConfigurationAction; +import org.elasticsearch.ingest.geoip.direct.TransportDeleteDatabaseConfigurationAction; +import org.elasticsearch.ingest.geoip.direct.TransportGetDatabaseConfigurationAction; +import org.elasticsearch.ingest.geoip.direct.TransportPutDatabaseConfigurationAction; import org.elasticsearch.ingest.geoip.stats.GeoIpDownloaderStats; import org.elasticsearch.ingest.geoip.stats.GeoIpStatsAction; import org.elasticsearch.ingest.geoip.stats.GeoIpStatsTransportAction; @@ -38,6 +50,7 @@ import org.elasticsearch.plugins.IngestPlugin; import org.elasticsearch.plugins.PersistentTaskPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.ReloadablePlugin; import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; @@ -57,13 +70,21 @@ import java.util.function.Supplier; import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; +import static org.elasticsearch.ingest.EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER; import static org.elasticsearch.ingest.IngestService.INGEST_ORIGIN; import static org.elasticsearch.ingest.geoip.GeoIpDownloader.DATABASES_INDEX; import static org.elasticsearch.ingest.geoip.GeoIpDownloader.DATABASES_INDEX_PATTERN; import static org.elasticsearch.ingest.geoip.GeoIpDownloader.GEOIP_DOWNLOADER; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; -public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, SystemIndexPlugin, Closeable, PersistentTaskPlugin, ActionPlugin { +public class IngestGeoIpPlugin extends Plugin + implements + IngestPlugin, + SystemIndexPlugin, + Closeable, + PersistentTaskPlugin, + ActionPlugin, + ReloadablePlugin { public static final Setting CACHE_SIZE = Setting.longSetting("ingest.geoip.cache_size", 1000, 0, Setting.Property.NodeScope); private static final int GEOIP_INDEX_MAPPINGS_VERSION = 1; /** @@ -78,6 +99,7 @@ public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, SystemInd private final SetOnce ingestService = new SetOnce<>(); private final SetOnce databaseRegistry = new SetOnce<>(); private GeoIpDownloaderTaskExecutor geoIpDownloaderTaskExecutor; + private EnterpriseGeoIpDownloaderTaskExecutor enterpriseGeoIpDownloaderTaskExecutor; @Override public List> getSettings() { @@ -86,7 +108,8 @@ public List> getSettings() { GeoIpDownloaderTaskExecutor.EAGER_DOWNLOAD_SETTING, GeoIpDownloaderTaskExecutor.ENABLED_SETTING, GeoIpDownloader.ENDPOINT_SETTING, - GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING + GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING, + EnterpriseGeoIpDownloaderTaskExecutor.MAXMIND_LICENSE_KEY_SETTING ); } @@ -123,7 +146,16 @@ public Collection createComponents(PluginServices services) { services.threadPool() ); geoIpDownloaderTaskExecutor.init(); - return List.of(databaseRegistry.get(), geoIpDownloaderTaskExecutor); + + enterpriseGeoIpDownloaderTaskExecutor = new EnterpriseGeoIpDownloaderTaskExecutor( + services.client(), + new HttpClient(), + services.clusterService(), + services.threadPool() + ); + enterpriseGeoIpDownloaderTaskExecutor.init(); + + return List.of(databaseRegistry.get(), geoIpDownloaderTaskExecutor, enterpriseGeoIpDownloaderTaskExecutor); } @Override @@ -139,12 +171,17 @@ public List> getPersistentTasksExecutor( SettingsModule settingsModule, IndexNameExpressionResolver expressionResolver ) { - return List.of(geoIpDownloaderTaskExecutor); + return List.of(geoIpDownloaderTaskExecutor, enterpriseGeoIpDownloaderTaskExecutor); } @Override public List> getActions() { - return List.of(new ActionHandler<>(GeoIpStatsAction.INSTANCE, GeoIpStatsTransportAction.class)); + return List.of( + new ActionHandler<>(GeoIpStatsAction.INSTANCE, GeoIpStatsTransportAction.class), + new ActionHandler<>(GetDatabaseConfigurationAction.INSTANCE, TransportGetDatabaseConfigurationAction.class), + new ActionHandler<>(DeleteDatabaseConfigurationAction.INSTANCE, TransportDeleteDatabaseConfigurationAction.class), + new ActionHandler<>(PutDatabaseConfigurationAction.INSTANCE, TransportPutDatabaseConfigurationAction.class) + ); } @Override @@ -159,22 +196,41 @@ public List getRestHandlers( Supplier nodesInCluster, Predicate clusterSupportsFeature ) { - return List.of(new RestGeoIpStatsAction()); + return List.of( + new RestGeoIpStatsAction(), + new RestGetDatabaseConfigurationAction(), + new RestDeleteDatabaseConfigurationAction(), + new RestPutDatabaseConfigurationAction() + ); } @Override public List getNamedXContent() { return List.of( new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(GEOIP_DOWNLOADER), GeoIpTaskParams::fromXContent), - new NamedXContentRegistry.Entry(PersistentTaskState.class, new ParseField(GEOIP_DOWNLOADER), GeoIpTaskState::fromXContent) + new NamedXContentRegistry.Entry(PersistentTaskState.class, new ParseField(GEOIP_DOWNLOADER), GeoIpTaskState::fromXContent), + new NamedXContentRegistry.Entry( + PersistentTaskParams.class, + new ParseField(ENTERPRISE_GEOIP_DOWNLOADER), + EnterpriseGeoIpTaskParams::fromXContent + ), + new NamedXContentRegistry.Entry( + PersistentTaskState.class, + new ParseField(ENTERPRISE_GEOIP_DOWNLOADER), + EnterpriseGeoIpTaskState::fromXContent + ) ); } @Override public List getNamedWriteables() { return List.of( + new NamedWriteableRegistry.Entry(Metadata.Custom.class, IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.GeoIpMetadataDiff::new), new NamedWriteableRegistry.Entry(PersistentTaskState.class, GEOIP_DOWNLOADER, GeoIpTaskState::new), new NamedWriteableRegistry.Entry(PersistentTaskParams.class, GEOIP_DOWNLOADER, GeoIpTaskParams::new), + new NamedWriteableRegistry.Entry(PersistentTaskState.class, ENTERPRISE_GEOIP_DOWNLOADER, EnterpriseGeoIpTaskState::new), + new NamedWriteableRegistry.Entry(PersistentTaskParams.class, ENTERPRISE_GEOIP_DOWNLOADER, EnterpriseGeoIpTaskParams::new), new NamedWriteableRegistry.Entry(Task.Status.class, GEOIP_DOWNLOADER, GeoIpDownloaderStats::new) ); } @@ -235,4 +291,9 @@ private static XContentBuilder mappings() { throw new UncheckedIOException("Failed to build mappings for " + DATABASES_INDEX, e); } } + + @Override + public void reload(Settings settings) { + enterpriseGeoIpDownloaderTaskExecutor.reload(settings); + } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java new file mode 100644 index 0000000000000..0a43d7a2d830b --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java @@ -0,0 +1,209 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * A database configuration is an identified (has an id) configuration of a named geoip location database to download, + * and the identifying information/configuration to download the named database from some database provider. + *

+ * That is, it has an id e.g. "my_db_config_1" and it says "download the file named XXXX from SomeCompany, and here's the + * magic token to use to do that." + */ +public record DatabaseConfiguration(String id, String name, Maxmind maxmind) implements Writeable, ToXContentObject { + + // id is a user selected signifier like 'my_domain_db' + // name is the name of a file that can be downloaded (like 'GeoIP2-Domain') + + // a configuration will have a 'type' like "maxmind", and that might have some more details, + // for now, though the important thing is that the json has to have it even though we don't model it meaningfully in this class + + public DatabaseConfiguration { + // these are invariants, not actual validation + Objects.requireNonNull(id); + Objects.requireNonNull(name); + Objects.requireNonNull(maxmind); + } + + /** + * An alphanumeric, followed by 0-126 alphanumerics, dashes, or underscores. That is, 1-127 alphanumerics, dashes, or underscores, + * but a leading dash or underscore isn't allowed (we're reserving leading dashes and underscores [and other odd characters] for + * Elastic and the future). + */ + private static final Pattern ID_PATTERN = Pattern.compile("\\p{Alnum}[_\\-\\p{Alnum}]{0,126}"); + + public static final Set MAXMIND_NAMES = Set.of( + "GeoIP2-Anonymous-IP", + "GeoIP2-City", + "GeoIP2-Connection-Type", + "GeoIP2-Country", + "GeoIP2-Domain", + "GeoIP2-Enterprise", + "GeoIP2-ISP" + + // in order to prevent a conflict between the (ordinary) geoip downloader and the enterprise geoip downloader, + // the enterprise geoip downloader is limited only to downloading the commercial files that the (ordinary) geoip downloader + // doesn't support out of the box -- in the future if we would like to relax this constraint, then we'll need to resolve that + // conflict at the same time. + + // "GeoLite2-ASN", + // "GeoLite2-City", + // "GeoLite2-Country" + ); + + private static final ParseField NAME = new ParseField("name"); + private static final ParseField MAXMIND = new ParseField("maxmind"); + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "database", + false, + (a, id) -> { + String name = (String) a[0]; + Maxmind maxmind = (Maxmind) a[1]; + return new DatabaseConfiguration(id, name, maxmind); + } + ); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME); + PARSER.declareObject(ConstructingObjectParser.constructorArg(), (parser, id) -> Maxmind.PARSER.apply(parser, null), MAXMIND); + } + + public DatabaseConfiguration(StreamInput in) throws IOException { + this(in.readString(), in.readString(), new Maxmind(in)); + } + + public static DatabaseConfiguration parse(XContentParser parser, String id) { + return PARSER.apply(parser, id); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeString(name); + maxmind.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("name", name); + builder.field("maxmind", maxmind); + builder.endObject(); + return builder; + } + + /** + * An id is intended to be alphanumerics, dashes, and underscores (only), but we're reserving leading dashes and underscores for + * ourselves in the future, that is, they're not for the ones that users can PUT. + */ + static void validateId(String id) throws IllegalArgumentException { + if (Strings.isNullOrEmpty(id)) { + throw new IllegalArgumentException("invalid database configuration id [" + id + "]: must not be null or empty"); + } + MetadataCreateIndexService.validateIndexOrAliasName( + id, + (id1, description) -> new IllegalArgumentException("invalid database configuration id [" + id1 + "]: " + description) + ); + int byteCount = id.getBytes(StandardCharsets.UTF_8).length; + if (byteCount > 127) { + throw new IllegalArgumentException( + "invalid database configuration id [" + id + "]: id is too long, (" + byteCount + " > " + 127 + ")" + ); + } + if (ID_PATTERN.matcher(id).matches() == false) { + throw new IllegalArgumentException( + "invalid database configuration id [" + + id + + "]: id doesn't match required rules (alphanumerics, dashes, and underscores, only)" + ); + } + } + + public ActionRequestValidationException validate() { + ActionRequestValidationException err = new ActionRequestValidationException(); + + // how do we cross the id validation divide here? or do we? it seems unfortunate to not invoke it at all. + + // name validation + if (Strings.hasText(name) == false) { + err.addValidationError("invalid name [" + name + "]: cannot be empty"); + } + + if (MAXMIND_NAMES.contains(name) == false) { + err.addValidationError("invalid name [" + name + "]: must be a supported name ([" + MAXMIND_NAMES + "])"); + } + + // important: the name must be unique across all configurations of this same type, + // but we validate that in the cluster state update, not here. + try { + validateId(id); + } catch (IllegalArgumentException e) { + err.addValidationError(e.getMessage()); + } + return err.validationErrors().isEmpty() ? null : err; + } + + public record Maxmind(String accountId) implements Writeable, ToXContentObject { + + public Maxmind { + // this is an invariant, not actual validation + Objects.requireNonNull(accountId); + } + + private static final ParseField ACCOUNT_ID = new ParseField("account_id"); + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("database", false, (a, id) -> { + String accountId = (String) a[0]; + return new Maxmind(accountId); + }); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), ACCOUNT_ID); + } + + public Maxmind(StreamInput in) throws IOException { + this(in.readString()); + } + + public static Maxmind parse(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(accountId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("account_id", accountId); + builder.endObject(); + return builder; + } + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadata.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadata.java new file mode 100644 index 0000000000000..574f97e4c5e64 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadata.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.cluster.SimpleDiffable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; + +/** + * {@code DatabaseConfigurationMetadata} encapsulates a {@link DatabaseConfiguration} as well as + * the additional meta information like version (a monotonically incrementing number), and last modified date. + */ +public record DatabaseConfigurationMetadata(DatabaseConfiguration database, long version, long modifiedDate) + implements + SimpleDiffable, + ToXContentObject { + + public static final ParseField DATABASE = new ParseField("database"); + public static final ParseField VERSION = new ParseField("version"); + public static final ParseField MODIFIED_DATE_MILLIS = new ParseField("modified_date_millis"); + public static final ParseField MODIFIED_DATE = new ParseField("modified_date"); + // later, things like this: + // static final ParseField LAST_SUCCESS = new ParseField("last_success"); + // static final ParseField LAST_FAILURE = new ParseField("last_failure"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "database_metadata", + true, + a -> { + DatabaseConfiguration database = (DatabaseConfiguration) a[0]; + return new DatabaseConfigurationMetadata(database, (long) a[1], (long) a[2]); + } + ); + static { + PARSER.declareObject(ConstructingObjectParser.constructorArg(), DatabaseConfiguration::parse, DATABASE); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), VERSION); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), MODIFIED_DATE_MILLIS); + } + + public static DatabaseConfigurationMetadata parse(XContentParser parser, String name) { + return PARSER.apply(parser, name); + } + + public DatabaseConfigurationMetadata(StreamInput in) throws IOException { + this(new DatabaseConfiguration(in), in.readVLong(), in.readVLong()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + // this is cluster state serialization, the id is implicit and doesn't need to included here + // (we'll be a in a json map where the id is the key) + builder.startObject(); + builder.field(VERSION.getPreferredName(), version); + builder.timeField(MODIFIED_DATE_MILLIS.getPreferredName(), MODIFIED_DATE.getPreferredName(), modifiedDate); + builder.field(DATABASE.getPreferredName(), database); + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + database.writeTo(out); + out.writeVLong(version); + out.writeVLong(modifiedDate); + } + + public static Diff readDiffFrom(StreamInput in) throws IOException { + return SimpleDiffable.readDiffFrom(DatabaseConfigurationMetadata::new, in); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DeleteDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DeleteDatabaseConfigurationAction.java new file mode 100644 index 0000000000000..843cc986c47e7 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DeleteDatabaseConfigurationAction.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; + +import java.io.IOException; +import java.util.Objects; + +public class DeleteDatabaseConfigurationAction extends ActionType { + public static final DeleteDatabaseConfigurationAction INSTANCE = new DeleteDatabaseConfigurationAction(); + public static final String NAME = "cluster:admin/ingest/geoip/database/delete"; + + protected DeleteDatabaseConfigurationAction() { + super(NAME); + } + + public static class Request extends AcknowledgedRequest { + + private final String databaseId; + + public Request(StreamInput in) throws IOException { + super(in); + databaseId = in.readString(); + } + + public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String databaseId) { + super(masterNodeTimeout, ackTimeout); + this.databaseId = Objects.requireNonNull(databaseId, "id may not be null"); + } + + public String getDatabaseId() { + return this.databaseId; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(databaseId); + } + + @Override + public int hashCode() { + return databaseId.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(databaseId, other.databaseId); + } + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java new file mode 100644 index 0000000000000..546c0c2df821d --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationMetadata.DATABASE; +import static org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationMetadata.MODIFIED_DATE; +import static org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationMetadata.MODIFIED_DATE_MILLIS; +import static org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationMetadata.VERSION; + +public class GetDatabaseConfigurationAction extends ActionType { + public static final GetDatabaseConfigurationAction INSTANCE = new GetDatabaseConfigurationAction(); + public static final String NAME = "cluster:admin/ingest/geoip/database/get"; + + protected GetDatabaseConfigurationAction() { + super(NAME); + } + + public static class Request extends AcknowledgedRequest { + + private final String[] databaseIds; + + public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String... databaseIds) { + super(masterNodeTimeout, ackTimeout); + this.databaseIds = Objects.requireNonNull(databaseIds, "ids may not be null"); + } + + public Request(StreamInput in) throws IOException { + super(in); + databaseIds = in.readStringArray(); + } + + public String[] getDatabaseIds() { + return this.databaseIds; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeStringArray(databaseIds); + } + + @Override + public int hashCode() { + return Arrays.hashCode(databaseIds); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + Request other = (Request) obj; + return Arrays.equals(databaseIds, other.databaseIds); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private final List databases; + + public Response(List databases) { + this.databases = List.copyOf(databases); // defensive copy + } + + public Response(StreamInput in) throws IOException { + this(in.readCollectionAsList(DatabaseConfigurationMetadata::new)); + } + + public List getDatabases() { + return this.databases; + } + + @Override + public String toString() { + return Strings.toString(this); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.startArray("databases"); + for (DatabaseConfigurationMetadata item : databases) { + DatabaseConfiguration database = item.database(); + builder.startObject(); + builder.field("id", database.id()); // serialize including the id -- this is get response serialization + builder.field(VERSION.getPreferredName(), item.version()); + builder.timeField(MODIFIED_DATE_MILLIS.getPreferredName(), MODIFIED_DATE.getPreferredName(), item.modifiedDate()); + builder.field(DATABASE.getPreferredName(), database); + builder.endObject(); + } + builder.endArray(); + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(databases); + } + + @Override + public int hashCode() { + return Objects.hash(databases); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + Response other = (Response) obj; + return databases.equals(other.databases); + } + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/PutDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/PutDatabaseConfigurationAction.java new file mode 100644 index 0000000000000..7bd5e1fa5cc68 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/PutDatabaseConfigurationAction.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +public class PutDatabaseConfigurationAction extends ActionType { + public static final PutDatabaseConfigurationAction INSTANCE = new PutDatabaseConfigurationAction(); + public static final String NAME = "cluster:admin/ingest/geoip/database/put"; + + protected PutDatabaseConfigurationAction() { + super(NAME); + } + + public static class Request extends AcknowledgedRequest { + + private final DatabaseConfiguration database; + + public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, DatabaseConfiguration database) { + super(masterNodeTimeout, ackTimeout); + this.database = database; + } + + public Request(StreamInput in) throws IOException { + super(in); + database = new DatabaseConfiguration(in); + } + + public DatabaseConfiguration getDatabase() { + return this.database; + } + + public static Request parseRequest(TimeValue masterNodeTimeout, TimeValue ackTimeout, String id, XContentParser parser) { + return new Request(masterNodeTimeout, ackTimeout, DatabaseConfiguration.parse(parser, id)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + database.writeTo(out); + } + + @Override + public ActionRequestValidationException validate() { + return database.validate(); + } + + @Override + public int hashCode() { + return Objects.hash(database); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + Request other = (Request) obj; + return database.equals(other.database); + } + + @Override + public String toString() { + return Strings.toString((b, p) -> b.field(database.id(), database)); + } + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java new file mode 100644 index 0000000000000..4dc263224ad0a --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.DELETE; +import static org.elasticsearch.rest.RestUtils.getAckTimeout; +import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; + +@ServerlessScope(Scope.INTERNAL) +public class RestDeleteDatabaseConfigurationAction extends BaseRestHandler { + + @Override + public List routes() { + return List.of(new Route(DELETE, "/_ingest/geoip/database/{id}")); + } + + @Override + public String getName() { + return "geoip_delete_database_configuration"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + final var req = new DeleteDatabaseConfigurationAction.Request( + getMasterNodeTimeout(request), + getAckTimeout(request), + request.param("id") + ); + return channel -> client.execute(DeleteDatabaseConfigurationAction.INSTANCE, req, new RestToXContentListener<>(channel)); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java new file mode 100644 index 0000000000000..b237ceb638918 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.rest.RestUtils.getAckTimeout; +import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; + +@ServerlessScope(Scope.INTERNAL) +public class RestGetDatabaseConfigurationAction extends BaseRestHandler { + + @Override + public List routes() { + return List.of(new Route(GET, "/_ingest/geoip/database"), new Route(GET, "/_ingest/geoip/database/{id}")); + } + + @Override + public String getName() { + return "geoip_get_database_configuration"; + } + + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) { + final var req = new GetDatabaseConfigurationAction.Request( + getMasterNodeTimeout(request), + getAckTimeout(request), + Strings.splitStringByCommaToArray(request.param("id")) + ); + return channel -> client.execute(GetDatabaseConfigurationAction.INSTANCE, req, new RestToXContentListener<>(channel)); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java new file mode 100644 index 0000000000000..62b01b930d5cd --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.ingest.geoip.direct.PutDatabaseConfigurationAction.Request; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.PUT; +import static org.elasticsearch.rest.RestUtils.getAckTimeout; +import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; + +@ServerlessScope(Scope.INTERNAL) +public class RestPutDatabaseConfigurationAction extends BaseRestHandler { + + @Override + public List routes() { + return List.of(new Route(PUT, "/_ingest/geoip/database/{id}")); + } + + @Override + public String getName() { + return "geoip_put_database_configuration"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + final Request req; + try (var parser = request.contentParser()) { + req = PutDatabaseConfigurationAction.Request.parseRequest( + getMasterNodeTimeout(request), + getAckTimeout(request), + request.param("id"), + parser + ); + } + return channel -> client.execute(PutDatabaseConfigurationAction.INSTANCE, req, new RestToXContentListener<>(channel)); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportDeleteDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportDeleteDatabaseConfigurationAction.java new file mode 100644 index 0000000000000..43aacee956279 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportDeleteDatabaseConfigurationAction.java @@ -0,0 +1,128 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateTaskListener; +import org.elasticsearch.cluster.SimpleBatchedExecutor; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.service.MasterServiceTaskQueue; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.Strings; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata; +import org.elasticsearch.ingest.geoip.direct.DeleteDatabaseConfigurationAction.Request; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.HashMap; +import java.util.Map; + +public class TransportDeleteDatabaseConfigurationAction extends TransportMasterNodeAction { + + private static final Logger logger = LogManager.getLogger(TransportDeleteDatabaseConfigurationAction.class); + + private static final SimpleBatchedExecutor DELETE_TASK_EXECUTOR = new SimpleBatchedExecutor<>() { + @Override + public Tuple executeTask(DeleteDatabaseConfigurationTask task, ClusterState clusterState) throws Exception { + return Tuple.tuple(task.execute(clusterState), null); + } + + @Override + public void taskSucceeded(DeleteDatabaseConfigurationTask task, Void unused) { + logger.trace("Updated cluster state for deletion of database configuration [{}]", task.databaseId); + task.listener.onResponse(AcknowledgedResponse.TRUE); + } + }; + + private final MasterServiceTaskQueue deleteDatabaseConfigurationTaskQueue; + + @Inject + public TransportDeleteDatabaseConfigurationAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver + ) { + super( + DeleteDatabaseConfigurationAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + Request::new, + indexNameExpressionResolver, + AcknowledgedResponse::readFrom, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.deleteDatabaseConfigurationTaskQueue = clusterService.createTaskQueue( + "delete-geoip-database-configuration-state-update", + Priority.NORMAL, + DELETE_TASK_EXECUTOR + ); + } + + @Override + protected void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) + throws Exception { + final String id = request.getDatabaseId(); + final IngestGeoIpMetadata geoIpMeta = state.metadata().custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY); + if (geoIpMeta.getDatabases().containsKey(id) == false) { + throw new ResourceNotFoundException("Database configuration not found: {}", id); + } + deleteDatabaseConfigurationTaskQueue.submitTask( + Strings.format("delete-geoip-database-configuration-[%s]", id), + new DeleteDatabaseConfigurationTask(listener, id), + null + ); + } + + private record DeleteDatabaseConfigurationTask(ActionListener listener, String databaseId) + implements + ClusterStateTaskListener { + + ClusterState execute(ClusterState currentState) throws Exception { + final IngestGeoIpMetadata geoIpMeta = currentState.metadata().custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY); + + logger.debug("deleting database configuration [{}]", databaseId); + Map databases = new HashMap<>(geoIpMeta.getDatabases()); + databases.remove(databaseId); + + Metadata currentMeta = currentState.metadata(); + return ClusterState.builder(currentState) + .metadata(Metadata.builder(currentMeta).putCustom(IngestGeoIpMetadata.TYPE, new IngestGeoIpMetadata(databases))) + .build(); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java new file mode 100644 index 0000000000000..a14a143e3f404 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class TransportGetDatabaseConfigurationAction extends TransportMasterNodeAction< + GetDatabaseConfigurationAction.Request, + GetDatabaseConfigurationAction.Response> { + + @Inject + public TransportGetDatabaseConfigurationAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver + ) { + super( + GetDatabaseConfigurationAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + GetDatabaseConfigurationAction.Request::new, + indexNameExpressionResolver, + GetDatabaseConfigurationAction.Response::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + } + + @Override + protected void masterOperation( + final Task task, + final GetDatabaseConfigurationAction.Request request, + final ClusterState state, + final ActionListener listener + ) { + final Set ids; + if (request.getDatabaseIds().length == 0) { + // if we did not ask for a specific name, then return all databases + ids = Set.of("*"); + } else { + ids = new LinkedHashSet<>(Arrays.asList(request.getDatabaseIds())); + } + + if (ids.size() > 1 && ids.stream().anyMatch(Regex::isSimpleMatchPattern)) { + throw new IllegalArgumentException( + "wildcard only supports a single value, please use comma-separated values or a single wildcard value" + ); + } + + final IngestGeoIpMetadata geoIpMeta = state.metadata().custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY); + List results = new ArrayList<>(); + + for (String id : ids) { + if (Regex.isSimpleMatchPattern(id)) { + for (Map.Entry entry : geoIpMeta.getDatabases().entrySet()) { + if (Regex.simpleMatch(id, entry.getKey())) { + results.add(entry.getValue()); + } + } + } else { + DatabaseConfigurationMetadata meta = geoIpMeta.getDatabases().get(id); + if (meta == null) { + listener.onFailure(new ResourceNotFoundException("database configuration not found: {}", id)); + return; + } else { + results.add(meta); + } + } + } + + listener.onResponse(new GetDatabaseConfigurationAction.Response(results)); + } + + @Override + protected ClusterBlockException checkBlock(GetDatabaseConfigurationAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java new file mode 100644 index 0000000000000..540be68671d38 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java @@ -0,0 +1,178 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateTaskListener; +import org.elasticsearch.cluster.SimpleBatchedExecutor; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.service.MasterServiceTaskQueue; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Strings; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata; +import org.elasticsearch.ingest.geoip.direct.PutDatabaseConfigurationAction.Request; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +public class TransportPutDatabaseConfigurationAction extends TransportMasterNodeAction { + + private static final Logger logger = LogManager.getLogger(TransportPutDatabaseConfigurationAction.class); + + private static final SimpleBatchedExecutor UPDATE_TASK_EXECUTOR = new SimpleBatchedExecutor<>() { + @Override + public Tuple executeTask(UpdateDatabaseConfigurationTask task, ClusterState clusterState) throws Exception { + return Tuple.tuple(task.execute(clusterState), null); + } + + @Override + public void taskSucceeded(UpdateDatabaseConfigurationTask task, Void unused) { + logger.trace("Updated cluster state for creation-or-update of database configuration [{}]", task.database.id()); + task.listener.onResponse(AcknowledgedResponse.TRUE); + } + }; + + private final MasterServiceTaskQueue updateDatabaseConfigurationTaskQueue; + + @Inject + public TransportPutDatabaseConfigurationAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver + ) { + super( + PutDatabaseConfigurationAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + Request::new, + indexNameExpressionResolver, + AcknowledgedResponse::readFrom, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.updateDatabaseConfigurationTaskQueue = clusterService.createTaskQueue( + "update-geoip-database-configuration-state-update", + Priority.NORMAL, + UPDATE_TASK_EXECUTOR + ); + } + + @Override + protected void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) { + final String id = request.getDatabase().id(); + updateDatabaseConfigurationTaskQueue.submitTask( + Strings.format("update-geoip-database-configuration-[%s]", id), + new UpdateDatabaseConfigurationTask(listener, request.getDatabase()), + null + ); + } + + /** + * Returns 'true' if the database configuration is effectually the same, and thus can be a no-op update. + */ + static boolean isNoopUpdate(@Nullable DatabaseConfigurationMetadata existingDatabase, DatabaseConfiguration newDatabase) { + if (existingDatabase == null) { + return false; + } else { + return newDatabase.equals(existingDatabase.database()); + } + } + + static void validatePrerequisites(DatabaseConfiguration database, ClusterState state) { + // we need to verify that the database represents a unique file (name) among the various databases for this same provider + IngestGeoIpMetadata geoIpMeta = state.metadata().custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY); + + Optional sameName = geoIpMeta.getDatabases() + .values() + .stream() + .map(DatabaseConfigurationMetadata::database) + // .filter(d -> d.type().equals(database.type())) // of the same type (right now the type is always just 'maxmind') + .filter(d -> d.id().equals(database.id()) == false) // and a different id + .filter(d -> d.name().equals(database.name())) // but has the same name! + .findFirst(); + + sameName.ifPresent(d -> { + throw new IllegalArgumentException( + Strings.format("database [%s] is already being downloaded via configuration [%s]", database.name(), d.id()) + ); + }); + } + + private record UpdateDatabaseConfigurationTask(ActionListener listener, DatabaseConfiguration database) + implements + ClusterStateTaskListener { + + ClusterState execute(ClusterState currentState) throws Exception { + IngestGeoIpMetadata geoIpMeta = currentState.metadata().custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY); + + String id = database.id(); + final DatabaseConfigurationMetadata existingDatabase = geoIpMeta.getDatabases().get(id); + // double-check for no-op in the state update task, in case it was changed/reset in the meantime + if (isNoopUpdate(existingDatabase, database)) { + return currentState; + } + + validatePrerequisites(database, currentState); + + Map databases = new HashMap<>(geoIpMeta.getDatabases()); + databases.put( + id, + new DatabaseConfigurationMetadata( + database, + existingDatabase == null ? 1 : existingDatabase.version() + 1, + Instant.now().toEpochMilli() + ) + ); + geoIpMeta = new IngestGeoIpMetadata(databases); + + if (existingDatabase == null) { + logger.debug("adding new database configuration [{}]", id); + } else { + logger.debug("updating existing database configuration [{}]", id); + } + + Metadata currentMeta = currentState.metadata(); + return ClusterState.builder(currentState) + .metadata(Metadata.builder(currentMeta).putCustom(IngestGeoIpMetadata.TYPE, geoIpMeta)) + .build(); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java index c7a1337f20e59..81836cda29568 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java @@ -166,7 +166,7 @@ public static class NodeResponse extends BaseNodeResponse { protected NodeResponse(StreamInput in) throws IOException { super(in); downloaderStats = in.readBoolean() ? new GeoIpDownloaderStats(in) : null; - if (in.getTransportVersion().onOrAfter(TransportVersions.GEOIP_CACHE_STATS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { cacheStats = in.readBoolean() ? new CacheStats(in) : null; } else { cacheStats = null; @@ -217,7 +217,7 @@ public void writeTo(StreamOutput out) throws IOException { if (downloaderStats != null) { downloaderStats.writeTo(out); } - if (out.getTransportVersion().onOrAfter(TransportVersions.GEOIP_CACHE_STATS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeBoolean(cacheStats != null); if (cacheStats != null) { cacheStats.writeTo(out); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java new file mode 100644 index 0000000000000..58cb566165db2 --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java @@ -0,0 +1,538 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.DocWriteRequest.OpType; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.admin.indices.flush.FlushAction; +import org.elasticsearch.action.admin.indices.flush.FlushRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.index.TransportIndexAction; +import org.elasticsearch.action.support.broadcast.BroadcastResponse; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.hash.MessageDigests; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.ingest.EnterpriseGeoIpTask; +import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration; +import org.elasticsearch.node.Node; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; +import org.elasticsearch.persistent.PersistentTasksService; +import org.elasticsearch.telemetry.metric.MeterRegistry; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentType; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.PasswordAuthentication; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; + +import static org.elasticsearch.ingest.geoip.DatabaseNodeServiceTests.createClusterState; +import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloader.MAX_CHUNK_SIZE; +import static org.elasticsearch.tasks.TaskId.EMPTY_TASK_ID; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +public class EnterpriseGeoIpDownloaderTests extends ESTestCase { + + private HttpClient httpClient; + private ClusterService clusterService; + private ThreadPool threadPool; + private MockClient client; + private EnterpriseGeoIpDownloader geoIpDownloader; + + @Before + public void setup() throws IOException { + httpClient = mock(HttpClient.class); + when(httpClient.getBytes(any(), anyString())).thenReturn( + "e4a3411cdd7b21eaf18675da5a7f9f360d33c6882363b2c19c38715834c9e836 GeoIP2-City_20240709.tar.gz".getBytes(StandardCharsets.UTF_8) + ); + clusterService = mock(ClusterService.class); + threadPool = new ThreadPool(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(), MeterRegistry.NOOP); + when(clusterService.getClusterSettings()).thenReturn( + new ClusterSettings(Settings.EMPTY, Set.of(GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING)) + ); + ClusterState state = createClusterState(new PersistentTasksCustomMetadata(1L, Map.of())); + when(clusterService.state()).thenReturn(state); + client = new MockClient(threadPool); + geoIpDownloader = new EnterpriseGeoIpDownloader( + client, + httpClient, + clusterService, + threadPool, + 1, + "", + "", + "", + EMPTY_TASK_ID, + Map.of(), + () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY), + (input) -> new HttpClient.PasswordAuthenticationHolder("name", "password".toCharArray()) + ) { + { + EnterpriseGeoIpTask.EnterpriseGeoIpTaskParams geoIpTaskParams = mock(EnterpriseGeoIpTask.EnterpriseGeoIpTaskParams.class); + when(geoIpTaskParams.getWriteableName()).thenReturn(EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER); + init(new PersistentTasksService(clusterService, threadPool, client), null, null, 0); + } + }; + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdownNow(); + } + + public void testGetChunkEndOfStream() throws IOException { + byte[] chunk = EnterpriseGeoIpDownloader.getChunk(new InputStream() { + @Override + public int read() { + return -1; + } + }); + assertArrayEquals(new byte[0], chunk); + chunk = EnterpriseGeoIpDownloader.getChunk(new ByteArrayInputStream(new byte[0])); + assertArrayEquals(new byte[0], chunk); + } + + public void testGetChunkLessThanChunkSize() throws IOException { + ByteArrayInputStream is = new ByteArrayInputStream(new byte[] { 1, 2, 3, 4 }); + byte[] chunk = EnterpriseGeoIpDownloader.getChunk(is); + assertArrayEquals(new byte[] { 1, 2, 3, 4 }, chunk); + chunk = EnterpriseGeoIpDownloader.getChunk(is); + assertArrayEquals(new byte[0], chunk); + + } + + public void testGetChunkExactlyChunkSize() throws IOException { + byte[] bigArray = new byte[MAX_CHUNK_SIZE]; + for (int i = 0; i < MAX_CHUNK_SIZE; i++) { + bigArray[i] = (byte) i; + } + ByteArrayInputStream is = new ByteArrayInputStream(bigArray); + byte[] chunk = EnterpriseGeoIpDownloader.getChunk(is); + assertArrayEquals(bigArray, chunk); + chunk = EnterpriseGeoIpDownloader.getChunk(is); + assertArrayEquals(new byte[0], chunk); + } + + public void testGetChunkMoreThanChunkSize() throws IOException { + byte[] bigArray = new byte[MAX_CHUNK_SIZE * 2]; + for (int i = 0; i < MAX_CHUNK_SIZE * 2; i++) { + bigArray[i] = (byte) i; + } + byte[] smallArray = new byte[MAX_CHUNK_SIZE]; + System.arraycopy(bigArray, 0, smallArray, 0, MAX_CHUNK_SIZE); + ByteArrayInputStream is = new ByteArrayInputStream(bigArray); + byte[] chunk = EnterpriseGeoIpDownloader.getChunk(is); + assertArrayEquals(smallArray, chunk); + System.arraycopy(bigArray, MAX_CHUNK_SIZE, smallArray, 0, MAX_CHUNK_SIZE); + chunk = EnterpriseGeoIpDownloader.getChunk(is); + assertArrayEquals(smallArray, chunk); + chunk = EnterpriseGeoIpDownloader.getChunk(is); + assertArrayEquals(new byte[0], chunk); + } + + public void testGetChunkRethrowsIOException() { + expectThrows(IOException.class, () -> EnterpriseGeoIpDownloader.getChunk(new InputStream() { + @Override + public int read() throws IOException { + throw new IOException(); + } + })); + } + + public void testIndexChunksNoData() throws IOException { + client.addHandler(FlushAction.INSTANCE, (FlushRequest request, ActionListener flushResponseActionListener) -> { + assertArrayEquals(new String[] { EnterpriseGeoIpDownloader.DATABASES_INDEX }, request.indices()); + flushResponseActionListener.onResponse(mock(BroadcastResponse.class)); + }); + client.addHandler( + RefreshAction.INSTANCE, + (RefreshRequest request, ActionListener flushResponseActionListener) -> { + assertArrayEquals(new String[] { EnterpriseGeoIpDownloader.DATABASES_INDEX }, request.indices()); + flushResponseActionListener.onResponse(mock(BroadcastResponse.class)); + } + ); + + InputStream empty = new ByteArrayInputStream(new byte[0]); + assertEquals( + Tuple.tuple(0, "d41d8cd98f00b204e9800998ecf8427e"), + geoIpDownloader.indexChunks( + "test", + empty, + 0, + MessageDigests.sha256(), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + 0 + ) + ); + } + + public void testIndexChunksMd5Mismatch() { + client.addHandler(FlushAction.INSTANCE, (FlushRequest request, ActionListener flushResponseActionListener) -> { + assertArrayEquals(new String[] { EnterpriseGeoIpDownloader.DATABASES_INDEX }, request.indices()); + flushResponseActionListener.onResponse(mock(BroadcastResponse.class)); + }); + client.addHandler( + RefreshAction.INSTANCE, + (RefreshRequest request, ActionListener flushResponseActionListener) -> { + assertArrayEquals(new String[] { EnterpriseGeoIpDownloader.DATABASES_INDEX }, request.indices()); + flushResponseActionListener.onResponse(mock(BroadcastResponse.class)); + } + ); + + IOException exception = expectThrows( + IOException.class, + () -> geoIpDownloader.indexChunks("test", new ByteArrayInputStream(new byte[0]), 0, MessageDigests.sha256(), "123123", 0) + ); + assertEquals( + "checksum mismatch, expected [123123], actual [e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855]", + exception.getMessage() + ); + } + + public void testIndexChunks() throws IOException { + byte[] bigArray = new byte[MAX_CHUNK_SIZE + 20]; + for (int i = 0; i < MAX_CHUNK_SIZE + 20; i++) { + bigArray[i] = (byte) i; + } + byte[][] chunksData = new byte[2][]; + chunksData[0] = new byte[MAX_CHUNK_SIZE]; + System.arraycopy(bigArray, 0, chunksData[0], 0, MAX_CHUNK_SIZE); + chunksData[1] = new byte[20]; + System.arraycopy(bigArray, MAX_CHUNK_SIZE, chunksData[1], 0, 20); + + AtomicInteger chunkIndex = new AtomicInteger(); + + client.addHandler(TransportIndexAction.TYPE, (IndexRequest request, ActionListener listener) -> { + int chunk = chunkIndex.getAndIncrement(); + assertEquals(OpType.CREATE, request.opType()); + assertThat(request.id(), Matchers.startsWith("test_" + (chunk + 15) + "_")); + assertEquals(XContentType.SMILE, request.getContentType()); + Map source = request.sourceAsMap(); + assertEquals("test", source.get("name")); + assertArrayEquals(chunksData[chunk], (byte[]) source.get("data")); + assertEquals(chunk + 15, source.get("chunk")); + listener.onResponse(mock(IndexResponse.class)); + }); + client.addHandler(FlushAction.INSTANCE, (FlushRequest request, ActionListener flushResponseActionListener) -> { + assertArrayEquals(new String[] { EnterpriseGeoIpDownloader.DATABASES_INDEX }, request.indices()); + flushResponseActionListener.onResponse(mock(BroadcastResponse.class)); + }); + client.addHandler( + RefreshAction.INSTANCE, + (RefreshRequest request, ActionListener flushResponseActionListener) -> { + assertArrayEquals(new String[] { EnterpriseGeoIpDownloader.DATABASES_INDEX }, request.indices()); + flushResponseActionListener.onResponse(mock(BroadcastResponse.class)); + } + ); + + InputStream big = new ByteArrayInputStream(bigArray); + assertEquals( + Tuple.tuple(17, "a67563dfa8f3cba8b8cff61eb989a749"), + geoIpDownloader.indexChunks( + "test", + big, + 15, + MessageDigests.sha256(), + "f2304545f224ff9ffcc585cb0a993723f911e03beb552cc03937dd443e931eab", + 0 + ) + ); + + assertEquals(2, chunkIndex.get()); + } + + public void testProcessDatabaseNew() throws IOException { + ByteArrayInputStream bais = new ByteArrayInputStream(new byte[0]); + when(httpClient.get(any(), any())).thenReturn(bais); + AtomicBoolean indexedChunks = new AtomicBoolean(false); + geoIpDownloader = new EnterpriseGeoIpDownloader( + client, + httpClient, + clusterService, + threadPool, + 1, + "", + "", + "", + EMPTY_TASK_ID, + Map.of(), + () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY), + (input) -> new HttpClient.PasswordAuthenticationHolder("name", "password".toCharArray()) + ) { + @Override + protected void updateTimestamp(String name, GeoIpTaskState.Metadata metadata) { + fail(); + } + + @Override + Tuple indexChunks( + String name, + InputStream is, + int chunk, + MessageDigest digest, + String expectedMd5, + long start + ) { + assertSame(bais, is); + assertEquals(0, chunk); + indexedChunks.set(true); + return Tuple.tuple(11, expectedMd5); + } + + @Override + void updateTaskState() { + assertEquals(0, state.getDatabases().get("test.mmdb").firstChunk()); + assertEquals(10, state.getDatabases().get("test.mmdb").lastChunk()); + } + + @Override + void deleteOldChunks(String name, int firstChunk) { + assertEquals("test.mmdb", name); + assertEquals(0, firstChunk); + } + }; + + geoIpDownloader.setState(EnterpriseGeoIpTaskState.EMPTY); + PasswordAuthentication auth = new PasswordAuthentication("name", "password".toCharArray()); + String id = randomIdentifier(); + DatabaseConfiguration databaseConfiguration = new DatabaseConfiguration(id, "test", new DatabaseConfiguration.Maxmind("name")); + geoIpDownloader.processDatabase(auth, databaseConfiguration); + assertThat(indexedChunks.get(), equalTo(true)); + } + + public void testProcessDatabaseUpdate() throws IOException { + ByteArrayInputStream bais = new ByteArrayInputStream(new byte[0]); + when(httpClient.get(any(), any())).thenReturn(bais); + AtomicBoolean indexedChunks = new AtomicBoolean(false); + geoIpDownloader = new EnterpriseGeoIpDownloader( + client, + httpClient, + clusterService, + threadPool, + 1, + "", + "", + "", + EMPTY_TASK_ID, + Map.of(), + () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY), + (input) -> new HttpClient.PasswordAuthenticationHolder("name", "password".toCharArray()) + ) { + @Override + protected void updateTimestamp(String name, GeoIpTaskState.Metadata metadata) { + fail(); + } + + @Override + Tuple indexChunks( + String name, + InputStream is, + int chunk, + MessageDigest digest, + String expectedMd5, + long start + ) { + assertSame(bais, is); + assertEquals(9, chunk); + indexedChunks.set(true); + return Tuple.tuple(1, expectedMd5); + } + + @Override + void updateTaskState() { + assertEquals(9, state.getDatabases().get("test.mmdb").firstChunk()); + assertEquals(10, state.getDatabases().get("test.mmdb").lastChunk()); + } + + @Override + void deleteOldChunks(String name, int firstChunk) { + assertEquals("test.mmdb", name); + assertEquals(9, firstChunk); + } + }; + + geoIpDownloader.setState(EnterpriseGeoIpTaskState.EMPTY.put("test.mmdb", new GeoIpTaskState.Metadata(0, 5, 8, "0", 0))); + PasswordAuthentication auth = new PasswordAuthentication("name", "password".toCharArray()); + String id = randomIdentifier(); + DatabaseConfiguration databaseConfiguration = new DatabaseConfiguration(id, "test", new DatabaseConfiguration.Maxmind("name")); + geoIpDownloader.processDatabase(auth, databaseConfiguration); + assertThat(indexedChunks.get(), equalTo(true)); + } + + public void testProcessDatabaseSame() throws IOException { + GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata( + 0, + 4, + 10, + "1", + 0, + "e4a3411cdd7b21eaf18675da5a7f9f360d33c6882363b2c19c38715834c9e836" + ); + EnterpriseGeoIpTaskState taskState = EnterpriseGeoIpTaskState.EMPTY.put("test.mmdb", metadata); + ByteArrayInputStream bais = new ByteArrayInputStream(new byte[0]); + when(httpClient.get(any(), any())).thenReturn(bais); + + geoIpDownloader = new EnterpriseGeoIpDownloader( + client, + httpClient, + clusterService, + threadPool, + 1, + "", + "", + "", + EMPTY_TASK_ID, + Map.of(), + () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY), + (input) -> new HttpClient.PasswordAuthenticationHolder("name", "password".toCharArray()) + ) { + @Override + protected void updateTimestamp(String name, GeoIpTaskState.Metadata newMetadata) { + assertEquals(metadata, newMetadata); + assertEquals("test.mmdb", name); + } + + @Override + Tuple indexChunks( + String name, + InputStream is, + int chunk, + MessageDigest digest, + String expectedChecksum, + long start + ) { + fail(); + return Tuple.tuple(0, expectedChecksum); + } + + @Override + void updateTaskState() { + fail(); + } + + @Override + void deleteOldChunks(String name, int firstChunk) { + fail(); + } + }; + geoIpDownloader.setState(taskState); + PasswordAuthentication auth = new PasswordAuthentication("name", "password".toCharArray()); + String id = randomIdentifier(); + DatabaseConfiguration databaseConfiguration = new DatabaseConfiguration(id, "test", new DatabaseConfiguration.Maxmind("name")); + geoIpDownloader.processDatabase(auth, databaseConfiguration); + } + + public void testUpdateDatabasesWriteBlock() { + ClusterState state = createClusterState(new PersistentTasksCustomMetadata(1L, Map.of())); + var geoIpIndex = state.getMetadata().getIndicesLookup().get(EnterpriseGeoIpDownloader.DATABASES_INDEX).getWriteIndex().getName(); + state = ClusterState.builder(state) + .blocks(new ClusterBlocks.Builder().addIndexBlock(geoIpIndex, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK)) + .build(); + when(clusterService.state()).thenReturn(state); + var e = expectThrows(ClusterBlockException.class, () -> geoIpDownloader.updateDatabases()); + assertThat( + e.getMessage(), + equalTo( + "index [" + + geoIpIndex + + "] blocked by: [TOO_MANY_REQUESTS/12/disk usage exceeded flood-stage watermark, " + + "index has read-only-allow-delete block];" + ) + ); + verifyNoInteractions(httpClient); + } + + public void testUpdateDatabasesIndexNotReady() throws IOException { + ClusterState state = createClusterState(new PersistentTasksCustomMetadata(1L, Map.of()), true); + var geoIpIndex = state.getMetadata().getIndicesLookup().get(EnterpriseGeoIpDownloader.DATABASES_INDEX).getWriteIndex().getName(); + state = ClusterState.builder(state) + .blocks(new ClusterBlocks.Builder().addIndexBlock(geoIpIndex, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK)) + .build(); + when(clusterService.state()).thenReturn(state); + geoIpDownloader.updateDatabases(); + verifyNoInteractions(httpClient); + } + + private GeoIpTaskState.Metadata newGeoIpTaskStateMetadata(boolean expired) { + Instant lastChecked; + if (expired) { + lastChecked = Instant.now().minus(randomIntBetween(31, 100), ChronoUnit.DAYS); + } else { + lastChecked = Instant.now().minus(randomIntBetween(0, 29), ChronoUnit.DAYS); + } + return new GeoIpTaskState.Metadata(0, 0, 0, randomAlphaOfLength(20), lastChecked.toEpochMilli()); + } + + private static class MockClient extends NoOpClient { + + private final Map, BiConsumer>> handlers = new HashMap<>(); + + private MockClient(ThreadPool threadPool) { + super(threadPool); + } + + public void addHandler( + ActionType action, + BiConsumer> listener + ) { + handlers.put(action, listener); + } + + @SuppressWarnings("unchecked") + @Override + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + if (handlers.containsKey(action)) { + BiConsumer> biConsumer = (BiConsumer>) handlers.get( + action + ); + biConsumer.accept(request, listener); + } else { + throw new IllegalStateException("unexpected action called [" + action.name() + "]"); + } + } + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskStateSerializationTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskStateSerializationTests.java new file mode 100644 index 0000000000000..a136f90780989 --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskStateSerializationTests.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractXContentSerializingTestCase; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class EnterpriseGeoIpTaskStateSerializationTests extends AbstractXContentSerializingTestCase { + @Override + protected GeoIpTaskState doParseInstance(XContentParser parser) throws IOException { + return GeoIpTaskState.fromXContent(parser); + } + + @Override + protected Writeable.Reader instanceReader() { + return GeoIpTaskState::new; + } + + @Override + protected GeoIpTaskState createTestInstance() { + GeoIpTaskState state = GeoIpTaskState.EMPTY; + int databaseCount = randomInt(20); + for (int i = 0; i < databaseCount; i++) { + state = state.put(randomAlphaOfLengthBetween(5, 10), createRandomMetadata()); + } + return state; + } + + @Override + protected GeoIpTaskState mutateInstance(GeoIpTaskState instance) { + Map databases = new HashMap<>(instance.getDatabases()); + switch (between(0, 2)) { + case 0: + String databaseName = randomValueOtherThanMany(databases::containsKey, () -> randomAlphaOfLengthBetween(5, 10)); + databases.put(databaseName, createRandomMetadata()); + return new GeoIpTaskState(databases); + case 1: + if (databases.size() > 0) { + String randomDatabaseName = databases.keySet().iterator().next(); + databases.put(randomDatabaseName, createRandomMetadata()); + } else { + databases.put(randomAlphaOfLengthBetween(5, 10), createRandomMetadata()); + } + return new GeoIpTaskState(databases); + case 2: + if (databases.size() > 0) { + String randomDatabaseName = databases.keySet().iterator().next(); + databases.remove(randomDatabaseName); + } else { + databases.put(randomAlphaOfLengthBetween(5, 10), createRandomMetadata()); + } + return new GeoIpTaskState(databases); + default: + throw new AssertionError("failure, got illegal switch case"); + } + } + + private GeoIpTaskState.Metadata createRandomMetadata() { + return new GeoIpTaskState.Metadata(randomLong(), randomInt(), randomInt(), randomAlphaOfLength(32), randomLong()); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java index 6a83fe69473f7..06b2605bd6d41 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java @@ -426,6 +426,55 @@ void deleteOldChunks(String name, int firstChunk) { assertEquals(0, stats.getFailedDownloads()); } + public void testCleanDatabases() throws IOException { + ByteArrayInputStream bais = new ByteArrayInputStream(new byte[0]); + when(httpClient.get("http://a.b/t1")).thenReturn(bais); + + final AtomicInteger count = new AtomicInteger(0); + + geoIpDownloader = new GeoIpDownloader( + client, + httpClient, + clusterService, + threadPool, + Settings.EMPTY, + 1, + "", + "", + "", + EMPTY_TASK_ID, + Map.of(), + () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY), + () -> GeoIpDownloaderTaskExecutor.EAGER_DOWNLOAD_SETTING.getDefault(Settings.EMPTY), + () -> true + ) { + @Override + void updateDatabases() throws IOException { + // noop + } + + @Override + void deleteOldChunks(String name, int firstChunk) { + count.incrementAndGet(); + assertEquals("test.mmdb", name); + assertEquals(21, firstChunk); + } + + @Override + void updateTaskState() { + // noop + } + }; + + geoIpDownloader.setState(GeoIpTaskState.EMPTY.put("test.mmdb", new GeoIpTaskState.Metadata(10, 10, 20, "md5", 20))); + geoIpDownloader.runDownloader(); + geoIpDownloader.runDownloader(); + GeoIpDownloaderStats stats = geoIpDownloader.getStatus(); + assertEquals(1, stats.getExpiredDatabases()); + assertEquals(2, count.get()); // somewhat surprising, not necessarily wrong + assertEquals(18, geoIpDownloader.state.getDatabases().get("test.mmdb").lastCheck()); // highly surprising, seems wrong + } + @SuppressWarnings("unchecked") public void testUpdateTaskState() { geoIpDownloader = new GeoIpDownloader( diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadataTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadataTests.java new file mode 100644 index 0000000000000..eca23cb13cd3d --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadataTests.java @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration; +import org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationMetadata; +import org.elasticsearch.test.AbstractChunkedSerializingTestCase; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class IngestGeoIpMetadataTests extends AbstractChunkedSerializingTestCase { + @Override + protected IngestGeoIpMetadata doParseInstance(XContentParser parser) throws IOException { + return IngestGeoIpMetadata.fromXContent(parser); + } + + @Override + protected Writeable.Reader instanceReader() { + return IngestGeoIpMetadata::new; + } + + @Override + protected IngestGeoIpMetadata createTestInstance() { + return randomIngestGeoIpMetadata(); + } + + @Override + protected IngestGeoIpMetadata mutateInstance(IngestGeoIpMetadata instance) throws IOException { + Map databases = new HashMap<>(instance.getDatabases()); + switch (between(0, 2)) { + case 0 -> { + String databaseId = randomValueOtherThanMany(databases::containsKey, ESTestCase::randomIdentifier); + databases.put(databaseId, randomDatabaseConfigurationMetadata(databaseId)); + return new IngestGeoIpMetadata(databases); + } + case 1 -> { + if (databases.size() > 0) { + String randomDatabaseId = databases.keySet().iterator().next(); + databases.put(randomDatabaseId, randomDatabaseConfigurationMetadata(randomDatabaseId)); + } else { + String databaseId = randomIdentifier(); + databases.put(databaseId, randomDatabaseConfigurationMetadata(databaseId)); + } + return new IngestGeoIpMetadata(databases); + } + case 2 -> { + if (databases.size() > 0) { + String randomDatabaseId = databases.keySet().iterator().next(); + databases.remove(randomDatabaseId); + } else { + String databaseId = randomIdentifier(); + databases.put(databaseId, randomDatabaseConfigurationMetadata(databaseId)); + } + return new IngestGeoIpMetadata(databases); + } + default -> throw new AssertionError("failure, got illegal switch case"); + } + } + + private IngestGeoIpMetadata randomIngestGeoIpMetadata() { + Map databases = new HashMap<>(); + for (int i = 0; i < randomIntBetween(0, 20); i++) { + String databaseId = randomIdentifier(); + databases.put(databaseId, randomDatabaseConfigurationMetadata(databaseId)); + } + return new IngestGeoIpMetadata(databases); + } + + private DatabaseConfigurationMetadata randomDatabaseConfigurationMetadata(String id) { + return new DatabaseConfigurationMetadata( + randomDatabaseConfiguration(id), + randomNonNegativeLong(), + randomPositiveTimeValue().millis() + ); + } + + private DatabaseConfiguration randomDatabaseConfiguration(String id) { + return new DatabaseConfiguration(id, randomAlphaOfLength(10), new DatabaseConfiguration.Maxmind(randomAlphaOfLength(10))); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadataTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadataTests.java new file mode 100644 index 0000000000000..f035416d48068 --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadataTests.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractXContentSerializingTestCase; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; + +import static org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.MAXMIND_NAMES; +import static org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationTests.randomDatabaseConfiguration; + +public class DatabaseConfigurationMetadataTests extends AbstractXContentSerializingTestCase { + + private String id; + + @Override + protected DatabaseConfigurationMetadata doParseInstance(XContentParser parser) throws IOException { + return DatabaseConfigurationMetadata.parse(parser, id); + } + + @Override + protected DatabaseConfigurationMetadata createTestInstance() { + id = randomAlphaOfLength(5); + return randomDatabaseConfigurationMetadata(id); + } + + public static DatabaseConfigurationMetadata randomDatabaseConfigurationMetadata(String id) { + return new DatabaseConfigurationMetadata( + new DatabaseConfiguration(id, randomFrom(MAXMIND_NAMES), new DatabaseConfiguration.Maxmind(randomAlphaOfLength(5))), + randomNonNegativeLong(), + randomPositiveTimeValue().millis() + ); + } + + @Override + protected DatabaseConfigurationMetadata mutateInstance(DatabaseConfigurationMetadata instance) { + switch (between(0, 2)) { + case 0: + return new DatabaseConfigurationMetadata( + randomValueOtherThan(instance.database(), () -> randomDatabaseConfiguration(randomAlphaOfLength(5))), + instance.version(), + instance.modifiedDate() + ); + case 1: + return new DatabaseConfigurationMetadata( + instance.database(), + randomValueOtherThan(instance.version(), ESTestCase::randomNonNegativeLong), + instance.modifiedDate() + ); + case 2: + return new DatabaseConfigurationMetadata( + instance.database(), + instance.version(), + randomValueOtherThan(instance.modifiedDate(), () -> ESTestCase.randomPositiveTimeValue().millis()) + ); + default: + throw new AssertionError("failure, got illegal switch case"); + } + } + + @Override + protected Writeable.Reader instanceReader() { + return DatabaseConfigurationMetadata::new; + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java new file mode 100644 index 0000000000000..02c067561b49c --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Maxmind; +import org.elasticsearch.test.AbstractXContentSerializingTestCase; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Set; + +import static org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.MAXMIND_NAMES; + +public class DatabaseConfigurationTests extends AbstractXContentSerializingTestCase { + + private String id; + + @Override + protected DatabaseConfiguration doParseInstance(XContentParser parser) throws IOException { + return DatabaseConfiguration.parse(parser, id); + } + + @Override + protected DatabaseConfiguration createTestInstance() { + id = randomAlphaOfLength(5); + return randomDatabaseConfiguration(id); + } + + public static DatabaseConfiguration randomDatabaseConfiguration(String id) { + return new DatabaseConfiguration(id, randomFrom(MAXMIND_NAMES), new Maxmind(randomAlphaOfLength(5))); + } + + @Override + protected DatabaseConfiguration mutateInstance(DatabaseConfiguration instance) { + switch (between(0, 2)) { + case 0: + return new DatabaseConfiguration(instance.id() + randomAlphaOfLength(2), instance.name(), instance.maxmind()); + case 1: + return new DatabaseConfiguration( + instance.id(), + randomValueOtherThan(instance.name(), () -> randomFrom(MAXMIND_NAMES)), + instance.maxmind() + ); + case 2: + return new DatabaseConfiguration( + instance.id(), + instance.name(), + new Maxmind(instance.maxmind().accountId() + randomAlphaOfLength(2)) + ); + default: + throw new AssertionError("failure, got illegal switch case"); + } + } + + @Override + protected Writeable.Reader instanceReader() { + return DatabaseConfiguration::new; + } + + public void testValidateId() { + Set invalidIds = Set.of("-foo", "_foo", "foo,bar", "foo bar", "foo*bar", "foo.bar"); + for (String id : invalidIds) { + expectThrows(IllegalArgumentException.class, "expected exception for " + id, () -> DatabaseConfiguration.validateId(id)); + } + Set validIds = Set.of("f-oo", "f_oo", "foobar"); + for (String id : validIds) { + DatabaseConfiguration.validateId(id); + } + // Note: the code checks for byte length, but randomAlphoOfLength is only using characters in the ascii subset + String longId = randomAlphaOfLength(128); + expectThrows(IllegalArgumentException.class, "expected exception for " + longId, () -> DatabaseConfiguration.validateId(longId)); + String longestAllowedId = randomAlphaOfLength(127); + DatabaseConfiguration.validateId(longestAllowedId); + String shortId = randomAlphaOfLengthBetween(1, 127); + DatabaseConfiguration.validateId(shortId); + expectThrows(IllegalArgumentException.class, "expected exception for empty string", () -> DatabaseConfiguration.validateId("")); + expectThrows(IllegalArgumentException.class, "expected exception for null string", () -> DatabaseConfiguration.validateId(null)); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationActionTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationActionTests.java new file mode 100644 index 0000000000000..710c3ee23916d --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationActionTests.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata; +import org.elasticsearch.test.ESTestCase; + +import java.util.HashMap; +import java.util.Map; + +public class TransportPutDatabaseConfigurationActionTests extends ESTestCase { + + public void testValidatePrerequisites() { + // Test that we reject two configurations with the same database name but different ids: + String name = randomAlphaOfLengthBetween(1, 50); + IngestGeoIpMetadata ingestGeoIpMetadata = randomIngestGeoIpMetadata(name); + ClusterState state = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder(Metadata.EMPTY_METADATA).putCustom(IngestGeoIpMetadata.TYPE, ingestGeoIpMetadata)) + .build(); + DatabaseConfiguration databaseConfiguration = randomDatabaseConfiguration(randomIdentifier(), name); + expectThrows( + IllegalArgumentException.class, + () -> TransportPutDatabaseConfigurationAction.validatePrerequisites(databaseConfiguration, state) + ); + + // Test that we do not reject two configurations with different database names: + String differentName = randomValueOtherThan(name, () -> randomAlphaOfLengthBetween(1, 50)); + DatabaseConfiguration databaseConfigurationForDifferentName = randomDatabaseConfiguration(randomIdentifier(), differentName); + TransportPutDatabaseConfigurationAction.validatePrerequisites(databaseConfigurationForDifferentName, state); + + // Test that we do not reject a configuration if none already exists: + TransportPutDatabaseConfigurationAction.validatePrerequisites(databaseConfiguration, ClusterState.EMPTY_STATE); + + // Test that we do not reject a configuration if one with the same database name AND id already exists: + DatabaseConfiguration databaseConfigurationSameNameSameId = ingestGeoIpMetadata.getDatabases() + .values() + .iterator() + .next() + .database(); + TransportPutDatabaseConfigurationAction.validatePrerequisites(databaseConfigurationSameNameSameId, state); + } + + private IngestGeoIpMetadata randomIngestGeoIpMetadata(String name) { + Map databases = new HashMap<>(); + String databaseId = randomIdentifier(); + databases.put(databaseId, randomDatabaseConfigurationMetadata(databaseId, name)); + return new IngestGeoIpMetadata(databases); + } + + private DatabaseConfigurationMetadata randomDatabaseConfigurationMetadata(String id, String name) { + return new DatabaseConfigurationMetadata( + randomDatabaseConfiguration(id, name), + randomNonNegativeLong(), + randomPositiveTimeValue().millis() + ); + } + + private DatabaseConfiguration randomDatabaseConfiguration(String id, String name) { + return new DatabaseConfiguration(id, name, new DatabaseConfiguration.Maxmind(randomAlphaOfLength(10))); + } +} diff --git a/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java index 58a6e3771b30d..0f0a0c998bd75 100644 --- a/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java +++ b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java @@ -46,7 +46,12 @@ public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase .module("reindex") .module("ingest-geoip") .systemProperty("ingest.geoip.downloader.enabled.default", "true") + // sets the plain (geoip.elastic.co) downloader endpoint, which is used in these tests .setting("ingest.geoip.downloader.endpoint", () -> fixture.getAddress(), s -> useFixture) + // also sets the enterprise downloader maxmind endpoint, to make sure we do not accidentally hit the real endpoint from tests + // note: it's not important that the downloading actually work at this point -- the rest tests (so far) don't exercise + // the downloading code because of license reasons -- but if they did, then it would be important that we're hitting a fixture + .systemProperty("ingest.geoip.downloader.maxmind.endpoint.default", () -> fixture.getAddress(), s -> useFixture) .build(); @ClassRule diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml new file mode 100644 index 0000000000000..6809443fdfbc3 --- /dev/null +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml @@ -0,0 +1,72 @@ +setup: + - requires: + cluster_features: ["geoip.downloader.database.configuration"] + reason: "geoip downloader database configuration APIs added in 8.15" + +--- +"Test adding, getting, and removing geoip databases": + - do: + ingest.put_geoip_database: + id: "my_database_1" + body: > + { + "name": "GeoIP2-City", + "maxmind": { + "account_id": "1234" + } + } + - match: { acknowledged: true } + + - do: + ingest.put_geoip_database: + id: "my_database_1" + body: > + { + "name": "GeoIP2-Country", + "maxmind": { + "account_id": "4321" + } + } + - match: { acknowledged: true } + + - do: + ingest.put_geoip_database: + id: "my_database_2" + body: > + { + "name": "GeoIP2-City", + "maxmind": { + "account_id": "1234" + } + } + - match: { acknowledged: true } + + - do: + ingest.get_geoip_database: + id: "my_database_1" + - length: { databases: 1 } + - match: { databases.0.id: "my_database_1" } + - gte: { databases.0.modified_date_millis: 0 } + - match: { databases.0.database.name: "GeoIP2-Country" } + - match: { databases.0.database.maxmind.account_id: "4321" } + + - do: + ingest.get_geoip_database: {} + - length: { databases: 2 } + + - do: + ingest.get_geoip_database: + id: "my_database_1,my_database_2" + - length: { databases: 2 } + + - do: + ingest.delete_geoip_database: + id: "my_database_1" + + - do: + ingest.get_geoip_database: {} + - length: { databases: 1 } + - match: { databases.0.id: "my_database_2" } + - gte: { databases.0.modified_date_millis: 0 } + - match: { databases.0.database.name: "GeoIP2-City" } + - match: { databases.0.database.maxmind.account_id: "1234" } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index 9a2653a61b60d..27d8695f9ae6a 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -655,6 +655,7 @@ public > IFD getForField( IndexFieldData.Builder builder = fieldType.fielddataBuilder( new FieldDataContext( delegate.getFullyQualifiedIndex().getName(), + delegate.getIndexSettings(), delegate::lookup, this::sourcePath, fielddataOperation diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java index c40a4f72bc133..47505919ba7d2 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java @@ -49,6 +49,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.TimeValue; @@ -992,7 +993,7 @@ private static class DummyTransportAsyncBulkByScrollAction extends TransportActi BulkByScrollResponse> { protected DummyTransportAsyncBulkByScrollAction(String actionName, ActionFilters actionFilters, TaskManager taskManager) { - super(actionName, actionFilters, taskManager); + super(actionName, actionFilters, taskManager, EsExecutors.DIRECT_EXECUTOR_SERVICE); } @Override diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java index b3558d4930ba3..d628a013a37bb 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.search.TransportSearchScrollAction; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.ParentTaskAssigningClient; import org.elasticsearch.client.internal.support.AbstractClient; import org.elasticsearch.common.bytes.BytesArray; @@ -42,12 +41,14 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.IntStream; import static org.apache.lucene.tests.util.TestUtil.randomSimpleString; import static org.elasticsearch.core.TimeValue.timeValueSeconds; +import static org.hamcrest.Matchers.instanceOf; public class ClientScrollableHitSourceTests extends ESTestCase { @@ -73,12 +74,11 @@ public void testRetrySuccess() throws InterruptedException { dotestBasicsWithRetry(retries, 0, retries, e -> fail()); } - public void testRetryFail() { - int retries = randomInt(10); - expectThrows( - EsRejectedExecutionException.class, - () -> PlainActionFuture.get(f -> dotestBasicsWithRetry(retries, retries + 1, retries + 1, f::onFailure), 0, TimeUnit.SECONDS) - ); + public void testRetryFail() throws InterruptedException { + final int retries = randomInt(10); + final var exceptionRef = new AtomicReference(); + dotestBasicsWithRetry(retries, retries + 1, retries + 1, exceptionRef::set); + assertThat(exceptionRef.get(), instanceOf(EsRejectedExecutionException.class)); } private void dotestBasicsWithRetry(int retries, int minFailures, int maxFailures, Consumer failureHandler) diff --git a/modules/repository-azure/build.gradle b/modules/repository-azure/build.gradle index d093816acd45f..5e1c294c1a30e 100644 --- a/modules/repository-azure/build.gradle +++ b/modules/repository-azure/build.gradle @@ -17,65 +17,51 @@ esplugin { } versions << [ - 'azure': '12.20.1', - 'azureCommon': '12.19.1', - 'azureCore': '1.34.0', - 'azureCoreHttpNetty': '1.12.7', - 'azureJackson': '2.15.4', - 'azureJacksonDatabind': '2.13.4.2', - 'azureAvro': '12.5.3', - - 'jakartaActivation': '1.2.1', - 'jakartaXMLBind': '2.3.2', - 'stax2API': '4.2.1', - 'woodstox': '6.4.0', - - 'reactorNetty': '1.0.39', - 'reactorCore': '3.4.34', - 'reactiveStreams': '1.0.4', + 'azureReactorNetty': '1.0.45', ] dependencies { - api "com.azure:azure-storage-blob:${versions.azure}" - api "com.azure:azure-storage-common:${versions.azureCommon}" - api "com.azure:azure-core-http-netty:${versions.azureCoreHttpNetty}" - api "com.azure:azure-core:${versions.azureCore}" - // jackson - api "com.fasterxml.jackson.core:jackson-core:${versions.azureJackson}" - api "com.fasterxml.jackson.core:jackson-databind:${versions.azureJacksonDatabind}" - api "com.fasterxml.jackson.core:jackson-annotations:${versions.azureJackson}" - - // jackson xml - api "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${versions.azureJackson}" - api "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.azureJackson}" - api "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:${versions.azureJackson}" - api "jakarta.activation:jakarta.activation-api:${versions.jakartaActivation}" - // The SDK uses javax.xml bindings - api "jakarta.xml.bind:jakarta.xml.bind-api:${versions.jakartaXMLBind}" - api "org.codehaus.woodstox:stax2-api:${versions.stax2API}" - api "com.fasterxml.woodstox:woodstox-core:${versions.woodstox}" - - // netty + // Microsoft + api "com.azure:azure-core-http-netty:1.15.1" + api "com.azure:azure-core:1.50.0" + api "com.azure:azure-json:1.1.0" + api "com.azure:azure-storage-blob:12.26.1" + api "com.azure:azure-storage-common:12.26.0" + api "com.azure:azure-storage-internal-avro:12.11.1" + api "com.azure:azure-xml:1.0.0" + + // Jackson + api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" + api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" + api "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${versions.jackson}" + api "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.jackson}" + api "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:${versions.jackson}" + + // Netty api "io.netty:netty-codec-dns:${versions.netty}" api "io.netty:netty-codec-http2:${versions.netty}" api "io.netty:netty-codec-socks:${versions.netty}" api "io.netty:netty-handler-proxy:${versions.netty}" api "io.netty:netty-resolver-dns:${versions.netty}" - // reactor - api "io.projectreactor.netty:reactor-netty-core:${versions.reactorNetty}" - api "io.projectreactor.netty:reactor-netty-http:${versions.reactorNetty}" - api "io.projectreactor:reactor-core:${versions.reactorCore}" - api "org.reactivestreams:reactive-streams:${versions.reactiveStreams}" + // Reactor + api "io.projectreactor.netty:reactor-netty-core:${versions.azureReactorNetty}" + api "io.projectreactor.netty:reactor-netty-http:${versions.azureReactorNetty}" + api "io.projectreactor:reactor-core:3.4.38" + api "org.reactivestreams:reactive-streams:1.0.4" + + // Others + api "com.fasterxml.woodstox:woodstox-core:6.4.0" + api "jakarta.activation:jakarta.activation-api:1.2.1" + api "jakarta.xml.bind:jakarta.xml.bind-api:2.3.3" + api "org.codehaus.woodstox:stax2-api:4.2.1" implementation project(":modules:transport-netty4") implementation("org.slf4j:slf4j-api:${versions.slf4j}") runtimeOnly "org.slf4j:slf4j-nop:${versions.slf4j}" // runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}") https://github.com/elastic/elasticsearch/issues/93714 - - runtimeOnly "com.azure:azure-storage-internal-avro:${versions.azureAvro}" - testImplementation project(':test:fixtures:azure-fixture') yamlRestTestImplementation project(':test:fixtures:azure-fixture') } diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java index e916b02e62b8e..15d47f6bec800 100644 --- a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java +++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java @@ -153,9 +153,8 @@ long getUploadBlockSize() { @SuppressForbidden(reason = "this test uses a HttpHandler to emulate an Azure endpoint") private static class AzureBlobStoreHttpHandler extends AzureHttpHandler implements BlobStoreHttpHandler { - AzureBlobStoreHttpHandler(final String account, final String container) { - super(account, container); + super(account, container, null /* no auth header validation - sometimes it's omitted in these tests (TODO why?) */); } } diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java index 2fc728a4fae34..1d7f8092e4939 100644 --- a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java +++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.rest.RestStatus; import org.junit.ClassRule; import java.io.ByteArrayInputStream; @@ -43,11 +44,14 @@ public class AzureStorageCleanupThirdPartyTests extends AbstractThirdPartyRepositoryTestCase { private static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("test.azure.fixture", "true")); + private static final String AZURE_ACCOUNT = System.getProperty("test.azure.account"); + @ClassRule public static AzureHttpFixture fixture = new AzureHttpFixture( - USE_FIXTURE, - System.getProperty("test.azure.account"), - System.getProperty("test.azure.container") + USE_FIXTURE ? AzureHttpFixture.Protocol.HTTP : AzureHttpFixture.Protocol.NONE, + AZURE_ACCOUNT, + System.getProperty("test.azure.container"), + AzureHttpFixture.sharedKeyForAccountPredicate(AZURE_ACCOUNT) ); @Override @@ -179,4 +183,11 @@ public void testMultiBlockUpload() throws Exception { })); future.get(); } + + public void testReadFromPositionLargerThanBlobLength() { + testReadFromPositionLargerThanBlobLength( + e -> asInstanceOf(BlobStorageException.class, e.getCause()).getStatusCode() == RestStatus.REQUESTED_RANGE_NOT_SATISFIED + .getStatus() + ); + } } diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java index 16a9f60a3d28d..d271c8d1e99a1 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java @@ -25,6 +25,8 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.Nullable; +import org.elasticsearch.repositories.blobstore.RequestedRangeNotSatisfiedException; +import org.elasticsearch.rest.RestStatus; import java.io.IOException; import java.io.InputStream; @@ -69,9 +71,12 @@ private InputStream openInputStream(OperationPurpose purpose, String blobName, l } catch (Exception e) { Throwable rootCause = Throwables.getRootCause(e); if (rootCause instanceof BlobStorageException blobStorageException) { - if (blobStorageException.getStatusCode() == 404) { + if (blobStorageException.getStatusCode() == RestStatus.NOT_FOUND.getStatus()) { throw new NoSuchFileException("Blob [" + blobKey + "] not found"); } + if (blobStorageException.getStatusCode() == RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus()) { + throw new RequestedRangeNotSatisfiedException(blobKey, position, length == null ? -1 : length, blobStorageException); + } } throw new IOException("Unable to get input stream for blob [" + blobKey + "]", e); } diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java index 73d969ee31b19..f6b1c2775926c 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java @@ -27,6 +27,8 @@ import org.elasticsearch.threadpool.ScalingExecutorBuilder; import org.elasticsearch.xcontent.NamedXContentRegistry; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -42,9 +44,8 @@ public class AzureRepositoryPlugin extends Plugin implements RepositoryPlugin, R public static final String NETTY_EVENT_LOOP_THREAD_POOL_NAME = "azure_event_loop"; static { - // Trigger static initialization with the plugin class loader - // so we have access to the proper xml parser - JacksonAdapter.createDefaultSerializerAdapter(); + // Trigger static initialization with the plugin class loader so we have access to the proper xml parser + AccessController.doPrivileged((PrivilegedAction) JacksonAdapter::createDefaultSerializerAdapter); } // protected for testing diff --git a/modules/repository-azure/src/yamlRestTest/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java b/modules/repository-azure/src/yamlRestTest/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java index b9d5705aff91e..c04b2bc6a6d7c 100644 --- a/modules/repository-azure/src/yamlRestTest/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java +++ b/modules/repository-azure/src/yamlRestTest/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java @@ -14,6 +14,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.core.Booleans; +import org.elasticsearch.test.TestTrustStore; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; @@ -28,7 +29,16 @@ public class RepositoryAzureClientYamlTestSuiteIT extends ESClientYamlSuiteTestC private static final String AZURE_TEST_KEY = System.getProperty("test.azure.key"); private static final String AZURE_TEST_SASTOKEN = System.getProperty("test.azure.sas_token"); - private static AzureHttpFixture fixture = new AzureHttpFixture(USE_FIXTURE, AZURE_TEST_ACCOUNT, AZURE_TEST_CONTAINER); + private static AzureHttpFixture fixture = new AzureHttpFixture( + USE_FIXTURE ? AzureHttpFixture.Protocol.HTTPS : AzureHttpFixture.Protocol.NONE, + AZURE_TEST_ACCOUNT, + AZURE_TEST_CONTAINER, + AzureHttpFixture.sharedKeyForAccountPredicate(AZURE_TEST_ACCOUNT) + ); + + private static TestTrustStore trustStore = new TestTrustStore( + () -> AzureHttpFixture.class.getResourceAsStream("azure-http-fixture.pem") + ); private static ElasticsearchCluster cluster = ElasticsearchCluster.local() .module("repository-azure") @@ -45,14 +55,15 @@ public class RepositoryAzureClientYamlTestSuiteIT extends ESClientYamlSuiteTestC ) .setting( "azure.client.integration_test.endpoint_suffix", - () -> "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=" + fixture.getAddress(), + () -> "ignored;DefaultEndpointsProtocol=https;BlobEndpoint=" + fixture.getAddress(), s -> USE_FIXTURE ) .setting("thread_pool.repository_azure.max", () -> String.valueOf(randomIntBetween(1, 10)), s -> USE_FIXTURE) + .systemProperty("javax.net.ssl.trustStore", () -> trustStore.getTrustStorePath().toString(), s -> USE_FIXTURE) .build(); @ClassRule - public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster); + public static TestRule ruleChain = RuleChain.outerRule(fixture).around(trustStore).around(cluster); public RepositoryAzureClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); diff --git a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index 6d2c015d7d922..e6625ea1b7f15 100644 --- a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -22,8 +22,6 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; -import org.elasticsearch.action.ActionRunnable; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.blobstore.BlobContainer; @@ -121,22 +119,13 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { return settings.build(); } - public void testDeleteSingleItem() { + public void testDeleteSingleItem() throws IOException { final String repoName = createRepository(randomRepositoryName()); final RepositoriesService repositoriesService = internalCluster().getAnyMasterNodeInstance(RepositoriesService.class); final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(repoName); - PlainActionFuture.get( - f -> repository.threadPool() - .generic() - .execute( - ActionRunnable.run( - f, - () -> repository.blobStore() - .blobContainer(repository.basePath()) - .deleteBlobsIgnoringIfNotExists(randomPurpose(), Iterators.single("foo")) - ) - ) - ); + repository.blobStore() + .blobContainer(repository.basePath()) + .deleteBlobsIgnoringIfNotExists(randomPurpose(), Iterators.single("foo")); } public void testChunkSize() { diff --git a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java index 4afa6f2a10b5c..3c13360029bce 100644 --- a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java +++ b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java @@ -11,6 +11,8 @@ import fixture.gcs.GoogleCloudStorageHttpFixture; import fixture.gcs.TestUtils; +import com.google.cloud.storage.StorageException; + import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.SecureSettings; @@ -18,6 +20,7 @@ import org.elasticsearch.core.Booleans; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase; +import org.elasticsearch.rest.RestStatus; import org.junit.ClassRule; import java.util.Base64; @@ -85,4 +88,10 @@ protected void createRepository(final String repoName) { .get(); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); } + + public void testReadFromPositionLargerThanBlobLength() { + testReadFromPositionLargerThanBlobLength( + e -> asInstanceOf(StorageException.class, e.getCause()).getCode() == RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus() + ); + } } diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRetryingInputStream.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRetryingInputStream.java index 025873878975a..5689123872cc3 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRetryingInputStream.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRetryingInputStream.java @@ -21,6 +21,8 @@ import org.elasticsearch.SpecialPermission; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.repositories.blobstore.RequestedRangeNotSatisfiedException; +import org.elasticsearch.rest.RestStatus; import java.io.FilterInputStream; import java.io.IOException; @@ -31,6 +33,7 @@ import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.List; +import java.util.function.Supplier; import java.util.stream.Stream; import static org.elasticsearch.core.Strings.format; @@ -68,6 +71,26 @@ class GoogleCloudStorageRetryingInputStream extends InputStream { // both start and end are inclusive bounds, following the definition in https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35 GoogleCloudStorageRetryingInputStream(Storage client, BlobId blobId, long start, long end) throws IOException { + this(client, () -> getStorage(client), blobId, start, end); + } + + // Used for testing only + GoogleCloudStorageRetryingInputStream( + com.google.cloud.storage.Storage client, + Supplier storage, + BlobId blobId + ) throws IOException { + this(client, storage, blobId, 0, Long.MAX_VALUE - 1); + } + + // Used for testing only + GoogleCloudStorageRetryingInputStream( + com.google.cloud.storage.Storage client, + Supplier storage, + BlobId blobId, + long start, + long end + ) throws IOException { if (start < 0L) { throw new IllegalArgumentException("start must be non-negative"); } @@ -80,8 +103,8 @@ class GoogleCloudStorageRetryingInputStream extends InputStream { this.end = end; this.maxAttempts = client.getOptions().getRetrySettings().getMaxAttempts(); SpecialPermission.check(); - storage = getStorage(client); - currentStream = openStream(); + this.storage = storage.get(); // to bypass static init for unit testing + this.currentStream = openStream(); } @SuppressForbidden(reason = "need access to storage client") @@ -109,7 +132,9 @@ private InputStream openStream() throws IOException { get.setReturnRawInputStream(true); if (currentOffset > 0 || start > 0 || end < Long.MAX_VALUE - 1) { - get.getRequestHeaders().setRange("bytes=" + Math.addExact(start, currentOffset) + "-" + end); + if (get.getRequestHeaders() != null) { + get.getRequestHeaders().setRange("bytes=" + Math.addExact(start, currentOffset) + "-" + end); + } } final HttpResponse resp = get.executeMedia(); final Long contentLength = resp.getHeaders().getContentLength(); @@ -126,13 +151,24 @@ private InputStream openStream() throws IOException { } catch (RetryHelper.RetryHelperException e) { throw StorageException.translateAndThrow(e); } - } catch (StorageException e) { - if (e.getCode() == 404) { + } catch (StorageException storageException) { + if (storageException.getCode() == RestStatus.NOT_FOUND.getStatus()) { throw addSuppressedExceptions( - new NoSuchFileException("Blob object [" + blobId.getName() + "] not found: " + e.getMessage()) + new NoSuchFileException("Blob object [" + blobId.getName() + "] not found: " + storageException.getMessage()) ); } - throw addSuppressedExceptions(e); + if (storageException.getCode() == RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus()) { + long currentPosition = Math.addExact(start, currentOffset); + throw addSuppressedExceptions( + new RequestedRangeNotSatisfiedException( + blobId.getName(), + currentPosition, + (end < Long.MAX_VALUE - 1) ? end - currentPosition + 1 : end, + storageException + ) + ); + } + throw addSuppressedExceptions(storageException); } } diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRetryingInputStreamTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRetryingInputStreamTests.java new file mode 100644 index 0000000000000..639b52ae238c5 --- /dev/null +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRetryingInputStreamTests.java @@ -0,0 +1,171 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.repositories.gcs; + +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.http.LowLevelHttpResponse; +import com.google.api.client.testing.http.HttpTesting; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.api.client.testing.http.MockLowLevelHttpRequest; +import com.google.api.client.testing.http.MockLowLevelHttpResponse; +import com.google.api.gax.retrying.RetrySettings; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; + +import org.elasticsearch.repositories.blobstore.RequestedRangeNotSatisfiedException; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Locale; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class GoogleCloudStorageRetryingInputStreamTests extends ESTestCase { + + private static final String BUCKET_NAME = "test-bucket"; + private static final String BLOB_NAME = "test-blob"; + + private final BlobId blobId = BlobId.of(BUCKET_NAME, BLOB_NAME); + + private com.google.api.services.storage.Storage storage; + private com.google.cloud.storage.Storage client; + private com.google.api.services.storage.Storage.Objects.Get get; + + @Before + public void init() throws IOException { + storage = mock(com.google.api.services.storage.Storage.class); + com.google.api.services.storage.Storage.Objects objects = mock(com.google.api.services.storage.Storage.Objects.class); + when(storage.objects()).thenReturn(objects); + + get = mock(com.google.api.services.storage.Storage.Objects.Get.class); + when(objects.get(BUCKET_NAME, BLOB_NAME)).thenReturn(get); + + client = mock(com.google.cloud.storage.Storage.class); + when(client.getOptions()).thenReturn( + StorageOptions.newBuilder() + .setProjectId("ignore") + .setRetrySettings(RetrySettings.newBuilder().setMaxAttempts(randomIntBetween(1, 3)).build()) + .build() + ); + } + + public void testReadWithinBlobLength() throws IOException { + byte[] bytes = randomByteArrayOfLength(randomIntBetween(1, 512)); + int position = randomIntBetween(0, Math.max(0, bytes.length - 1)); + int maxLength = bytes.length - position; // max length to read if length param exceeds buffer size + int length = randomIntBetween(0, Integer.MAX_VALUE - 1); + + GoogleCloudStorageRetryingInputStream stream; + boolean readWithExactPositionAndLength = randomBoolean(); + if (readWithExactPositionAndLength) { + stream = createRetryingInputStream(bytes, position, length); + } else { + stream = createRetryingInputStream(bytes); + } + try (stream) { + var out = new ByteArrayOutputStream(); + var readLength = org.elasticsearch.core.Streams.copy(stream, out); + if (readWithExactPositionAndLength) { + assertThat(readLength, equalTo((long) Math.min(length, maxLength))); + assertArrayEquals(out.toByteArray(), Arrays.copyOfRange(bytes, position, position + Math.min(length, maxLength))); + } else { + assertThat(readLength, equalTo((long) bytes.length)); + assertArrayEquals(out.toByteArray(), bytes); + } + } + } + + public void testReadBeyondBlobLengthThrowsRequestedRangeNotSatisfiedException() { + byte[] bytes = randomByteArrayOfLength(randomIntBetween(1, 512)); + int position = bytes.length + randomIntBetween(0, 100); + int length = randomIntBetween(1, 100); + var exception = expectThrows(RequestedRangeNotSatisfiedException.class, () -> { + try (var ignored = createRetryingInputStream(bytes, position, length)) { + fail(); + } + }); + assertThat(exception.getResource(), equalTo(BLOB_NAME)); + assertThat(exception.getPosition(), equalTo((long) position)); + assertThat(exception.getLength(), equalTo((long) length)); + assertThat( + exception.getMessage(), + equalTo( + String.format( + Locale.ROOT, + "Requested range [position=%d, length=%d] cannot be satisfied for [%s]", + position, + length, + BLOB_NAME + ) + ) + ); + assertThat(exception.getCause(), instanceOf(StorageException.class)); + } + + private GoogleCloudStorageRetryingInputStream createRetryingInputStream(byte[] data) throws IOException { + + HttpTransport transport = getMockHttpTransport(data, 0, data.length); + HttpRequest httpRequest = transport.createRequestFactory().buildGetRequest(HttpTesting.SIMPLE_GENERIC_URL); + HttpResponse httpResponse = httpRequest.execute(); + when(get.executeMedia()).thenReturn(httpResponse); + + return new GoogleCloudStorageRetryingInputStream(client, () -> storage, blobId); + } + + private GoogleCloudStorageRetryingInputStream createRetryingInputStream(byte[] data, int position, int length) throws IOException { + + if (position >= data.length) { + when(get.executeMedia()).thenThrow( + new StorageException(RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus(), "Test range not satisfied") + ); + } else { + HttpTransport transport = getMockHttpTransport(data, position, length); + HttpRequest httpRequest = transport.createRequestFactory().buildGetRequest(HttpTesting.SIMPLE_GENERIC_URL); + HttpResponse httpResponse = httpRequest.execute(); + when(get.executeMedia()).thenReturn(httpResponse); + } + + return new GoogleCloudStorageRetryingInputStream(client, () -> storage, blobId, position, position + length - 1); + } + + private static HttpTransport getMockHttpTransport(byte[] data, int position, int length) { + InputStream content = new ByteArrayInputStream(data, position, length); + long contentLength = position + length - 1; + HttpTransport transport = new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) throws IOException { + return new MockLowLevelHttpRequest() { + @Override + public LowLevelHttpResponse execute() throws IOException { + MockLowLevelHttpResponse result = new MockLowLevelHttpResponse(); + result.setContent(content); + result.setContentLength(contentLength); + result.setContentType("application/octet-stream"); + result.setStatusCode(RestStatus.OK.getStatus()); + return result; + } + }; + } + }; + return transport; + } +} diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index c97e26651d4ee..1132111826563 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -14,8 +14,6 @@ import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; -import org.elasticsearch.action.ActionRunnable; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; @@ -418,23 +416,14 @@ public void testEnforcedCooldownPeriod() throws IOException { final BytesReference serialized = BytesReference.bytes( modifiedRepositoryData.snapshotsToXContent(XContentFactory.jsonBuilder(), SnapshotsService.OLD_SNAPSHOT_FORMAT) ); - PlainActionFuture.get( - f -> repository.threadPool() - .generic() - .execute( - ActionRunnable.run( - f, - () -> repository.blobStore() - .blobContainer(repository.basePath()) - .writeBlobAtomic( - randomNonDataPurpose(), - BlobStoreRepository.INDEX_FILE_PREFIX + modifiedRepositoryData.getGenId(), - serialized, - true - ) - ) - ) - ); + repository.blobStore() + .blobContainer(repository.basePath()) + .writeBlobAtomic( + randomNonDataPurpose(), + BlobStoreRepository.INDEX_FILE_PREFIX + modifiedRepositoryData.getGenId(), + serialized, + true + ); final String newSnapshotName = "snapshot-new"; final long beforeThrottledSnapshot = repository.threadPool().relativeTimeInNanos(); diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java index 2359176abf715..65c5189504a19 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java @@ -15,7 +15,6 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.blobstore.OptionalBytesReference; @@ -25,7 +24,6 @@ import org.elasticsearch.common.settings.SecureSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException; import org.elasticsearch.core.Booleans; import org.elasticsearch.core.TimeValue; import org.elasticsearch.indices.recovery.RecoverySettings; @@ -33,7 +31,6 @@ import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase; import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.repositories.blobstore.RequestedRangeNotSatisfiedException; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.fixtures.minio.MinioTestContainer; @@ -45,18 +42,15 @@ import java.io.IOException; import java.util.Collection; import java.util.List; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.blankOrNullString; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.not; @@ -167,19 +161,12 @@ public long absoluteTimeInMillis() { class TestHarness { boolean tryCompareAndSet(BytesReference expected, BytesReference updated) { - return PlainActionFuture.get( - future -> blobContainer.compareAndSetRegister(randomPurpose(), "key", expected, updated, future), - 10, - TimeUnit.SECONDS - ); + return safeAwait(l -> blobContainer.compareAndSetRegister(randomPurpose(), "key", expected, updated, l)); } BytesReference readRegister() { - return PlainActionFuture.get( - future -> blobContainer.getRegister(randomPurpose(), "key", future.map(OptionalBytesReference::bytesReference)), - 10, - TimeUnit.SECONDS - ); + final OptionalBytesReference result = safeAwait(l -> blobContainer.getRegister(randomPurpose(), "key", l)); + return result.bytesReference(); } List listMultipartUploads() { @@ -231,37 +218,8 @@ List listMultipartUploads() { } public void testReadFromPositionLargerThanBlobLength() { - final var blobName = randomIdentifier(); - final var blobBytes = randomBytesReference(randomIntBetween(100, 2_000)); - - final var repository = getRepository(); - executeOnBlobStore(repository, blobStore -> { - blobStore.writeBlob(randomPurpose(), blobName, blobBytes, true); - return null; - }); - - long position = randomLongBetween(blobBytes.length(), Long.MAX_VALUE - 1L); - long length = randomLongBetween(1L, Long.MAX_VALUE - position); - - var exception = expectThrows(UncategorizedExecutionException.class, () -> readBlob(repository, blobName, position, length)); - assertThat(exception.getCause(), instanceOf(ExecutionException.class)); - assertThat(exception.getCause().getCause(), instanceOf(RequestedRangeNotSatisfiedException.class)); - assertThat( - exception.getCause().getCause().getMessage(), - containsString( - "Requested range [position=" - + position - + ", length=" - + length - + "] cannot be satisfied for [" - + repository.basePath().buildAsString() - + blobName - + ']' - ) - ); - assertThat( - asInstanceOf(AmazonS3Exception.class, exception.getRootCause()).getStatusCode(), - equalTo(RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus()) + testReadFromPositionLargerThanBlobLength( + e -> asInstanceOf(AmazonS3Exception.class, e.getCause()).getStatusCode() == RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus() ); } } diff --git a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java index 6b4b0a52b643a..2d378c12823ff 100644 --- a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java +++ b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.node.Node; import org.elasticsearch.tasks.Task; @@ -33,7 +34,7 @@ public TransportMainAction( ActionFilters actionFilters, ClusterService clusterService ) { - super(MainRestPlugin.MAIN_ACTION.name(), actionFilters, transportService.getTaskManager()); + super(MainRestPlugin.MAIN_ACTION.name(), actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.nodeName = Node.NODE_NAME_SETTING.get(settings); this.clusterService = clusterService; } diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedContinuationsIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedContinuationsIT.java index c4c35b410af78..46684faf9fb66 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedContinuationsIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedContinuationsIT.java @@ -20,7 +20,6 @@ import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.CountDownActionListener; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.TransportAction; @@ -71,6 +70,7 @@ import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskCancelledException; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.MockLog; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.threadpool.ThreadPool; @@ -435,13 +435,22 @@ public static class TransportYieldsContinuationsAction extends TransportAction listener) { - executor.execute(ActionRunnable.supply(listener, () -> new Response(request.failIndex, executor))); + var response = new Response(request.failIndex, executor); + try { + listener.onResponse(response); + } catch (Exception e) { + ESTestCase.fail(e); + } } } @@ -585,18 +594,22 @@ public static class TransportInfiniteContinuationsAction extends TransportAction @Inject public TransportInfiniteContinuationsAction(ActionFilters actionFilters, TransportService transportService) { - super(TYPE.name(), actionFilters, transportService.getTaskManager()); - this.executor = transportService.getThreadPool().executor(ThreadPool.Names.GENERIC); + this(actionFilters, transportService, transportService.getThreadPool().executor(ThreadPool.Names.GENERIC)); + } + + TransportInfiniteContinuationsAction(ActionFilters actionFilters, TransportService transportService, ExecutorService executor) { + super(TYPE.name(), actionFilters, transportService.getTaskManager(), executor); + this.executor = executor; } @Override protected void doExecute(Task task, Request request, ActionListener listener) { - executor.execute( - ActionRunnable.supply( - ActionTestUtils.assertNoFailureListener(listener::onResponse), - () -> new Response(randomFrom(executor, EsExecutors.DIRECT_EXECUTOR_SERVICE)) - ) - ); + var response = new Response(randomFrom(executor, EsExecutors.DIRECT_EXECUTOR_SERVICE)); + try { + listener.onResponse(response); + } catch (Exception e) { + ESTestCase.fail(e); + } } } diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java index 6eaddf51c02b4..cedb68b25a4bf 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java @@ -86,16 +86,13 @@ public void executeHandshake( } public void testConnectException() throws UnknownHostException { - try { - connectToNode( - serviceA, - DiscoveryNodeUtils.create("C", new TransportAddress(InetAddress.getByName("localhost"), 9876), emptyMap(), emptySet()) - ); - fail("Expected ConnectTransportException"); - } catch (ConnectTransportException e) { - assertThat(e.getMessage(), containsString("connect_exception")); - assertThat(e.getMessage(), containsString("[127.0.0.1:9876]")); - } + final var e = connectToNodeExpectFailure( + serviceA, + DiscoveryNodeUtils.create("C", new TransportAddress(InetAddress.getByName("localhost"), 9876), emptyMap(), emptySet()), + null + ); + assertThat(e.getMessage(), containsString("connect_exception")); + assertThat(e.getMessage(), containsString("[127.0.0.1:9876]")); } public void testDefaultKeepAliveSettings() throws IOException { @@ -236,10 +233,7 @@ public void testTimeoutPerConnection() throws IOException { final ConnectionProfile profile = builder.build(); // now with the 1ms timeout we got and test that is it's applied long startTime = System.nanoTime(); - ConnectTransportException ex = expectThrows( - ConnectTransportException.class, - () -> openConnection(service, second, profile) - ); + ConnectTransportException ex = openConnectionExpectFailure(service, second, profile); final long now = System.nanoTime(); final long timeTaken = TimeValue.nsecToMSec(now - startTime); assertTrue( diff --git a/muted-tests.yml b/muted-tests.yml index ac0b03cc4b4fa..df491aa34b896 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -4,8 +4,7 @@ tests: method: "testGuessIsDayFirstFromLocale" - class: "org.elasticsearch.test.rest.ClientYamlTestSuiteIT" issue: "https://github.com/elastic/elasticsearch/issues/108857" - method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\ - \ dependent mappings / dates}" + method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}" - class: "org.elasticsearch.upgrades.SearchStatesIT" issue: "https://github.com/elastic/elasticsearch/issues/108991" method: "testCanMatch" @@ -14,8 +13,7 @@ tests: method: "testTrainedModelInference" - class: "org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT" issue: "https://github.com/elastic/elasticsearch/issues/109188" - method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\ - \ dependent mappings / dates}" + method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}" - class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT" issue: "https://github.com/elastic/elasticsearch/issues/109189" method: "test {p0=esql/70_locale/Date format with Italian locale}" @@ -30,8 +28,7 @@ tests: method: "testTimestampFieldTypeExposedByAllIndicesServices" - class: "org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT" issue: "https://github.com/elastic/elasticsearch/issues/109318" - method: "test {yaml=analysis-common/50_char_filters/pattern_replace error handling\ - \ (too complex pattern)}" + method: "test {yaml=analysis-common/50_char_filters/pattern_replace error handling (too complex pattern)}" - class: "org.elasticsearch.xpack.ml.integration.ClassificationHousePricingIT" issue: "https://github.com/elastic/elasticsearch/issues/101598" method: "testFeatureImportanceValues" @@ -61,9 +58,6 @@ tests: - class: org.elasticsearch.upgrades.SecurityIndexRolesMetadataMigrationIT method: testMetadataMigratedAfterUpgrade issue: https://github.com/elastic/elasticsearch/issues/110232 -- class: org.elasticsearch.compute.lucene.ValueSourceReaderTypeConversionTests - method: testLoadAll - issue: https://github.com/elastic/elasticsearch/issues/110244 - class: org.elasticsearch.backwards.SearchWithMinCompatibleSearchNodeIT method: testMinVersionAsNewVersion issue: https://github.com/elastic/elasticsearch/issues/95384 @@ -76,12 +70,62 @@ tests: - class: "org.elasticsearch.xpack.searchablesnapshots.FrozenSearchableSnapshotsIntegTests" issue: "https://github.com/elastic/elasticsearch/issues/110408" method: "testCreateAndRestorePartialSearchableSnapshot" -- class: "org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT" - issue: "https://github.com/elastic/elasticsearch/issues/110719" - method: "test {p0=search.vectors/45_knn_search_byte/Test nonexistent field}" -- class: "org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT" - issue: "https://github.com/elastic/elasticsearch/issues/110720" - method: "test {p0=search.vectors/40_knn_search/Test nonexistent field}" +- class: org.elasticsearch.xpack.security.LicenseDLSFLSRoleIT + method: testQueryDLSFLSRolesShowAsDisabled + issue: https://github.com/elastic/elasticsearch/issues/110729 +- class: org.elasticsearch.xpack.security.authz.store.NativePrivilegeStoreCacheTests + method: testPopulationOfCacheWhenLoadingPrivilegesForAllApplications + issue: https://github.com/elastic/elasticsearch/issues/110789 +- class: org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFileTests + method: testCacheFileCreatedAsSparseFile + issue: https://github.com/elastic/elasticsearch/issues/110801 +- class: org.elasticsearch.nativeaccess.PreallocateTests + method: testPreallocate + issue: https://github.com/elastic/elasticsearch/issues/110948 +- class: org.elasticsearch.nativeaccess.VectorSystemPropertyTests + method: testSystemPropertyDisabled + issue: https://github.com/elastic/elasticsearch/issues/110949 +- class: org.elasticsearch.xpack.esql.spatial.SpatialPushDownGeoPointIT + method: testPushedDownQueriesSingleValue + issue: https://github.com/elastic/elasticsearch/issues/111084 +- class: org.elasticsearch.xpack.esql.spatial.SpatialPushDownCartesianPointIT + method: testPushedDownQueriesSingleValue + issue: https://github.com/elastic/elasticsearch/issues/110982 +- class: org.elasticsearch.multi_node.GlobalCheckpointSyncActionIT + issue: https://github.com/elastic/elasticsearch/issues/111124 +- class: org.elasticsearch.cluster.PrevalidateShardPathIT + method: testCheckShards + issue: https://github.com/elastic/elasticsearch/issues/111134 +- class: org.elasticsearch.packaging.test.DockerTests + method: test021InstallPlugin + issue: https://github.com/elastic/elasticsearch/issues/110343 +- class: org.elasticsearch.xpack.security.authc.oidc.OpenIdConnectAuthIT + method: testAuthenticateWithImplicitFlow + issue: https://github.com/elastic/elasticsearch/issues/111191 +- class: org.elasticsearch.xpack.restart.FullClusterRestartIT + method: testDisableFieldNameField {cluster=UPGRADED} + issue: https://github.com/elastic/elasticsearch/issues/111222 +- class: org.elasticsearch.repositories.azure.AzureBlobContainerRetriesTests + method: testReadNonexistentBlobThrowsNoSuchFileException + issue: https://github.com/elastic/elasticsearch/issues/111233 +- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT + method: test {inlinestats.BeforeKeep ASYNC} + issue: https://github.com/elastic/elasticsearch/issues/111257 +- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT + method: test {inlinestats.BeforeKeep ASYNC} + issue: https://github.com/elastic/elasticsearch/issues/111259 +- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT + method: test {inlinestats.BeforeKeep SYNC} + issue: https://github.com/elastic/elasticsearch/issues/111260 +- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT + method: test {inlinestats.BeforeKeep SYNC} + issue: https://github.com/elastic/elasticsearch/issues/111262 +- class: org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT + method: testInlineStatsProfile {SYNC} + issue: https://github.com/elastic/elasticsearch/issues/111263 +- class: org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT + method: testInlineStatsProfile {ASYNC} + issue: https://github.com/elastic/elasticsearch/issues/111264 # Examples: # @@ -112,4 +156,12 @@ tests: # method: "test {union_types.MultiIndexIpStringStatsInline}" # issue: "https://github.com/elastic/elasticsearch/..." # Note that this mutes for the unit-test-like CsvTests only. -# Muting for the integration tests needs to be done for each IT class individually. +# Muting all the integration tests can be done using the class "org.elasticsearch.xpack.esql.**". +# Consider however, that some tests are named as "test {file.test SYNC}" and "ASYNC" in the integration tests. +# To mute all 3 tests safely everywhere use: +# - class: "org.elasticsearch.xpack.esql.**" +# method: "test {union_types.MultiIndexIpStringStatsInline}" +# issue: "https://github.com/elastic/elasticsearch/..." +# - class: "org.elasticsearch.xpack.esql.**" +# method: "test {union_types.MultiIndexIpStringStatsInline *}" +# issue: "https://github.com/elastic/elasticsearch/..." diff --git a/plugins/examples/gradle/wrapper/gradle-wrapper.properties b/plugins/examples/gradle/wrapper/gradle-wrapper.properties index 515ab9d5f1822..efe2ff3449216 100644 --- a/plugins/examples/gradle/wrapper/gradle-wrapper.properties +++ b/plugins/examples/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=f8b4f4772d302c8ff580bc40d0f56e715de69b163546944f787c87abf209c961 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-all.zip +distributionSha256Sum=258e722ec21e955201e31447b0aed14201765a3bfbae296a46cf60b70e66db70 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index 4e3a53d64a841..a5319387a2b68 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -335,7 +335,7 @@ public void testStoreParameterDefaults() throws IOException { var source = source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> { b.field("field", "1234"); if (timeSeriesIndexMode) { - b.field("@timestamp", randomMillisUpToYear9999()); + b.field("@timestamp", "2000-10-10T23:40:53.384Z"); b.field("dimension", "dimension1"); } }, null); diff --git a/qa/packaging/build.gradle b/qa/packaging/build.gradle index 02bc30ecd6b39..758dfe6661766 100644 --- a/qa/packaging/build.gradle +++ b/qa/packaging/build.gradle @@ -36,13 +36,3 @@ tasks.named("test").configure { enabled = false } tasks.register('destructivePackagingTest') { dependsOn 'destructiveDistroTest' } - -tasks.named('resolveAllDependencies') { - // Don't try and resolve all distros but only the latest patch versions of each minor - def latestBugfixVersions = org.elasticsearch.gradle.internal.info.BuildParams.getBwcVersions().getIndexCompatible() - .groupBy { [it.major, it.minor] } - .collectEntries { key, value -> [key, value.max()] } - .values() - - configs = configurations.matching { configName -> latestBugfixVersions.any { v -> configName.name.endsWith(v.toString()) } } -} diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java index 1925b1e8f36ab..2ce9eef29d903 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java @@ -20,7 +20,6 @@ import static java.nio.file.attribute.PosixFilePermissions.fromString; import static org.elasticsearch.packaging.util.FileUtils.append; -import static org.hamcrest.Matchers.equalTo; import static org.junit.Assume.assumeFalse; public class ConfigurationTests extends PackagingTestCase { @@ -50,13 +49,15 @@ public void test20HostnameSubstitution() throws Exception { // security auto-config requires that the archive owner and the node process user be the same Platforms.onWindows(() -> sh.chown(confPath, installation.getOwner())); assertWhileRunning(() -> { - final String nameResponse = ServerUtils.makeRequest( - Request.Get("https://localhost:9200/_cat/nodes?h=name"), - "test_superuser", - "test_superuser_password", - ServerUtils.getCaCert(confPath) - ).strip(); - assertThat(nameResponse, equalTo("mytesthost")); + assertBusy(() -> { + final String nameResponse = ServerUtils.makeRequest( + Request.Get("https://localhost:9200/_cat/nodes?h=name"), + "test_superuser", + "test_superuser_password", + ServerUtils.getCaCert(confPath) + ).strip(); + assertEquals("mytesthost", nameResponse); + }); }); Platforms.onWindows(() -> sh.chown(confPath)); }); diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index f9723f30cc371..18668b842b2d3 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -1231,8 +1231,7 @@ public void test500Readiness() throws Exception { assertBusy(() -> assertTrue(readinessProbe(9399))); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99508") - public void test600Interrupt() { + public void test600Interrupt() throws Exception { waitForElasticsearch(installation, "elastic", PASSWORD); final Result containerLogs = getContainerLogs(); @@ -1242,10 +1241,12 @@ public void test600Interrupt() { final int maxPid = infos.stream().map(i -> i.pid()).max(Integer::compareTo).get(); sh.run("bash -c 'kill -int " + maxPid + "'"); // send ctrl+c to all java processes - final Result containerLogsAfter = getContainerLogs(); - assertThat("Container logs should contain stopping ...", containerLogsAfter.stdout(), containsString("stopping ...")); - assertThat("No errors stdout", containerLogsAfter.stdout(), not(containsString("java.security.AccessControlException:"))); - assertThat("No errors stderr", containerLogsAfter.stderr(), not(containsString("java.security.AccessControlException:"))); + assertBusy(() -> { + final Result containerLogsAfter = getContainerLogs(); + assertThat("Container logs should contain stopping ...", containerLogsAfter.stdout(), containsString("stopping ...")); + assertThat("No errors stdout", containerLogsAfter.stdout(), not(containsString("java.security.AccessControlException:"))); + assertThat("No errors stderr", containerLogsAfter.stderr(), not(containsString("java.security.AccessControlException:"))); + }); } } diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsUpgradeIT.java index c80911fe5fbcf..8ffaec5506f1d 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsUpgradeIT.java @@ -15,11 +15,10 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.FeatureFlag; -import org.elasticsearch.test.cluster.local.DefaultLocalClusterSpecBuilder; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.cluster.util.Version; import org.elasticsearch.test.cluster.util.resource.Resource; -import org.junit.BeforeClass; +import org.elasticsearch.test.junit.RunnableTestRuleAdapter; import org.junit.ClassRule; import org.junit.rules.RuleChain; import org.junit.rules.TemporaryFolder; @@ -33,10 +32,9 @@ public class FileSettingsUpgradeIT extends ParameterizedRollingUpgradeTestCase { - @BeforeClass - public static void checkVersion() { - assumeTrue("Only valid when upgrading from pre-file settings", getOldClusterTestVersion().before(new Version(8, 4, 0))); - } + private static final RunnableTestRuleAdapter versionLimit = new RunnableTestRuleAdapter( + () -> assumeTrue("Only valid when upgrading from pre-file settings", getOldClusterTestVersion().before(new Version(8, 4, 0))) + ); private static final String settingsJSON = """ { @@ -53,7 +51,8 @@ public static void checkVersion() { private static final TemporaryFolder repoDirectory = new TemporaryFolder(); - private static final ElasticsearchCluster cluster = new DefaultLocalClusterSpecBuilder().distribution(DistributionType.DEFAULT) + private static final ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) .version(getOldClusterTestVersion()) .nodes(NODE_NUM) .setting("path.repo", new Supplier<>() { @@ -69,7 +68,7 @@ public String get() { .build(); @ClassRule - public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(cluster); + public static TestRule ruleChain = RuleChain.outerRule(versionLimit).around(repoDirectory).around(cluster); public FileSettingsUpgradeIT(@Name("upgradedNodes") int upgradedNodes) { super(upgradedNodes); diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BlockedSearcherRestCancellationTestCase.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BlockedSearcherRestCancellationTestCase.java index 8803ad4af7348..250fe5a3a79fb 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BlockedSearcherRestCancellationTestCase.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BlockedSearcherRestCancellationTestCase.java @@ -30,6 +30,7 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.EnginePlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.tasks.Task; import java.util.ArrayList; import java.util.Collection; @@ -76,6 +77,10 @@ void runTest(Request request, String actionPrefix) throws Exception { createIndex("test", Settings.builder().put(BLOCK_SEARCHER_SETTING.getKey(), true).build()); ensureGreen("test"); + assert request.getOptions().containsHeader(Task.X_OPAQUE_ID_HTTP_HEADER) == false; + final var opaqueId = getTestClass().getSimpleName() + "-" + getTestName() + "-" + randomUUID(); + request.setOptions(request.getOptions().toBuilder().addHeader(Task.X_OPAQUE_ID_HTTP_HEADER, opaqueId)); + final List searcherBlocks = new ArrayList<>(); for (final IndicesService indicesService : internalCluster().getInstances(IndicesService.class)) { for (final IndexService indexService : indicesService) { @@ -96,7 +101,8 @@ void runTest(Request request, String actionPrefix) throws Exception { } final PlainActionFuture future = new PlainActionFuture<>(); - logger.info("--> sending request"); + logger.info("--> sending request, opaque id={}", opaqueId); + final Cancellable cancellable = getRestClient().performRequestAsync(request, wrapAsRestResponseListener(future)); awaitTaskWithPrefix(actionPrefix); @@ -108,7 +114,7 @@ void runTest(Request request, String actionPrefix) throws Exception { cancellable.cancel(); expectThrows(CancellationException.class, future::actionGet); - assertAllCancellableTasksAreCancelled(actionPrefix); + assertAllCancellableTasksAreCancelled(actionPrefix, opaqueId); } finally { Releasables.close(releasables); } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_geoip_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_geoip_database.json new file mode 100644 index 0000000000000..ef6dc94dd27a6 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_geoip_database.json @@ -0,0 +1,31 @@ +{ + "ingest.delete_geoip_database":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/TODO.html", + "description":"Deletes a geoip database configuration" + }, + "stability":"stable", + "visibility":"public", + "headers":{ + "accept": [ "application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_ingest/geoip/database/{id}", + "methods":[ + "DELETE" + ], + "parts":{ + "id":{ + "type":"list", + "description":"A comma-separated list of geoip database configurations to delete" + } + } + } + ] + }, + "params":{ + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_geoip_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_geoip_database.json new file mode 100644 index 0000000000000..96f028e2e5251 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_geoip_database.json @@ -0,0 +1,37 @@ +{ + "ingest.get_geoip_database":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/TODO.html", + "description":"Returns geoip database configuration." + }, + "stability":"stable", + "visibility":"public", + "headers":{ + "accept": [ "application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_ingest/geoip/database", + "methods":[ + "GET" + ] + }, + { + "path":"/_ingest/geoip/database/{id}", + "methods":[ + "GET" + ], + "parts":{ + "id":{ + "type":"list", + "description":"A comma-separated list of geoip database configurations to get; use `*` to get all geoip database configurations" + } + } + } + ] + }, + "params":{ + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_geoip_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_geoip_database.json new file mode 100644 index 0000000000000..07f9e37740279 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_geoip_database.json @@ -0,0 +1,35 @@ +{ + "ingest.put_geoip_database":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/TODO.html", + "description":"Puts the configuration for a geoip database to be downloaded" + }, + "stability":"stable", + "visibility":"public", + "headers":{ + "accept": [ "application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_ingest/geoip/database/{id}", + "methods":[ + "PUT" + ], + "parts":{ + "id":{ + "type":"string", + "description":"The id of the database configuration" + } + } + } + ] + }, + "params":{ + }, + "body":{ + "description":"The database configuration definition", + "required":true + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/security.create_cross_cluster_api_key.json b/rest-api-spec/src/main/resources/rest-api-spec/api/security.create_cross_cluster_api_key.json index 6fd74f9eba3e3..88d6b97067492 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/security.create_cross_cluster_api_key.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/security.create_cross_cluster_api_key.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html", "description": "Creates a cross-cluster API key for API key based remote cluster access." }, - "stability": "beta", + "stability": "stable", "visibility": "public", "headers": { "accept": [ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/security.update_cross_cluster_api_key.json b/rest-api-spec/src/main/resources/rest-api-spec/api/security.update_cross_cluster_api_key.json index 9428089a31e80..e59d6c1efccf8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/security.update_cross_cluster_api_key.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/security.update_cross_cluster_api_key.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-update-cross-cluster-api-key.html", "description": "Updates attributes of an existing cross-cluster API key." }, - "stability": "beta", + "stability": "stable", "visibility": "public", "headers": { "accept": [ diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/70_index_mode.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/70_index_mode.yml new file mode 100644 index 0000000000000..9da6d2c5f086e --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/70_index_mode.yml @@ -0,0 +1,59 @@ +--- +setup: + - requires: + cluster_features: "mapper.query_index_mode" + reason: "require index_mode" + + - do: + indices.create: + index: test_metrics + body: + settings: + index: + mode: time_series + routing_path: [container] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + container: + type: keyword + time_series_dimension: true + + - do: + indices.create: + index: test + body: + mappings: + properties: + "@timestamp": + type: date + +--- +Field-caps: + - do: + field_caps: + index: "test*" + fields: "*" + body: { index_filter: { term: { _index_mode: "time_series" } } } + - match: { indices: [ "test_metrics" ] } + - do: + field_caps: + index: "test*" + fields: "*" + body: { index_filter: { term: { _index_mode: "logs" } } } + - match: { indices: [ ] } + - do: + field_caps: + index: "test*" + fields: "*" + body: { index_filter: { term: { _index_mode: "standard" } } } + - match: { indices: [ "test" ] } + - do: + field_caps: + index: "test*" + fields: "*" + - match: { indices: [ "test" , "test_metrics" ] } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml index 4976e5e15adbe..07cb154449a70 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml @@ -5,8 +5,8 @@ setup: capabilities: - method: PUT path: /{index} - capabilities: [logs_index_mode] - reason: "Support for 'logs' index mode capability required" + capabilities: [logsdb_index_mode] + reason: "Support for 'logsdb' index mode capability required" --- create logs index: @@ -15,8 +15,8 @@ create logs index: capabilities: - method: PUT path: /{index} - capabilities: [ logs_index_mode ] - reason: "Support for 'logs' index mode capability required" + capabilities: [ logsdb_index_mode ] + reason: "Support for 'logsdb' index mode capability required" - do: indices.create: @@ -24,7 +24,7 @@ create logs index: body: settings: index: - mode: logs + mode: logsdb number_of_replicas: 0 number_of_shards: 2 mappings: @@ -75,7 +75,7 @@ create logs index: index: test - is_true: test - - match: { test.settings.index.mode: "logs" } + - match: { test.settings.index.mode: "logsdb" } - do: indices.get_mapping: @@ -89,8 +89,8 @@ using default timestamp field mapping: capabilities: - method: PUT path: /{index} - capabilities: [ logs_index_mode ] - reason: "Support for 'logs' index mode capability required" + capabilities: [ logsdb_index_mode ] + reason: "Support for 'logsdb' index mode capability required" - do: indices.create: @@ -98,7 +98,7 @@ using default timestamp field mapping: body: settings: index: - mode: logs + mode: logsdb number_of_replicas: 0 number_of_shards: 2 mappings: @@ -121,17 +121,16 @@ missing hostname field: capabilities: - method: PUT path: /{index} - capabilities: [ logs_index_mode ] - reason: "Support for 'logs' index mode capability required" + capabilities: [ logsdb_index_mode ] + reason: "Support for 'logsdb' index mode capability required" - do: - catch: bad_request indices.create: index: test-hostname-missing body: settings: index: - mode: logs + mode: logsdb number_of_replicas: 0 number_of_shards: 2 mappings: @@ -147,9 +146,12 @@ missing hostname field: message: type: text - - match: { error.root_cause.0.type: "illegal_argument_exception" } - - match: { error.type: "illegal_argument_exception" } - - match: { error.reason: "unknown index sort field:[host.name]" } + - do: + indices.get_settings: + index: test-hostname-missing + + - is_true: test-hostname-missing + - match: { test-hostname-missing.settings.index.mode: "logsdb" } --- missing sort field: @@ -158,8 +160,8 @@ missing sort field: capabilities: - method: PUT path: /{index} - capabilities: [ logs_index_mode ] - reason: "Support for 'logs' index mode capability required" + capabilities: [ logsdb_index_mode ] + reason: "Support for 'logsdb' index mode capability required" - do: catch: bad_request @@ -168,7 +170,7 @@ missing sort field: body: settings: index: - mode: logs + mode: logsdb number_of_replicas: 0 number_of_shards: 2 sort: @@ -199,8 +201,8 @@ non-default sort settings: capabilities: - method: PUT path: /{index} - capabilities: [ logs_index_mode ] - reason: "Support for 'logs' index mode capability required" + capabilities: [ logsdb_index_mode ] + reason: "Support for 'logsdb' index mode capability required" - do: indices.create: @@ -209,7 +211,7 @@ non-default sort settings: settings: index: - mode: logs + mode: logsdb number_of_shards: 2 number_of_replicas: 0 sort: @@ -235,7 +237,7 @@ non-default sort settings: index: test-sort - is_true: test-sort - - match: { test-sort.settings.index.mode: "logs" } + - match: { test-sort.settings.index.mode: "logsdb" } - match: { test-sort.settings.index.sort.field.0: "agent_id" } - match: { test-sort.settings.index.sort.field.1: "@timestamp" } - match: { test-sort.settings.index.sort.order.0: "asc" } @@ -252,8 +254,8 @@ override sort order settings: capabilities: - method: PUT path: /{index} - capabilities: [ logs_index_mode ] - reason: "Support for 'logs' index mode capability required" + capabilities: [ logsdb_index_mode ] + reason: "Support for 'logsdb' index mode capability required" - do: indices.create: @@ -262,7 +264,7 @@ override sort order settings: settings: index: - mode: logs + mode: logsdb number_of_shards: 2 number_of_replicas: 0 sort: @@ -287,7 +289,7 @@ override sort order settings: index: test-sort-order - is_true: test-sort-order - - match: { test-sort-order.settings.index.mode: "logs" } + - match: { test-sort-order.settings.index.mode: "logsdb" } - match: { test-sort-order.settings.index.sort.field.0: null } - match: { test-sort-order.settings.index.sort.field.1: null } - match: { test-sort-order.settings.index.sort.order.0: "asc" } @@ -300,8 +302,8 @@ override sort missing settings: capabilities: - method: PUT path: /{index} - capabilities: [ logs_index_mode ] - reason: "Support for 'logs' index mode capability required" + capabilities: [ logsdb_index_mode ] + reason: "Support for 'logsdb' index mode capability required" - do: indices.create: @@ -310,7 +312,7 @@ override sort missing settings: settings: index: - mode: logs + mode: logsdb number_of_shards: 2 number_of_replicas: 0 sort: @@ -335,7 +337,7 @@ override sort missing settings: index: test-sort-missing - is_true: test-sort-missing - - match: { test-sort-missing.settings.index.mode: "logs" } + - match: { test-sort-missing.settings.index.mode: "logsdb" } - match: { test-sort-missing.settings.index.sort.field.0: null } - match: { test-sort-missing.settings.index.sort.field.1: null } - match: { test-sort-missing.settings.index.sort.missing.0: "_last" } @@ -348,8 +350,8 @@ override sort mode settings: capabilities: - method: PUT path: /{index} - capabilities: [ logs_index_mode ] - reason: "Support for 'logs' index mode capability required" + capabilities: [ logsdb_index_mode ] + reason: "Support for 'logsdb' index mode capability required" - do: indices.create: @@ -358,7 +360,7 @@ override sort mode settings: settings: index: - mode: logs + mode: logsdb number_of_shards: 2 number_of_replicas: 0 sort: @@ -383,7 +385,7 @@ override sort mode settings: index: test-sort-mode - is_true: test-sort-mode - - match: { test-sort-mode.settings.index.mode: "logs" } + - match: { test-sort-mode.settings.index.mode: "logsdb" } - match: { test-sort-mode.settings.index.sort.field.0: null } - match: { test-sort-mode.settings.index.sort.field.1: null } - match: { test-sort-mode.settings.index.sort.mode.0: "max" } @@ -397,8 +399,8 @@ override sort field using nested field type in sorting: capabilities: - method: PUT path: /{index} - capabilities: [ logs_index_mode ] - reason: "Support for 'logs' index mode capability required" + capabilities: [ logsdb_index_mode ] + reason: "Support for 'logsdb' index mode capability required" - do: catch: bad_request @@ -407,7 +409,7 @@ override sort field using nested field type in sorting: body: settings: index: - mode: logs + mode: logsdb number_of_replicas: 0 number_of_shards: 2 sort: @@ -444,8 +446,8 @@ override sort field using nested field type: capabilities: - method: PUT path: /{index} - capabilities: [ logs_index_mode ] - reason: "Support for 'logs' index mode capability required" + capabilities: [ logsdb_index_mode ] + reason: "Support for 'logsdb' index mode capability required" - do: indices.create: @@ -453,7 +455,7 @@ override sort field using nested field type: body: settings: index: - mode: logs + mode: logsdb number_of_replicas: 0 number_of_shards: 2 mappings: @@ -484,8 +486,8 @@ routing path not allowed in logs mode: capabilities: - method: PUT path: /{index} - capabilities: [ logs_index_mode ] - reason: "Support for 'logs' index mode capability required" + capabilities: [ logsdb_index_mode ] + reason: "Support for 'logsdb' index mode capability required" - do: catch: bad_request @@ -494,7 +496,7 @@ routing path not allowed in logs mode: body: settings: index: - mode: logs + mode: logsdb number_of_replicas: 0 number_of_shards: 2 routing_path: [ "host.name", "agent_id" ] @@ -524,8 +526,8 @@ start time not allowed in logs mode: capabilities: - method: PUT path: /{index} - capabilities: [ logs_index_mode ] - reason: "Support for 'logs' index mode capability required" + capabilities: [ logsdb_index_mode ] + reason: "Support for 'logsdb' index mode capability required" - do: catch: bad_request @@ -534,7 +536,7 @@ start time not allowed in logs mode: body: settings: index: - mode: logs + mode: logsdb number_of_replicas: 0 number_of_shards: 2 time_series: @@ -565,8 +567,8 @@ end time not allowed in logs mode: capabilities: - method: PUT path: /{index} - capabilities: [ logs_index_mode ] - reason: "Support for 'logs' index mode capability required" + capabilities: [ logsdb_index_mode ] + reason: "Support for 'logsdb' index mode capability required" - do: catch: bad_request @@ -575,7 +577,7 @@ end time not allowed in logs mode: body: settings: index: - mode: logs + mode: logsdb number_of_replicas: 0 number_of_shards: 2 time_series: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml new file mode 100644 index 0000000000000..d209c839d904b --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml @@ -0,0 +1,94 @@ +--- +stored _source mode is not supported: + - requires: + test_runner_features: [capabilities] + capabilities: + - method: PUT + path: /{index} + capabilities: [logsdb_index_mode] + reason: "Support for 'logsdb' index mode capability required" + + - skip: + known_issues: + - cluster_feature: "gte_v8.15.0" + fixed_by: "gte_v8.16.0" + reason: "Development of logs index mode spans 8.15 and 8.16" + + - do: + catch: bad_request + indices.create: + index: test-stored-source + body: + settings: + index: + mode: logsdb + mappings: + _source: + mode: stored + properties: + "@timestamp": + type: date + host.name: + type: keyword + + - match: { error.type: "mapper_parsing_exception" } + - match: { error.root_cause.0.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" } + +--- +disabled _source is not supported: + - requires: + test_runner_features: [capabilities] + capabilities: + - method: PUT + path: /{index} + capabilities: [logsdb_index_mode] + reason: "Support for 'logsdb' index mode capability required" + + - skip: + known_issues: + - cluster_feature: "gte_v8.15.0" + fixed_by: "gte_v8.16.0" + reason: "Development of logs index mode spans 8.15 and 8.16" + + - do: + catch: bad_request + indices.create: + index: test-disabled-source + body: + settings: + index: + mode: logsdb + mappings: + _source: + enabled: false + properties: + "@timestamp": + type: date + host.name: + type: keyword + + - match: { error.type: "mapper_parsing_exception" } + - match: { error.root_cause.0.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" } + + - do: + catch: bad_request + indices.create: + index: test-disabled-source + body: + settings: + index: + mode: logsdb + mappings: + _source: + mode: disabled + properties: + "@timestamp": + type: date + host.name: + type: keyword + + - match: { error.type: "mapper_parsing_exception" } + - match: { error.root_cause.0.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml index ac0f8aec4f3d0..9d6e8da8c1e1e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml @@ -417,7 +417,7 @@ - requires: test_runner_features: [arbitrary_key] - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.query_index_mode"] reason: "_ignored_source added to mappings" - do: @@ -478,36 +478,133 @@ # 6. _ignored # 7. _ignored_source # 8. _index - # 9. _nested_path - # 10. _routing - # 11. _seq_no - # 12. _source - # 13. _tier - # 14. _version - # 15. @timestamp - # 16. authors.age - # 17. authors.company - # 18. authors.company.keyword - # 19. authors.name.last_name - # 20. authors.name.first_name - # 21. authors.name.full_name - # 22. link - # 23. title - # 24. url + # 9. _index_mode + # 10. _nested_path + # 11. _routing + # 12. _seq_no + # 13. _source + # 14. _tier + # 15. _version + # 16. @timestamp + # 17. authors.age + # 18. authors.company + # 19. authors.company.keyword + # 20. authors.name.last_name + # 21. authors.name.first_name + # 22. authors.name.full_name + # 23. link + # 24. title + # 25. url # Object mappers: - # 25. authors - # 26. authors.name + # 26. authors + # 27. authors.name # Runtime field mappers: - # 27. a_source_field + # 28. a_source_field - - gte: { nodes.$node_id.indices.mappings.total_count: 27 } + - gte: { nodes.$node_id.indices.mappings.total_count: 28 } - is_true: nodes.$node_id.indices.mappings.total_estimated_overhead - gte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 26624 } - - match: { nodes.$node_id.indices.indices.index1.mappings.total_count: 27 } + - match: { nodes.$node_id.indices.indices.index1.mappings.total_count: 28 } - is_true: nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead - - match: { nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead_in_bytes: 27648 } + - match: { nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead_in_bytes: 28672 } --- +"Lucene segment level fields stats": + + - requires: + cluster_features: ["mapper.segment_level_fields_stats"] + reason: "segment level fields stats" + + - do: + indices.create: + index: index1 + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + routing.rebalance.enable: none + mappings: + runtime: + a_source_field: + type: keyword + properties: + "@timestamp": + type: date + authors: + properties: + age: + type: long + company: + type: text + fields: + keyword: + type: keyword + ignore_above: 256 + name: + properties: + first_name: + type: keyword + full_name: + type: text + last_name: + type: keyword + link: + type: alias + path: url + title: + type: text + url: + type: keyword + - do: + cluster.state: {} + + - set: + routing_table.indices.index1.shards.0.0.node: node_id + + - do: + nodes.stats: { metric: _all, level: "indices", human: true } + + - do: + index: + index: index1 + body: { "title": "foo", "@timestamp": "2023-10-15T14:12:12" } + - do: + indices.flush: + index: index1 + - do: + nodes.stats: { metric: _all, level: "indices", human: true } + + - gte: { nodes.$node_id.indices.mappings.total_count: 28 } + - lte: { nodes.$node_id.indices.mappings.total_count: 29 } + - gte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 28672 } + - lte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 29696 } + - match: { nodes.$node_id.indices.mappings.total_segments: 1 } + - gte: { nodes.$node_id.indices.mappings.total_segment_fields: 28 } + - lte: { nodes.$node_id.indices.mappings.total_segment_fields: 29 } + - gte: { nodes.$node_id.indices.mappings.average_fields_per_segment: 28 } + - lte: { nodes.$node_id.indices.mappings.average_fields_per_segment: 29 } + + - do: + index: + index: index1 + body: { "title": "bar", "@timestamp": "2023-11-15T14:12:12" } + - do: + indices.flush: + index: index1 + - do: + nodes.stats: { metric: _all, level: "indices", human: true } + + - gte: { nodes.$node_id.indices.mappings.total_count: 28 } + - lte: { nodes.$node_id.indices.mappings.total_count: 29 } + - gte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 28672 } + - lte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 29696 } + - match: { nodes.$node_id.indices.mappings.total_segments: 2 } + - gte: { nodes.$node_id.indices.mappings.total_segment_fields: 56 } + - lte: { nodes.$node_id.indices.mappings.total_segment_fields: 58 } + - gte: { nodes.$node_id.indices.mappings.average_fields_per_segment: 28 } + - lte: { nodes.$node_id.indices.mappings.average_fields_per_segment: 29 } +--- + "indices mappings does not exist in shards level": - requires: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml index 3502a5e643087..855daeaa7f163 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml @@ -2,9 +2,8 @@ setup: - requires: cluster_features: "gte_v8.15.0" reason: 'updatable dense vector field types was added in 8.15' - - skip: - reason: "contains is a newly added assertion" - features: contains + - requires: + test_runner_features: [ contains ] --- "Test create and update dense vector mapping with per-doc indexing and flush": - do: @@ -1016,6 +1015,45 @@ setup: index_options: type: int8_flat +--- +"Disallowed dense vector update path hnsw --> int4_flat": + - requires: + cluster_features: "gte_v8.16.0" + reason: 'updatable dense vector field type for int4 was added in 8.16' + - do: + indices.create: + index: test_index + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: hnsw + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: hnsw } + + - do: + catch: /illegal_argument_exception/ + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_flat + --- "Disallowed dense vector update path int8_hnsw --> flat": - do: @@ -1088,6 +1126,67 @@ setup: index_options: type: int8_flat +--- +"Disallowed dense vector update path int4_hnsw --> int8_flat, int4_flat, flat": + - requires: + cluster_features: "gte_v8.16.0" + reason: 'updatable dense vector field type for int4 was added in 8.16' + - do: + indices.create: + index: test_index + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_hnsw + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int4_hnsw } + + - do: + catch: /illegal_argument_exception/ + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int8_flat + - do: + catch: /illegal_argument_exception/ + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_flat + - do: + catch: /illegal_argument_exception/ + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: flat + --- "Disallowed dense vector update path int8_flat --> flat": - do: @@ -1124,6 +1223,56 @@ setup: index_options: type: flat +--- +"Disallowed dense vector update path int4_flat --> flat, int8_flat": + - requires: + cluster_features: "gte_v8.16.0" + reason: 'updatable dense vector field type for int4 was added in 8.16' + - do: + indices.create: + index: test_index + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_flat + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int4_flat } + + - do: + catch: /illegal_argument_exception/ + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: flat + - do: + catch: /illegal_argument_exception/ + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int8_flat + --- "Allowed dense vector updates on same type but different other index_options, hnsw": - do: @@ -1320,6 +1469,103 @@ setup: ef_construction: 200 confidence_interval: 0.3 +--- +"Allowed dense vector updates on same type but different other index_options, int4_hnsw": + - requires: + cluster_features: "gte_v8.16.0" + reason: 'updatable dense vector field type for int4 was added in 8.16' + - requires: + test_runner_features: [ contains ] + - do: + indices.create: + index: test_index + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_hnsw + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int4_hnsw } + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_hnsw + m: 32 + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int4_hnsw } + - match: { test_index.mappings.properties.embedding.index_options.m: 32 } + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_hnsw + m: 32 + ef_construction: 200 + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int4_hnsw } + - match: { test_index.mappings.properties.embedding.index_options.m: 32 } + - match: { test_index.mappings.properties.embedding.index_options.ef_construction: 200 } + + - do: + catch: /illegal_argument_exception/ # fails because m = 10 is less than the current value of 32 + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int8_hnsw + ef_construction: 200 + m: 10 + + - do: + catch: /illegal_argument_exception/ # fails because m = 16 by default, which is less than the current value of 32 + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int8_hnsw + ef_construction: 200 + --- "Allowed dense vector updates on same type but different other index_options, int8_flat": - do: @@ -1363,3 +1609,492 @@ setup: - match: { test_index.mappings.properties.embedding.type: dense_vector } - match: { test_index.mappings.properties.embedding.index_options.type: int8_flat } - match: { test_index.mappings.properties.embedding.index_options.confidence_interval: 0.3 } + +--- +"Allowed dense vector updates on same type but different other index_options, int4_flat": + - requires: + cluster_features: "gte_v8.16.0" + reason: 'updatable dense vector field type for int4 was added in 8.16' + - requires: + test_runner_features: [ contains ] + - do: + indices.create: + index: test_index + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_flat + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int4_flat } + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_flat + confidence_interval: 0.3 + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int4_flat } + - match: { test_index.mappings.properties.embedding.index_options.confidence_interval: 0.3 } + +--- +"Test create and update dense vector mapping to int4 with per-doc indexing and flush": + - requires: + cluster_features: "gte_v8.16.0" + reason: 'updatable dense vector field type for int4 was added in 8.16' + - requires: + test_runner_features: [ contains ] + - do: + indices.create: + index: test_index + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: flat + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: flat } + + - do: + index: + index: test_index + id: "1" + body: + embedding: [ 1, 1, 1, 1 ] + - do: + index: + index: test_index + id: "2" + body: + embedding: [ 1, 1, 1, 2 ] + - do: + index: + index: test_index + id: "3" + body: + embedding: [ 1, 1, 1, 3 ] + - do: + index: + index: test_index + id: "4" + body: + embedding: [ 1, 1, 1, 4 ] + - do: + index: + index: test_index + id: "5" + body: + embedding: [ 1, 1, 1, 5 ] + + - do: + indices.flush: { } + + - do: + index: + index: test_index + id: "6" + body: + embedding: [ 1, 1, 1, 6 ] + - do: + index: + index: test_index + id: "7" + body: + embedding: [ 1, 1, 1, 7 ] + - do: + index: + index: test_index + id: "8" + body: + embedding: [ 1, 1, 1, 8 ] + - do: + index: + index: test_index + id: "9" + body: + embedding: [ 1, 1, 1, 9 ] + - do: + index: + index: test_index + id: "10" + body: + embedding: [ 1, 1, 1, 10 ] + + - do: + indices.flush: { } + + - do: + indices.refresh: {} + + - do: + search: + index: test_index + body: + size: 3 + query: + knn: + field: embedding + query_vector: [1, 1, 1, 1] + num_candidates: 10 + + - match: { hits.total.value: 10 } + - length: {hits.hits: 3} + - contains: { hits.hits: { _id: "1" } } + - contains: { hits.hits: { _id: "2" } } + - contains: { hits.hits: { _id: "3" } } + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_flat + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int4_flat } + + - do: + index: + index: test_index + id: "11" + body: + embedding: [ 2, 1, 1, 1 ] + - do: + index: + index: test_index + id: "12" + body: + embedding: [ 3, 1, 1, 2 ] + - do: + index: + index: test_index + id: "13" + body: + embedding: [ 4, 1, 1, 3 ] + - do: + index: + index: test_index + id: "14" + body: + embedding: [ 5, 1, 1, 4 ] + - do: + index: + index: test_index + id: "15" + body: + embedding: [ 6, 1, 1, 5 ] + + - do: + indices.flush: { } + + - do: + index: + index: test_index + id: "16" + body: + embedding: [ 7, 1, 1, 6 ] + - do: + index: + index: test_index + id: "17" + body: + embedding: [ 8, 1, 1, 7 ] + - do: + index: + index: test_index + id: "18" + body: + embedding: [ 9, 1, 1, 8 ] + - do: + index: + index: test_index + id: "19" + body: + embedding: [ 10, 1, 1, 9 ] + - do: + index: + index: test_index + id: "20" + body: + embedding: [ 1, 11, 1, 10 ] + + - do: + indices.flush: { } + + - do: + indices.refresh: {} + + - do: + search: + index: test_index + body: + size: 3 + query: + knn: + field: embedding + query_vector: [ 1, 1, 1, 1 ] + num_candidates: 20 + + - match: { hits.total.value: 20 } + - length: { hits.hits: 3 } + - contains: { hits.hits: { _id: "1" } } + - contains: { hits.hits: { _id: "11" } } + - contains: { hits.hits: { _id: "2" } } + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int8_hnsw + m: 3 + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int8_hnsw } + + - do: + index: + index: test_index + id: "21" + body: + embedding: [ 1, 1, 2, 1 ] + - do: + index: + index: test_index + id: "22" + body: + embedding: [ 1, 1, 3, 1 ] + - do: + index: + index: test_index + id: "23" + body: + embedding: [ 1, 1, 4, 1 ] + - do: + index: + index: test_index + id: "24" + body: + embedding: [ 1, 1, 5, 1 ] + - do: + index: + index: test_index + id: "25" + body: + embedding: [ 1, 1, 6, 1 ] + + - do: + indices.flush: { } + + - do: + index: + index: test_index + id: "26" + body: + embedding: [ 1, 1, 7, 1 ] + - do: + index: + index: test_index + id: "27" + body: + embedding: [ 1, 1, 8, 1 ] + - do: + index: + index: test_index + id: "28" + body: + embedding: [ 1, 1, 9, 1 ] + - do: + index: + index: test_index + id: "29" + body: + embedding: [ 1, 1, 10, 1 ] + - do: + index: + index: test_index + id: "30" + body: + embedding: [ 1, 1, 11, 1 ] + + - do: + indices.flush: { } + + - do: + indices.refresh: {} + + - do: + search: + index: test_index + body: + size: 4 + query: + knn: + field: embedding + query_vector: [ 1, 1, 1, 1 ] + num_candidates: 30 + + - match: { hits.total.value: 30 } + - length: { hits.hits: 4 } + - contains: {hits.hits: {_id: "1"}} + - contains: {hits.hits: {_id: "11"}} + - contains: {hits.hits: {_id: "2"}} + - contains: {hits.hits: {_id: "21"}} + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + index_options: + type: int4_hnsw + ef_construction: 200 + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.index_options.type: int4_hnsw } + + - do: + index: + index: test_index + id: "31" + body: + embedding: [ 1, 1, 1, 2 ] + - do: + index: + index: test_index + id: "32" + body: + embedding: [ 1, 1, 1, 3 ] + - do: + index: + index: test_index + id: "33" + body: + embedding: [ 1, 1, 1, 4 ] + - do: + index: + index: test_index + id: "34" + body: + embedding: [ 1, 1, 1, 5 ] + - do: + index: + index: test_index + id: "35" + body: + embedding: [ 1, 1, 1, 6 ] + + - do: + indices.flush: { } + + - do: + index: + index: test_index + id: "36" + body: + embedding: [ 1, 1, 1, 7 ] + - do: + index: + index: test_index + id: "37" + body: + embedding: [ 1, 1, 1, 8 ] + - do: + index: + index: test_index + id: "38" + body: + embedding: [ 1, 1, 1, 9 ] + - do: + index: + index: test_index + id: "39" + body: + embedding: [ 1, 1, 1, 10 ] + - do: + index: + index: test_index + id: "40" + body: + embedding: [ 1, 1, 1, 11 ] + + - do: + indices.flush: { } + + - do: + indices.refresh: {} + + - do: + search: + index: test_index + body: + size: 5 + query: + knn: + field: embedding + query_vector: [ 1, 1, 1, 1 ] + num_candidates: 40 + + - match: { hits.total.value: 40 } + - length: { hits.hits: 5 } + - contains: {hits.hits: {_id: "1"}} + - contains: {hits.hits: {_id: "11"}} + - contains: {hits.hits: {_id: "2"}} + - contains: {hits.hits: {_id: "21"}} + - contains: {hits.hits: {_id: "31"}} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml index 825bcecf33fce..df5d451e3a2e1 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml @@ -283,15 +283,11 @@ setup: - match: {hits.total.value: 0} --- -"Test nonexistent field": +"Test nonexistent field is match none": - requires: - cluster_features: "gte_v8.4.0" - reason: 'kNN added to search endpoint in 8.4' - - skip: cluster_features: "gte_v8.16.0" reason: 'non-existent field handling improved in 8.16' - do: - catch: bad_request search: index: test body: @@ -302,17 +298,29 @@ setup: k: 2 num_candidates: 3 - - match: { error.root_cause.0.type: "query_shard_exception" } - - match: { error.root_cause.0.reason: "failed to create query: field [nonexistent] does not exist in the mapping" } + - length: {hits.hits: 0} ---- -"Test nonexistent field is match none": - - requires: - cluster_features: "gte_v8.16.0" - reason: 'non-existent field handling improved in 8.16' - do: + indices.create: + index: test_nonexistent + body: + mappings: + properties: + name: + type: keyword + vector: + type: dense_vector + element_type: float + dims: 5 + index: true + similarity: l2_norm + settings: + index.query.parse.allow_unmapped_fields: false + + - do: + catch: bad_request search: - index: test + index: test_nonexistent body: fields: [ "name" ] knn: @@ -321,7 +329,8 @@ setup: k: 2 num_candidates: 3 - - length: {hits.hits: 0} + - match: { error.root_cause.0.type: "query_shard_exception" } + - match: { error.root_cause.0.reason: "No field mapping can be found for the field with name [nonexistent]" } --- "KNN Vector similarity search only": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_byte.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_byte.yml index 806e5ff73b355..0cedfaa873095 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_byte.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_byte.yml @@ -147,12 +147,11 @@ setup: - match: { error.root_cause.0.reason: "cannot set [search_type] when using [knn] search, since the search type is determined automatically" } --- -"Test nonexistent field": - - skip: +"Test nonexistent field is match none": + - requires: cluster_features: 'gte_v8.16.0' reason: 'non-existent field handling improved in 8.16' - do: - catch: bad_request search: index: test body: @@ -163,16 +162,29 @@ setup: k: 2 num_candidates: 3 - - match: { error.root_cause.0.type: "query_shard_exception" } - - match: { error.root_cause.0.reason: "failed to create query: field [nonexistent] does not exist in the mapping" } ---- -"Test nonexistent field is match none": - - requires: - cluster_features: 'gte_v8.16.0' - reason: 'non-existent field handling improved in 8.16' + - length: {hits.hits: 0} + - do: + indices.create: + index: test_nonexistent + body: + mappings: + properties: + name: + type: keyword + vector: + type: dense_vector + element_type: byte + dims: 5 + index: true + similarity: cosine + settings: + index.query.parse.allow_unmapped_fields: false + + - do: + catch: bad_request search: - index: test + index: test_nonexistent body: fields: [ "name" ] knn: @@ -181,7 +193,8 @@ setup: k: 2 num_candidates: 3 - - length: {hits.hits: 0} + - match: { error.root_cause.0.type: "query_shard_exception" } + - match: { error.root_cause.0.reason: "No field mapping can be found for the field with name [nonexistent]" } --- "Vector similarity search only": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/171_term_query.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/171_term_query.yml new file mode 100644 index 0000000000000..5ab65b0c69e8a --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/171_term_query.yml @@ -0,0 +1,37 @@ +--- +"case insensitive term query on blank keyword is consistent": + - requires: + cluster_features: [ "gte_v8.16.0" ] + reason: "query consistency bug fix in 8.16.0" + - do: + indices.create: + index: index_with_blank_keyword + body: + settings: + number_of_shards: 1 + mappings: + properties: + keyword_field: + type: keyword + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "index_with_blank_keyword", "_id": "1"}}' + - '{"keyword_field": ""}' + + - do: + search: + rest_total_hits_as_int: true + index: index_with_blank_keyword + body: {"query" : {"term" : {"keyword_field" : {"value": ""}}}} + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: index_with_blank_keyword + body: { "query": { "term": { "keyword_field": {"value": "", "case_insensitive": true } } } } + + - match: { hits.total: 1 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml index d4cf3ade2aa4e..5928dce2c104e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml @@ -113,6 +113,150 @@ setup: - match: { docs.1.doc._source.foo: "rab" } - match: { docs.1.doc.executed_pipelines: ["my-pipeline", "my-final-pipeline"] } +--- +"Test mapping validation": + + - skip: + features: headers + + - requires: + cluster_features: ["simulate.mapping.validation"] + reason: "ingest simulate index mapping validation added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "my-pipeline" + body: > + { + "processors": [ + ] + } + - match: { acknowledged: true } + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "my-final-pipeline" + body: > + { + "processors": [ + ] + } + - match: { acknowledged: true } + + - do: + indices.create: + index: strict_index + body: + settings: + default_pipeline: "my-pipeline" + final_pipeline: "my-final-pipeline" + number_of_shards: 2 + number_of_replicas: 0 + mappings: + dynamic: strict + properties: + foo: + type: text + + - do: + indices.create: + index: lenient_index + body: + settings: + default_pipeline: "my-pipeline" + final_pipeline: "my-final-pipeline" + number_of_shards: 2 + number_of_replicas: 0 + mappings: + dynamic: true + properties: + foo: + type: text + + - do: + cluster.health: + index: lenient_index + wait_for_status: green + + - do: + headers: + Content-Type: application/json + simulate.ingest: + body: > + { + "docs": [ + { + "_index": "strict_index", + "_id": "id", + "_source": { + "foob": "bar" + } + }, + { + "_index": "strict_index", + "_id": "id", + "_source": { + "foo": "rab" + } + } + ], + "pipeline_substitutions": { + "my-pipeline": { + "processors": [ + ] + } + } + } + - length: { docs: 2 } + - match: { docs.0.doc._source.foob: "bar" } + - match: { docs.0.doc.executed_pipelines: ["my-pipeline", "my-final-pipeline"] } + - match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" } + - match: { docs.0.doc.error.reason: "[1:9] mapping set to strict, dynamic introduction of [foob] within [_doc] is not allowed" } + - match: { docs.1.doc._source.foo: "rab" } + - match: { docs.1.doc.executed_pipelines: ["my-pipeline", "my-final-pipeline"] } + - not_exists: docs.1.doc.error + + - do: + headers: + Content-Type: application/json + simulate.ingest: + body: > + { + "docs": [ + { + "_index": "lenient_index", + "_id": "id", + "_source": { + "foob": "bar" + } + }, + { + "_index": "lenient_index", + "_id": "id", + "_source": { + "foo": "rab" + } + } + ], + "pipeline_substitutions": { + "my-pipeline": { + "processors": [ + ] + } + } + } + - length: { docs: 2 } + - match: { docs.0.doc._source.foob: "bar" } + - match: { docs.0.doc.executed_pipelines: ["my-pipeline", "my-final-pipeline"] } + - not_exists: docs.0.doc.error + - match: { docs.1.doc._source.foo: "rab" } + - match: { docs.1.doc.executed_pipelines: ["my-pipeline", "my-final-pipeline"] } + - not_exists: docs.1.doc.error + --- "Test index templates with pipelines": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index 3c76653960386..dae50704dd0d0 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -337,3 +337,20 @@ sort by tsid: - match: {hits.hits.7.sort: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o", 1619635864467]} - match: {hits.hits.7.fields._tsid: [ "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} + +--- +aggs by index_mode: + - requires: + cluster_features: ["mapper.query_index_mode"] + reason: require _index_mode metadata field + - do: + search: + index: test + body: + aggs: + modes: + terms: + field: "_index_mode" + - match: {aggregations.modes.buckets.0.key: "time_series"} + - match: {aggregations.modes.buckets.0.doc_count: 8} + diff --git a/server/build.gradle b/server/build.gradle index e62abed2bc75a..deadd0a330ef8 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -40,8 +40,6 @@ dependencies { implementation project(":libs:elasticsearch-simdvec") implementation project(':libs:elasticsearch-plugin-classloader') - // no compile dependency by server, but server defines security policy for this codebase so it i> - runtimeOnly project(":libs:elasticsearch-preallocate") // lucene api "org.apache.lucene:lucene-core:${versions.lucene}" @@ -70,7 +68,6 @@ dependencies { // access to native functions implementation project(':libs:elasticsearch-native') - api "net.java.dev.jna:jna:${versions.jna}" api "co.elastic.logging:log4j2-ecs-layout:${versions.ecsLogging}" api "co.elastic.logging:ecs-logging-core:${versions.ecsLogging}" diff --git a/server/licenses/jna-LICENSE.txt b/server/licenses/jna-LICENSE.txt deleted file mode 100644 index f433b1a53f5b8..0000000000000 --- a/server/licenses/jna-LICENSE.txt +++ /dev/null @@ -1,177 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS diff --git a/server/licenses/jna-NOTICE.txt b/server/licenses/jna-NOTICE.txt deleted file mode 100644 index 8d1c8b69c3fce..0000000000000 --- a/server/licenses/jna-NOTICE.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkAfterWriteFsyncFailureIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkAfterWriteFsyncFailureIT.java index 6a4e973d8fcc5..d531686bb5207 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkAfterWriteFsyncFailureIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkAfterWriteFsyncFailureIT.java @@ -29,6 +29,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.index.IndexSettings.INDEX_REFRESH_INTERVAL_SETTING; +import static org.elasticsearch.indices.IndicesService.WRITE_DANGLING_INDICES_INFO_SETTING; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; @@ -48,7 +49,11 @@ public static void removeDisruptFSyncFS() { PathUtilsForTesting.teardown(); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/110551") + @Override + protected Settings nodeSettings() { + return Settings.builder().put(WRITE_DANGLING_INDICES_INFO_SETTING.getKey(), false).build(); + } + public void testFsyncFailureDoesNotAdvanceLocalCheckpoints() { String indexName = randomIdentifier(); client().admin() diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java index b89cea7dff089..c4737468a766c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java @@ -105,7 +105,9 @@ public TestAction( actionFilters, Request::new, Request::new, - threadPool.executor(ThreadPool.Names.GENERIC) + threadPool.executor(ThreadPool.Names.GENERIC), + SyncGlobalCheckpointAfterOperation.DoNotSync, + PrimaryActionExecution.RejectOnOverload ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/InitialClusterStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/InitialClusterStateIT.java index 3cd7ce60d9035..97112b97cc130 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/InitialClusterStateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/InitialClusterStateIT.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequest; import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; import org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; @@ -19,8 +18,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; -import java.util.concurrent.TimeUnit; - import static org.elasticsearch.node.Node.INITIAL_STATE_TIMEOUT_SETTING; @ESIntegTestCase.ClusterScope(numDataNodes = 0, autoManageMasterNodes = false) @@ -40,10 +37,8 @@ private static void assertClusterUuid(boolean expectCommitted, String expectedVa assertEquals(expectCommitted, metadata.clusterUUIDCommitted()); assertEquals(expectedValue, metadata.clusterUUID()); - final ClusterStatsResponse response = PlainActionFuture.get( - fut -> client(nodeName).execute(TransportClusterStatsAction.TYPE, new ClusterStatsRequest(), fut), - 10, - TimeUnit.SECONDS + final ClusterStatsResponse response = safeAwait( + listener -> client(nodeName).execute(TransportClusterStatsAction.TYPE, new ClusterStatsRequest(), listener) ); assertEquals(expectedValue, response.getClusterUUID()); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDeciderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDeciderIT.java new file mode 100644 index 0000000000000..2490eade46d31 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDeciderIT.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.cluster.routing.allocation.decider; + +import org.elasticsearch.cluster.ClusterModule; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESIntegTestCase; + +@ESIntegTestCase.ClusterScope(numDataNodes = 0) +public class ClusterRebalanceAllocationDeciderIT extends ESIntegTestCase { + public void testDefault() { + internalCluster().startNode(); + assertEquals( + ClusterRebalanceAllocationDecider.ClusterRebalanceType.ALWAYS, + ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.get( + internalCluster().getInstance(ClusterService.class).getSettings() + ) + ); + } + + public void testDefaultLegacyAllocator() { + internalCluster().startNode( + Settings.builder().put(ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING.getKey(), ClusterModule.BALANCED_ALLOCATOR) + ); + assertEquals( + ClusterRebalanceAllocationDecider.ClusterRebalanceType.INDICES_ALL_ACTIVE, + ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.get( + internalCluster().getInstance(ClusterService.class).getSettings() + ) + ); + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java index 3202f5513e9ac..31dd002a6af7d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.index.IndexVersion; @@ -152,7 +153,9 @@ public void testIncreaseReplicasOverLimit() { + firstShardCount + "]/[" + dataNodes * shardsPerNode - + "] maximum normal shards open;"; + + "] maximum normal shards open; for more information, see " + + ReferenceDocs.MAX_SHARDS_PER_NODE + + ";"; assertEquals(expectedError, e.getMessage()); } Metadata clusterState = clusterAdmin().prepareState().get().getState().metadata(); @@ -211,7 +214,9 @@ public void testChangingMultipleIndicesOverLimit() { + totalShardsBefore + "]/[" + dataNodes * shardsPerNode - + "] maximum normal shards open;"; + + "] maximum normal shards open; for more information, see " + + ReferenceDocs.MAX_SHARDS_PER_NODE + + ";"; assertEquals(expectedError, e.getMessage()); } Metadata clusterState = clusterAdmin().prepareState().get().getState().metadata(); @@ -403,7 +408,9 @@ private void verifyException(int dataNodes, ShardCounts counts, IllegalArgumentE + currentShards + "]/[" + maxShards - + "] maximum normal shards open;"; + + "] maximum normal shards open; for more information, see " + + ReferenceDocs.MAX_SHARDS_PER_NODE + + ";"; assertEquals(expectedError, e.getMessage()); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java index ded319fd0848d..86e1d2e332f36 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; import org.elasticsearch.action.bulk.BulkRequest; @@ -100,7 +101,7 @@ public void testIngestStatsNamesAndTypes() throws IOException { client().bulk(bulkRequest).actionGet(); { - NodesStatsResponse nodesStatsResponse = clusterAdmin().nodesStats(new NodesStatsRequest().addMetric("ingest")).actionGet(); + NodesStatsResponse nodesStatsResponse = clusterAdmin().nodesStats(new NodesStatsRequest().addMetric(Metric.INGEST)).actionGet(); assertThat(nodesStatsResponse.getNodes().size(), equalTo(1)); NodeStats stats = nodesStatsResponse.getNodes().get(0); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverIT.java index 58d1d7d88ec55..7797371a2823b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.plugins.internal; +import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; @@ -107,7 +108,8 @@ public IndexResult index(Index index) throws IOException { config().getMapperService(), DocumentSizeAccumulator.EMPTY_INSTANCE ); - documentParsingReporter.onIndexingCompleted(index.parsedDoc()); + ParsedDocument parsedDocument = index.parsedDoc(); + documentParsingReporter.onIndexingCompleted(parsedDocument); return result; } @@ -122,15 +124,9 @@ public TestDocumentParsingProviderPlugin() {} @Override public DocumentParsingProvider getDocumentParsingProvider() { return new DocumentParsingProvider() { - - @Override - public DocumentSizeObserver newFixedSizeDocumentObserver(long normalisedBytesParsed) { - return new TestDocumentSizeObserver(); - } - @Override - public DocumentSizeObserver newDocumentSizeObserver() { - return new TestDocumentSizeObserver(); + public DocumentSizeObserver newDocumentSizeObserver(DocWriteRequest request) { + return new TestDocumentSizeObserver(0L); } @Override @@ -155,8 +151,7 @@ public TestDocumentSizeReporter(String indexName) { @Override public void onIndexingCompleted(ParsedDocument parsedDocument) { - DocumentSizeObserver documentSizeObserver = parsedDocument.getDocumentSizeObserver(); - COUNTER.addAndGet(documentSizeObserver.normalisedBytesParsed()); + COUNTER.addAndGet(parsedDocument.getDocumentSizeObserver().normalisedBytesParsed()); assertThat(indexName, equalTo(TEST_INDEX_NAME)); } } @@ -164,10 +159,15 @@ public void onIndexingCompleted(ParsedDocument parsedDocument) { public static class TestDocumentSizeObserver implements DocumentSizeObserver { long counter = 0; + public TestDocumentSizeObserver(long counter) { + this.counter = counter; + } + @Override public XContentParser wrapParser(XContentParser xContentParser) { hasWrappedParser = true; return new FilterXContentParserWrapper(xContentParser) { + @Override public Token nextToken() throws IOException { counter++; @@ -180,5 +180,6 @@ public Token nextToken() throws IOException { public long normalisedBytesParsed() { return counter; } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java index 5b44a949ab784..8335b3c0c4249 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java @@ -7,7 +7,6 @@ */ package org.elasticsearch.readiness; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; @@ -397,24 +396,24 @@ public void testReadyAfterCorrectFileSettings() throws Exception { } private void causeClusterStateUpdate() { - PlainActionFuture.get( - fut -> internalCluster().getCurrentMasterNodeInstance(ClusterService.class) - .submitUnbatchedStateUpdateTask("poke", new ClusterStateUpdateTask() { - @Override - public ClusterState execute(ClusterState currentState) { - return ClusterState.builder(currentState).build(); - } - - @Override - public void onFailure(Exception e) { - assert false : e; - } - - @Override - public void clusterStateProcessed(ClusterState initialState, ClusterState newState) { - fut.onResponse(null); - } - }) - ); + final var latch = new CountDownLatch(1); + internalCluster().getCurrentMasterNodeInstance(ClusterService.class) + .submitUnbatchedStateUpdateTask("poke", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) { + return ClusterState.builder(currentState).build(); + } + + @Override + public void onFailure(Exception e) { + assert false : e; + } + + @Override + public void clusterStateProcessed(ClusterState initialState, ClusterState newState) { + latch.countDown(); + } + }); + safeAwait(latch); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionIT.java new file mode 100644 index 0000000000000..7e9fa065bb4dd --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionIT.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.rest.action.admin.cluster; + +import org.elasticsearch.action.admin.cluster.allocation.TransportGetAllocationStatsAction; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.transport.MockTransportService; + +import java.util.Collection; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.Matchers.equalTo; + +public class RestNodesStatsActionIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return CollectionUtils.appendToCopy(super.nodePlugins(), MockTransportService.TestPlugin.class); + } + + public void testSendOnlyNecessaryElectedMasterNodeStatsRequest() { + var node = internalCluster().startDataOnlyNode(); + + var getAllocationStatsActions = new AtomicInteger(0); + MockTransportService.getInstance(node).addSendBehavior((connection, requestId, action, request, options) -> { + if (Objects.equals(action, TransportGetAllocationStatsAction.TYPE.name())) { + getAllocationStatsActions.incrementAndGet(); + } + connection.sendRequest(requestId, action, request, options); + }); + + var metrics = randomSubsetOf(Metric.values().length, Metric.values()); + client(node).admin().cluster().nodesStats(new NodesStatsRequest().addMetrics(metrics)).actionGet(); + + var shouldSendGetAllocationStatsRequest = metrics.contains(Metric.ALLOCATIONS) || metrics.contains(Metric.FS); + assertThat(getAllocationStatsActions.get(), equalTo(shouldSendGetAllocationStatsRequest ? 1 : 0)); + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/CollapseSearchResultsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/CollapseSearchResultsIT.java index f5fdd752a6f57..aa721122c2160 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/CollapseSearchResultsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/CollapseSearchResultsIT.java @@ -61,4 +61,27 @@ public void testCollapseWithDocValueFields() { } ); } + + public void testCollapseWithFields() { + final String indexName = "test_collapse"; + createIndex(indexName); + final String collapseField = "collapse_field"; + final String otherField = "other_field"; + assertAcked(indicesAdmin().preparePutMapping(indexName).setSource(collapseField, "type=keyword", otherField, "type=keyword")); + index(indexName, "id_1_0", Map.of(collapseField, "value1", otherField, "other_value1")); + index(indexName, "id_1_1", Map.of(collapseField, "value1", otherField, "other_value2")); + index(indexName, "id_2_0", Map.of(collapseField, "value2", otherField, "other_value3")); + refresh(indexName); + + assertNoFailuresAndResponse( + prepareSearch(indexName).setQuery(new MatchAllQueryBuilder()) + .setFetchSource(false) + .addFetchField(otherField) + .setCollapse(new CollapseBuilder(collapseField).setInnerHits(new InnerHitBuilder("ih").setSize(2))), + searchResponse -> { + assertEquals(collapseField, searchResponse.getHits().getCollapseField()); + assertEquals(Set.of(new BytesRef("value1"), new BytesRef("value2")), Set.of(searchResponse.getHits().getCollapseValues())); + } + ); + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java index 0a6fceea9a3f1..d9d6979ffd710 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java @@ -2177,6 +2177,15 @@ public void testHighlightNoMatchSize() throws IOException { field.highlighterType("unified"); assertNotHighlighted(prepareSearch("test").highlighter(new HighlightBuilder().field(field)), 0, "text"); + + // Check when the requested fragment size equals the size of the string + var anotherText = "I am unusual and don't end with your regular )token)"; + indexDoc("test", "1", "text", anotherText); + refresh(); + for (String type : new String[] { "plain", "unified", "fvh" }) { + field.highlighterType(type).noMatchSize(anotherText.length()).numOfFragments(0); + assertHighlight(prepareSearch("test").highlighter(new HighlightBuilder().field(field)), 0, "text", 0, 1, equalTo(anotherText)); + } } public void testHighlightNoMatchSizeWithMultivaluedFields() throws IOException { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CloneSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CloneSnapshotIT.java index a16a19f66085b..d7c7acf9737a1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CloneSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CloneSnapshotIT.java @@ -8,11 +8,9 @@ package org.elasticsearch.snapshots; import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotIndexStatus; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.SnapshotsInProgress; @@ -33,6 +31,7 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; +import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; @@ -78,8 +77,14 @@ public void testShardClone() throws Exception { } else { currentShardGen = repositoryData.shardGenerations().getShardGen(indexId, shardId); } - final ShardSnapshotResult shardSnapshotResult = PlainActionFuture.get( - f -> repository.cloneShardSnapshot(sourceSnapshotInfo.snapshotId(), targetSnapshotId, repositoryShardId, currentShardGen, f) + final ShardSnapshotResult shardSnapshotResult = safeAwait( + listener -> repository.cloneShardSnapshot( + sourceSnapshotInfo.snapshotId(), + targetSnapshotId, + repositoryShardId, + currentShardGen, + listener + ) ); final ShardGeneration newShardGeneration = shardSnapshotResult.getGeneration(); @@ -107,8 +112,14 @@ public void testShardClone() throws Exception { assertTrue(snapshotFiles.get(0).isSame(snapshotFiles.get(1))); // verify that repeated cloning is idempotent - final ShardSnapshotResult shardSnapshotResult2 = PlainActionFuture.get( - f -> repository.cloneShardSnapshot(sourceSnapshotInfo.snapshotId(), targetSnapshotId, repositoryShardId, newShardGeneration, f) + final ShardSnapshotResult shardSnapshotResult2 = safeAwait( + listener -> repository.cloneShardSnapshot( + sourceSnapshotInfo.snapshotId(), + targetSnapshotId, + repositoryShardId, + newShardGeneration, + listener + ) ); assertEquals(newShardGeneration, shardSnapshotResult2.getGeneration()); assertEquals(shardSnapshotResult.getSegmentCount(), shardSnapshotResult2.getSegmentCount()); @@ -640,7 +651,7 @@ public void testStartCloneWithSuccessfulShardSnapshotPendingFinalization() throw try { awaitClusterState(clusterState -> { final List entries = SnapshotsInProgress.get(clusterState).forRepo(repoName); - return entries.size() == 2 && entries.get(1).shardsByRepoShardId().isEmpty() == false; + return entries.size() == 2 && entries.get(1).shardSnapshotStatusByRepoShardId().isEmpty() == false; }); assertFalse(blockedSnapshot.isDone()); } finally { @@ -677,9 +688,9 @@ public void testStartCloneDuringRunningDelete() throws Exception { logger.info("--> waiting for snapshot clone to be fully initialized"); awaitClusterState(state -> { for (SnapshotsInProgress.Entry entry : SnapshotsInProgress.get(state).forRepo(repoName)) { - if (entry.shardsByRepoShardId().isEmpty() == false) { + if (entry.shardSnapshotStatusByRepoShardId().isEmpty() == false) { assertEquals(sourceSnapshot, entry.source().getName()); - for (SnapshotsInProgress.ShardSnapshotStatus value : entry.shardsByRepoShardId().values()) { + for (SnapshotsInProgress.ShardSnapshotStatus value : entry.shardSnapshotStatusByRepoShardId().values()) { assertSame(value, SnapshotsInProgress.ShardSnapshotStatus.UNASSIGNED_QUEUED); } return true; @@ -880,21 +891,12 @@ private static BlobStoreIndexShardSnapshots readShardGeneration( BlobStoreRepository repository, RepositoryShardId repositoryShardId, ShardGeneration generation - ) { - return PlainActionFuture.get( - f -> repository.threadPool() - .generic() - .execute( - ActionRunnable.supply( - f, - () -> BlobStoreRepository.INDEX_SHARD_SNAPSHOTS_FORMAT.read( - repository.getMetadata().name(), - repository.shardContainer(repositoryShardId.index(), repositoryShardId.shardId()), - generation.toBlobNamePart(), - NamedXContentRegistry.EMPTY - ) - ) - ) + ) throws IOException { + return BlobStoreRepository.INDEX_SHARD_SNAPSHOTS_FORMAT.read( + repository.getMetadata().name(), + repository.shardContainer(repositoryShardId.index(), repositoryShardId.shardId()), + generation.getGenerationUUID(), + NamedXContentRegistry.EMPTY ); } @@ -903,18 +905,6 @@ private static BlobStoreIndexShardSnapshot readShardSnapshot( RepositoryShardId repositoryShardId, SnapshotId snapshotId ) { - return PlainActionFuture.get( - f -> repository.threadPool() - .generic() - .execute( - ActionRunnable.supply( - f, - () -> repository.loadShardSnapshot( - repository.shardContainer(repositoryShardId.index(), repositoryShardId.shardId()), - snapshotId - ) - ) - ) - ); + return repository.loadShardSnapshot(repository.shardContainer(repositoryShardId.index(), repositoryShardId.shardId()), snapshotId); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java index 01a18a58f663c..abcac0cade456 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java @@ -8,12 +8,10 @@ package org.elasticsearch.snapshots; import org.elasticsearch.action.ActionRequestBuilder; -import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.Metadata; @@ -21,6 +19,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Strings; import org.elasticsearch.index.IndexVersion; @@ -33,7 +32,6 @@ import org.elasticsearch.repositories.ShardGenerations; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.fs.FsRepository; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentFactory; import java.nio.channels.SeekableByteChannel; @@ -308,18 +306,8 @@ public void testHandlingMissingRootLevelSnapshotMetadata() throws Exception { ); logger.info("--> verify that repo is assumed in old metadata format"); - final ThreadPool threadPool = internalCluster().getCurrentMasterNodeInstance(ThreadPool.class); assertThat( - PlainActionFuture.get( - // any other executor than generic and management - f -> threadPool.executor(ThreadPool.Names.SNAPSHOT) - .execute( - ActionRunnable.supply( - f, - () -> SnapshotsService.minCompatibleVersion(IndexVersion.current(), getRepositoryData(repoName), null) - ) - ) - ), + SnapshotsService.minCompatibleVersion(IndexVersion.current(), getRepositoryData(repoName), null), is(SnapshotsService.OLD_SNAPSHOT_FORMAT) ); @@ -328,15 +316,7 @@ public void testHandlingMissingRootLevelSnapshotMetadata() throws Exception { logger.info("--> verify that repository is assumed in new metadata format after removing corrupted snapshot"); assertThat( - PlainActionFuture.get( - f -> threadPool.generic() - .execute( - ActionRunnable.supply( - f, - () -> SnapshotsService.minCompatibleVersion(IndexVersion.current(), getRepositoryData(repoName), null) - ) - ) - ), + SnapshotsService.minCompatibleVersion(IndexVersion.current(), getRepositoryData(repoName), null), is(IndexVersion.current()) ); final RepositoryData finalRepositoryData = getRepositoryData(repoName); @@ -381,7 +361,10 @@ public void testMountCorruptedRepositoryData() throws Exception { Files.write(repo.resolve("index-" + repositoryData.getGenId()), randomByteArrayOfLength(randomIntBetween(1, 100))); logger.info("--> verify loading repository data throws RepositoryException"); - expectThrows(RepositoryException.class, () -> getRepositoryData(repository)); + asInstanceOf( + RepositoryException.class, + safeAwaitFailure(RepositoryData.class, l -> repository.getRepositoryData(EsExecutors.DIRECT_EXECUTOR_SERVICE, l)) + ); final String otherRepoName = "other-repo"; assertAcked( @@ -393,7 +376,10 @@ public void testMountCorruptedRepositoryData() throws Exception { final Repository otherRepo = getRepositoryOnMaster(otherRepoName); logger.info("--> verify loading repository data from newly mounted repository throws RepositoryException"); - expectThrows(RepositoryException.class, () -> getRepositoryData(otherRepo)); + asInstanceOf( + RepositoryException.class, + safeAwaitFailure(RepositoryData.class, l -> repository.getRepositoryData(EsExecutors.DIRECT_EXECUTOR_SERVICE, l)) + ); } public void testHandleSnapshotErrorWithBwCFormat() throws Exception { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java index a96d127429b75..057d7124f83d9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java @@ -482,7 +482,7 @@ public void run() { // We must wait for all the cleanup work to be enqueued (with the throttled runner at least) so we can be sure of exactly how it // will execute. The cleanup work is enqueued by the master service thread on completion of the cluster state update which increases // the root blob generation in the repo metadata, so it is sufficient to wait for another no-op task to run on the master service: - PlainActionFuture.get(fut -> clusterService.createTaskQueue("test", Priority.NORMAL, new SimpleBatchedExecutor<>() { + safeAwait(listener -> clusterService.createTaskQueue("test", Priority.NORMAL, new SimpleBatchedExecutor<>() { @Override public Tuple executeTask(ClusterStateTaskListener clusterStateTaskListener, ClusterState clusterState) { return Tuple.tuple(clusterState, null); @@ -490,9 +490,9 @@ public Tuple executeTask(ClusterStateTaskListener clusterS @Override public void taskSucceeded(ClusterStateTaskListener clusterStateTaskListener, Object ignored) { - fut.onResponse(null); + listener.onResponse(null); } - }).submitTask("test", e -> fail(), null), 10, TimeUnit.SECONDS); + }).submitTask("test", e -> fail(), null)); final IntSupplier queueLength = () -> threadPool.stats() .stats() diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java index 7f90b57204fc8..a45471f273732 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java @@ -19,7 +19,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.action.support.ActionTestUtils; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; @@ -88,18 +87,16 @@ public void testRestartNodeDuringSnapshot() throws Exception { }); addUnassignedShardsWatcher(clusterService, indexName); - PlainActionFuture.get( - fut -> putShutdownMetadata( + safeAwait( + (ActionListener listener) -> putShutdownMetadata( clusterService, SingleNodeShutdownMetadata.builder() .setType(SingleNodeShutdownMetadata.Type.RESTART) .setStartedAtMillis(clusterService.threadPool().absoluteTimeInMillis()) .setReason("test"), originalNode, - fut - ), - 10, - TimeUnit.SECONDS + listener + ) ); assertFalse(snapshotCompletesWithoutPausingListener.isDone()); unblockAllDataNodes(repoName); // lets the shard snapshot continue so the snapshot can succeed @@ -451,11 +448,7 @@ private static void addUnassignedShardsWatcher(ClusterService clusterService, St } private static void putShutdownForRemovalMetadata(String nodeName, ClusterService clusterService) { - PlainActionFuture.get( - fut -> putShutdownForRemovalMetadata(clusterService, nodeName, fut), - 10, - TimeUnit.SECONDS - ); + safeAwait((ActionListener listener) -> putShutdownForRemovalMetadata(clusterService, nodeName, listener)); } private static void flushMasterQueue(ClusterService clusterService, ActionListener listener) { @@ -525,7 +518,7 @@ public void clusterStateProcessed(ClusterState initialState, ClusterState newSta } private static void clearShutdownMetadata(ClusterService clusterService) { - PlainActionFuture.get(fut -> clusterService.submitUnbatchedStateUpdateTask("remove restart marker", new ClusterStateUpdateTask() { + safeAwait(listener -> clusterService.submitUnbatchedStateUpdateTask("remove restart marker", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { return currentState.copyAndUpdateMetadata(mdb -> mdb.putCustom(NodesShutdownMetadata.TYPE, NodesShutdownMetadata.EMPTY)); @@ -538,8 +531,8 @@ public void onFailure(Exception e) { @Override public void clusterStateProcessed(ClusterState initialState, ClusterState newState) { - fut.onResponse(null); + listener.onResponse(null); } - }), 10, TimeUnit.SECONDS); + })); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java b/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java index 41fc3d2b759ff..339187f56d8c3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java @@ -29,7 +29,6 @@ import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; @@ -585,85 +584,70 @@ public void testRandomIDsAndVersions() throws Exception { } final AtomicInteger upto = new AtomicInteger(); - final CountDownLatch startingGun = new CountDownLatch(1); - Thread[] threads = new Thread[TestUtil.nextInt(random, 1, TEST_NIGHTLY ? 20 : 5)]; final long startTime = System.nanoTime(); - for (int i = 0; i < threads.length; i++) { - final int threadID = i; - threads[i] = new Thread() { - @Override - public void run() { - try { - // final Random threadRandom = RandomizedContext.current().getRandom(); - final Random threadRandom = random(); - startingGun.await(); - while (true) { - - // TODO: sometimes use bulk: - - int index = upto.getAndIncrement(); - if (index >= idVersions.length) { - break; - } - if (index % 100 == 0) { - logger.trace("{}: index={}", Thread.currentThread().getName(), index); - } - IDAndVersion idVersion = idVersions[index]; - - String id = idVersion.id; - idVersion.threadID = threadID; - idVersion.indexStartTime = System.nanoTime() - startTime; - long version = idVersion.version; - if (idVersion.delete) { - try { - idVersion.response = client().prepareDelete("test", id) - .setVersion(version) - .setVersionType(VersionType.EXTERNAL) - .get(); - } catch (VersionConflictEngineException vcee) { - // OK: our version is too old - assertThat(version, lessThanOrEqualTo(truth.get(id).version)); - idVersion.versionConflict = true; - } - } else { - try { - idVersion.response = prepareIndex("test").setId(id) - .setSource("foo", "bar") - .setVersion(version) - .setVersionType(VersionType.EXTERNAL) - .get(); - - } catch (VersionConflictEngineException vcee) { - // OK: our version is too old - assertThat(version, lessThanOrEqualTo(truth.get(id).version)); - idVersion.versionConflict = true; - } - } - idVersion.indexFinishTime = System.nanoTime() - startTime; - - if (threadRandom.nextInt(100) == 7) { - logger.trace("--> {}: TEST: now refresh at {}", threadID, System.nanoTime() - startTime); - refresh(); - logger.trace("--> {}: TEST: refresh done at {}", threadID, System.nanoTime() - startTime); - } - if (threadRandom.nextInt(100) == 7) { - logger.trace("--> {}: TEST: now flush at {}", threadID, System.nanoTime() - startTime); - flush(); - logger.trace("--> {}: TEST: flush done at {}", threadID, System.nanoTime() - startTime); - } + startInParallel(TestUtil.nextInt(random, 1, TEST_NIGHTLY ? 20 : 5), threadID -> { + try { + // final Random threadRandom = RandomizedContext.current().getRandom(); + final Random threadRandom = random(); + while (true) { + + // TODO: sometimes use bulk: + + int index = upto.getAndIncrement(); + if (index >= idVersions.length) { + break; + } + if (index % 100 == 0) { + logger.trace("{}: index={}", Thread.currentThread().getName(), index); + } + IDAndVersion idVersion = idVersions[index]; + + String id = idVersion.id; + idVersion.threadID = threadID; + idVersion.indexStartTime = System.nanoTime() - startTime; + long v = idVersion.version; + if (idVersion.delete) { + try { + idVersion.response = client().prepareDelete("test", id) + .setVersion(v) + .setVersionType(VersionType.EXTERNAL) + .get(); + } catch (VersionConflictEngineException vcee) { + // OK: our version is too old + assertThat(v, lessThanOrEqualTo(truth.get(id).version)); + idVersion.versionConflict = true; + } + } else { + try { + idVersion.response = prepareIndex("test").setId(id) + .setSource("foo", "bar") + .setVersion(v) + .setVersionType(VersionType.EXTERNAL) + .get(); + + } catch (VersionConflictEngineException vcee) { + // OK: our version is too old + assertThat(v, lessThanOrEqualTo(truth.get(id).version)); + idVersion.versionConflict = true; } - } catch (Exception e) { - throw new RuntimeException(e); } - } - }; - threads[i].start(); - } + idVersion.indexFinishTime = System.nanoTime() - startTime; - startingGun.countDown(); - for (Thread thread : threads) { - thread.join(); - } + if (threadRandom.nextInt(100) == 7) { + logger.trace("--> {}: TEST: now refresh at {}", threadID, System.nanoTime() - startTime); + refresh(); + logger.trace("--> {}: TEST: refresh done at {}", threadID, System.nanoTime() - startTime); + } + if (threadRandom.nextInt(100) == 7) { + logger.trace("--> {}: TEST: now flush at {}", threadID, System.nanoTime() - startTime); + flush(); + logger.trace("--> {}: TEST: flush done at {}", threadID, System.nanoTime() - startTime); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); // Verify against truth: boolean failed = false; diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index aaf8b3d0c8d84..1c07b5b4564ec 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -34,7 +34,6 @@ requires org.elasticsearch.tdigest; requires org.elasticsearch.simdvec; - requires com.sun.jna; requires hppc; requires HdrHistogram; requires jopt.simple; @@ -189,7 +188,6 @@ exports org.elasticsearch.common.compress; exports org.elasticsearch.common.document; exports org.elasticsearch.common.file; - exports org.elasticsearch.common.filesystem; exports org.elasticsearch.common.geo; exports org.elasticsearch.common.hash; exports org.elasticsearch.common.inject; @@ -423,6 +421,7 @@ provides org.elasticsearch.features.FeatureSpecification with + org.elasticsearch.action.bulk.BulkFeatures, org.elasticsearch.features.FeatureInfrastructureFeatures, org.elasticsearch.health.HealthFeatures, org.elasticsearch.cluster.service.TransportFeatures, @@ -431,6 +430,7 @@ org.elasticsearch.indices.IndicesFeatures, org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures, org.elasticsearch.index.mapper.MapperFeatures, + org.elasticsearch.ingest.IngestGeoIpFeatures, org.elasticsearch.search.SearchFeatures, org.elasticsearch.script.ScriptFeatures, org.elasticsearch.search.retriever.RetrieversFeatures, @@ -464,4 +464,5 @@ org.elasticsearch.serverless.shardhealth, org.elasticsearch.serverless.apifiltering; exports org.elasticsearch.lucene.spatial; + } diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchException.java b/server/src/main/java/org/elasticsearch/ElasticsearchException.java index 2983a2d62de71..046c049bff0d8 100644 --- a/server/src/main/java/org/elasticsearch/ElasticsearchException.java +++ b/server/src/main/java/org/elasticsearch/ElasticsearchException.java @@ -1908,19 +1908,19 @@ private enum ElasticsearchExceptionHandle { FailureIndexNotSupportedException.class, FailureIndexNotSupportedException::new, 178, - TransportVersions.ADD_FAILURE_STORE_INDICES_OPTIONS + TransportVersions.V_8_14_0 ), NOT_PERSISTENT_TASK_NODE_EXCEPTION( NotPersistentTaskNodeException.class, NotPersistentTaskNodeException::new, 179, - TransportVersions.ADD_PERSISTENT_TASK_EXCEPTIONS + TransportVersions.V_8_14_0 ), PERSISTENT_TASK_NODE_NOT_ASSIGNED_EXCEPTION( PersistentTaskNodeNotAssignedException.class, PersistentTaskNodeNotAssignedException::new, 180, - TransportVersions.ADD_PERSISTENT_TASK_EXCEPTIONS + TransportVersions.V_8_14_0 ), RESOURCE_ALREADY_UPLOADED_EXCEPTION( ResourceAlreadyUploadedException.class, diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 65606465b8502..4889709b89259 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -100,49 +100,8 @@ static TransportVersion def(int id) { public static final TransportVersion V_8_12_1 = def(8_560_00_1); public static final TransportVersion V_8_13_0 = def(8_595_00_0); public static final TransportVersion V_8_13_4 = def(8_595_00_1); - // 8.14.0+ - public static final TransportVersion RANDOM_AGG_SHARD_SEED = def(8_596_00_0); - public static final TransportVersion ESQL_TIMINGS = def(8_597_00_0); - public static final TransportVersion DATA_STREAM_AUTO_SHARDING_EVENT = def(8_598_00_0); - public static final TransportVersion ADD_FAILURE_STORE_INDICES_OPTIONS = def(8_599_00_0); - public static final TransportVersion ESQL_ENRICH_OPERATOR_STATUS = def(8_600_00_0); - public static final TransportVersion ESQL_SERIALIZE_ARRAY_VECTOR = def(8_601_00_0); - public static final TransportVersion ESQL_SERIALIZE_ARRAY_BLOCK = def(8_602_00_0); - public static final TransportVersion ADD_DATA_STREAM_GLOBAL_RETENTION = def(8_603_00_0); - public static final TransportVersion ALLOCATION_STATS = def(8_604_00_0); - public static final TransportVersion ESQL_EXTENDED_ENRICH_TYPES = def(8_605_00_0); - public static final TransportVersion KNN_EXPLICIT_BYTE_QUERY_VECTOR_PARSING = def(8_606_00_0); - public static final TransportVersion ESQL_EXTENDED_ENRICH_INPUT_TYPE = def(8_607_00_0); - public static final TransportVersion ESQL_SERIALIZE_BIG_VECTOR = def(8_608_00_0); - public static final TransportVersion AGGS_EXCLUDED_DELETED_DOCS = def(8_609_00_0); - public static final TransportVersion ESQL_SERIALIZE_BIG_ARRAY = def(8_610_00_0); - public static final TransportVersion AUTO_SHARDING_ROLLOVER_CONDITION = def(8_611_00_0); - public static final TransportVersion KNN_QUERY_VECTOR_BUILDER = def(8_612_00_0); - public static final TransportVersion USE_DATA_STREAM_GLOBAL_RETENTION = def(8_613_00_0); - public static final TransportVersion ML_COMPLETION_INFERENCE_SERVICE_ADDED = def(8_614_00_0); - public static final TransportVersion ML_INFERENCE_EMBEDDING_BYTE_ADDED = def(8_615_00_0); - public static final TransportVersion ML_INFERENCE_L2_NORM_SIMILARITY_ADDED = def(8_616_00_0); - public static final TransportVersion SEARCH_NODE_LOAD_AUTOSCALING = def(8_617_00_0); - public static final TransportVersion ESQL_ES_SOURCE_OPTIONS = def(8_618_00_0); - public static final TransportVersion ADD_PERSISTENT_TASK_EXCEPTIONS = def(8_619_00_0); - public static final TransportVersion ESQL_REDUCER_NODE_FRAGMENT = def(8_620_00_0); - public static final TransportVersion FAILURE_STORE_ROLLOVER = def(8_621_00_0); - public static final TransportVersion CCR_STATS_API_TIMEOUT_PARAM = def(8_622_00_0); - public static final TransportVersion ESQL_ORDINAL_BLOCK = def(8_623_00_0); - public static final TransportVersion ML_INFERENCE_COHERE_RERANK = def(8_624_00_0); - public static final TransportVersion INDEXING_PRESSURE_DOCUMENT_REJECTIONS_COUNT = def(8_625_00_0); - public static final TransportVersion ALIAS_ACTION_RESULTS = def(8_626_00_0); - public static final TransportVersion HISTOGRAM_AGGS_KEY_SORTED = def(8_627_00_0); - public static final TransportVersion INFERENCE_FIELDS_METADATA = def(8_628_00_0); - public static final TransportVersion ML_INFERENCE_TIMEOUT_ADDED = def(8_629_00_0); - public static final TransportVersion MODIFY_DATA_STREAM_FAILURE_STORES = def(8_630_00_0); - public static final TransportVersion ML_INFERENCE_RERANK_NEW_RESPONSE_FORMAT = def(8_631_00_0); - public static final TransportVersion HIGHLIGHTERS_TAGS_ON_FIELD_LEVEL = def(8_632_00_0); - public static final TransportVersion TRACK_FLUSH_TIME_EXCLUDING_WAITING_ON_LOCKS = def(8_633_00_0); - public static final TransportVersion ML_INFERENCE_AZURE_OPENAI_EMBEDDINGS = def(8_634_00_0); - public static final TransportVersion ILM_SHRINK_ENABLE_WRITE = def(8_635_00_0); - public static final TransportVersion GEOIP_CACHE_STATS = def(8_636_00_0); - public static final TransportVersion SHUTDOWN_REQUEST_TIMEOUTS_FIX_8_14 = def(8_636_00_1); + public static final TransportVersion V_8_14_0 = def(8_636_00_1); + // 8.15.0+ public static final TransportVersion WATERMARK_THRESHOLDS_STATS = def(8_637_00_0); public static final TransportVersion ENRICH_CACHE_ADDITIONAL_STATS = def(8_638_00_0); public static final TransportVersion ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED = def(8_639_00_0); @@ -209,8 +168,16 @@ static TransportVersion def(int id) { public static final TransportVersion ML_INFERENCE_GOOGLE_VERTEX_AI_RERANKING_ADDED = def(8_700_00_0); public static final TransportVersion VERSIONED_MASTER_NODE_REQUESTS = def(8_701_00_0); public static final TransportVersion ML_INFERENCE_AMAZON_BEDROCK_ADDED = def(8_702_00_0); + public static final TransportVersion ENTERPRISE_GEOIP_DOWNLOADER_BACKPORT_8_15 = def(8_702_00_1); public static final TransportVersion ML_INFERENCE_DONT_DELETE_WHEN_SEMANTIC_TEXT_EXISTS = def(8_703_00_0); public static final TransportVersion INFERENCE_ADAPTIVE_ALLOCATIONS = def(8_704_00_0); + public static final TransportVersion INDEX_REQUEST_UPDATE_BY_SCRIPT_ORIGIN = def(8_705_00_0); + public static final TransportVersion ML_INFERENCE_COHERE_UNUSED_RERANK_SETTINGS_REMOVED = def(8_706_00_0); + public static final TransportVersion ENRICH_CACHE_STATS_SIZE_ADDED = def(8_707_00_0); + public static final TransportVersion ENTERPRISE_GEOIP_DOWNLOADER = def(8_708_00_0); + public static final TransportVersion NODES_STATS_ENUM_SET = def(8_709_00_0); + public static final TransportVersion MASTER_NODE_METRICS = def(8_710_00_0); + public static final TransportVersion SEGMENT_LEVEL_FIELDS_STATS = def(8_711_00_0); /* * STOP! READ THIS FIRST! No, really, @@ -275,7 +242,7 @@ static TransportVersion def(int id) { * Reference to the minimum transport version that can be used with CCS. * This should be the transport version used by the previous minor release. */ - public static final TransportVersion MINIMUM_CCS_VERSION = SHUTDOWN_REQUEST_TIMEOUTS_FIX_8_14; + public static final TransportVersion MINIMUM_CCS_VERSION = V_8_14_0; static final NavigableMap VERSION_IDS = getAllVersionIds(TransportVersions.class); diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 00ffcdd0f4d9e..fefe2ea486485 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -179,6 +179,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_14_1 = new Version(8_14_01_99); public static final Version V_8_14_2 = new Version(8_14_02_99); public static final Version V_8_14_3 = new Version(8_14_03_99); + public static final Version V_8_14_4 = new Version(8_14_04_99); public static final Version V_8_15_0 = new Version(8_15_00_99); public static final Version V_8_16_0 = new Version(8_16_00_99); public static final Version CURRENT = V_8_16_0; diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index b550755ce7bdd..a9c6894355cb6 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -30,6 +30,7 @@ import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction; import org.elasticsearch.action.admin.cluster.migration.TransportPostFeatureUpgradeAction; import org.elasticsearch.action.admin.cluster.node.capabilities.TransportNodesCapabilitiesAction; +import org.elasticsearch.action.admin.cluster.node.features.TransportNodesFeaturesAction; import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction; import org.elasticsearch.action.admin.cluster.node.reload.TransportNodesReloadSecureSettingsAction; @@ -621,6 +622,7 @@ public void reg actions.register(TransportNodesInfoAction.TYPE, TransportNodesInfoAction.class); actions.register(TransportRemoteInfoAction.TYPE, TransportRemoteInfoAction.class); actions.register(TransportNodesCapabilitiesAction.TYPE, TransportNodesCapabilitiesAction.class); + actions.register(TransportNodesFeaturesAction.TYPE, TransportNodesFeaturesAction.class); actions.register(RemoteClusterNodesAction.TYPE, RemoteClusterNodesAction.TransportAction.class); actions.register(TransportNodesStatsAction.TYPE, TransportNodesStatsAction.class); actions.register(TransportNodesUsageAction.TYPE, TransportNodesUsageAction.class); diff --git a/server/src/main/java/org/elasticsearch/action/NoShardAvailableActionException.java b/server/src/main/java/org/elasticsearch/action/NoShardAvailableActionException.java index e018cf48fcefc..564055aa36750 100644 --- a/server/src/main/java/org/elasticsearch/action/NoShardAvailableActionException.java +++ b/server/src/main/java/org/elasticsearch/action/NoShardAvailableActionException.java @@ -18,8 +18,6 @@ public final class NoShardAvailableActionException extends ElasticsearchException { - private static final StackTraceElement[] EMPTY_STACK_TRACE = new StackTraceElement[0]; - // This is set so that no StackTrace is serialized in the scenario when we wrap other shard failures. // It isn't necessary to serialize this field over the wire as the empty stack trace is serialized instead. private final boolean onShardFailureWrapper; @@ -57,8 +55,8 @@ public NoShardAvailableActionException(StreamInput in) throws IOException { } @Override - public StackTraceElement[] getStackTrace() { - return onShardFailureWrapper ? EMPTY_STACK_TRACE : super.getStackTrace(); + public Throwable fillInStackTrace() { + return this; // this exception doesn't imply a bug, no need for a stack trace } @Override @@ -67,7 +65,7 @@ public void printStackTrace(PrintWriter s) { super.printStackTrace(s); } else { // Override to simply print the first line of the trace, which is the current exception. - // Since we aren't serializing the repetitive stacktrace onShardFailureWrapper, we shouldn't print it out either + // Note: This will also omit the cause chain or any suppressed exceptions. s.println(this); } } diff --git a/server/src/main/java/org/elasticsearch/action/ResolvedIndices.java b/server/src/main/java/org/elasticsearch/action/ResolvedIndices.java index 2ff0b476dc60b..822a75c4dec42 100644 --- a/server/src/main/java/org/elasticsearch/action/ResolvedIndices.java +++ b/server/src/main/java/org/elasticsearch/action/ResolvedIndices.java @@ -27,6 +27,10 @@ import java.util.Map; import java.util.Set; +/** + * Container for information about results of the resolution of index expression. + * Contains local indices, map of remote indices and metadata. + */ public class ResolvedIndices { @Nullable private final SearchContextId searchContextId; diff --git a/server/src/main/java/org/elasticsearch/action/UnavailableShardsException.java b/server/src/main/java/org/elasticsearch/action/UnavailableShardsException.java index b4120804c20df..647e98e3599f5 100644 --- a/server/src/main/java/org/elasticsearch/action/UnavailableShardsException.java +++ b/server/src/main/java/org/elasticsearch/action/UnavailableShardsException.java @@ -45,4 +45,9 @@ public UnavailableShardsException(StreamInput in) throws IOException { public RestStatus status() { return RestStatus.SERVICE_UNAVAILABLE; } + + @Override + public Throwable fillInStackTrace() { + return this; // this exception doesn't imply a bug, no need for a stack trace + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java index 560ef6feae1e4..bf10149517bb8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; @@ -36,6 +37,7 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.EnumSet; import java.util.Map; public class TransportGetAllocationStatsAction extends TransportMasterNodeReadAction< @@ -76,7 +78,7 @@ public TransportGetAllocationStatsAction( @Override protected void doExecute(Task task, Request request, ActionListener listener) { - if (clusterService.state().getMinTransportVersion().before(TransportVersions.ALLOCATION_STATS)) { + if (clusterService.state().getMinTransportVersion().before(TransportVersions.V_8_14_0)) { // The action is not available before ALLOCATION_STATS listener.onResponse(new Response(Map.of(), null)); return; @@ -88,10 +90,11 @@ protected void doExecute(Task task, Request request, ActionListener li protected void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) throws Exception { listener.onResponse( new Response( - allocationStatsService.stats(), - featureService.clusterHasFeature(clusterService.state(), AllocationStatsFeatures.INCLUDE_DISK_THRESHOLD_SETTINGS) - ? diskThresholdSettings - : null + request.metrics().contains(Metric.ALLOCATIONS) ? allocationStatsService.stats() : Map.of(), + request.metrics().contains(Metric.FS) + && featureService.clusterHasFeature(clusterService.state(), AllocationStatsFeatures.INCLUDE_DISK_THRESHOLD_SETTINGS) + ? diskThresholdSettings + : null ) ); } @@ -103,19 +106,32 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) public static class Request extends MasterNodeReadRequest { - public Request(TimeValue masterNodeTimeout, TaskId parentTaskId) { + private final EnumSet metrics; + + public Request(TimeValue masterNodeTimeout, TaskId parentTaskId, EnumSet metrics) { super(masterNodeTimeout); setParentTask(parentTaskId); + this.metrics = metrics; } public Request(StreamInput in) throws IOException { super(in); + this.metrics = in.getTransportVersion().onOrAfter(TransportVersions.MASTER_NODE_METRICS) + ? in.readEnumSet(Metric.class) + : EnumSet.of(Metric.ALLOCATIONS, Metric.FS); } @Override public void writeTo(StreamOutput out) throws IOException { - assert out.getTransportVersion().onOrAfter(TransportVersions.ALLOCATION_STATS); + assert out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0); super.writeTo(out); + if (out.getTransportVersion().onOrAfter(TransportVersions.MASTER_NODE_METRICS)) { + out.writeEnumSet(metrics); + } + } + + public EnumSet metrics() { + return metrics; } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodeFeatures.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodeFeatures.java new file mode 100644 index 0000000000000..b33520624d114 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodeFeatures.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.node.features; + +import org.elasticsearch.action.support.nodes.BaseNodeResponse; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Set; + +public class NodeFeatures extends BaseNodeResponse { + + private final Set features; + + public NodeFeatures(StreamInput in) throws IOException { + super(in); + features = in.readCollectionAsImmutableSet(StreamInput::readString); + } + + public NodeFeatures(Set features, DiscoveryNode node) { + super(node); + this.features = Set.copyOf(features); + } + + public Set nodeFeatures() { + return features; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeCollection(features, StreamOutput::writeString); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodesFeaturesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodesFeaturesRequest.java new file mode 100644 index 0000000000000..83b6fff7cf2b2 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodesFeaturesRequest.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.node.features; + +import org.elasticsearch.action.support.nodes.BaseNodesRequest; + +public class NodesFeaturesRequest extends BaseNodesRequest { + public NodesFeaturesRequest(String... nodes) { + super(nodes); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodesFeaturesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodesFeaturesResponse.java new file mode 100644 index 0000000000000..0fca588216b15 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodesFeaturesResponse.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.node.features; + +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.action.support.nodes.BaseNodesResponse; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.List; + +public class NodesFeaturesResponse extends BaseNodesResponse { + public NodesFeaturesResponse(ClusterName clusterName, List nodes, List failures) { + super(clusterName, nodes, failures); + } + + @Override + protected List readNodesFrom(StreamInput in) throws IOException { + return TransportAction.localOnly(); + } + + @Override + protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { + TransportAction.localOnly(); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/TransportNodesFeaturesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/TransportNodesFeaturesAction.java new file mode 100644 index 0000000000000..d1b7a4f1b7e95 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/TransportNodesFeaturesAction.java @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.node.features; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.nodes.TransportNodesAction; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.core.UpdateForV9; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.List; + +@UpdateForV9 +// @UpdateForV10 // this can be removed in v10. It may be called by v8 nodes to v9 nodes. +public class TransportNodesFeaturesAction extends TransportNodesAction< + NodesFeaturesRequest, + NodesFeaturesResponse, + TransportNodesFeaturesAction.NodeFeaturesRequest, + NodeFeatures> { + + public static final ActionType TYPE = new ActionType<>("cluster:monitor/nodes/features"); + + private final FeatureService featureService; + + @Inject + public TransportNodesFeaturesAction( + ThreadPool threadPool, + ClusterService clusterService, + TransportService transportService, + ActionFilters actionFilters, + FeatureService featureService + ) { + super( + TYPE.name(), + clusterService, + transportService, + actionFilters, + NodeFeaturesRequest::new, + threadPool.executor(ThreadPool.Names.MANAGEMENT) + ); + this.featureService = featureService; + } + + @Override + protected NodesFeaturesResponse newResponse( + NodesFeaturesRequest request, + List responses, + List failures + ) { + return new NodesFeaturesResponse(clusterService.getClusterName(), responses, failures); + } + + @Override + protected NodeFeaturesRequest newNodeRequest(NodesFeaturesRequest request) { + return new NodeFeaturesRequest(); + } + + @Override + protected NodeFeatures newNodeResponse(StreamInput in, DiscoveryNode node) throws IOException { + return new NodeFeatures(in); + } + + @Override + protected NodeFeatures nodeOperation(NodeFeaturesRequest request, Task task) { + return new NodeFeatures(featureService.getNodeFeatures().keySet(), transportService.getLocalNode()); + } + + public static class NodeFeaturesRequest extends TransportRequest { + public NodeFeaturesRequest(StreamInput in) throws IOException { + super(in); + } + + public NodeFeaturesRequest() {} + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java index a438983e855e9..1a53cec1bdbd7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java @@ -125,7 +125,7 @@ public NodeStats(StreamInput in) throws IOException { repositoriesStats = in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X) ? in.readOptionalWriteable(RepositoriesStats::new) : null; - nodeAllocationStats = in.getTransportVersion().onOrAfter(TransportVersions.ALLOCATION_STATS) + nodeAllocationStats = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) ? in.readOptionalWriteable(NodeAllocationStats::new) : null; } @@ -337,7 +337,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeOptionalWriteable(repositoriesStats); } - if (out.getTransportVersion().onOrAfter(TransportVersions.ALLOCATION_STATS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalWriteable(nodeAllocationStats); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java index c0329db1c1110..b1fbe3f16a18b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.cluster.node.stats; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.Strings; @@ -16,10 +17,9 @@ import org.elasticsearch.tasks.TaskId; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; /** * A request to get node (cluster) level stats. @@ -51,7 +51,7 @@ public NodesStatsRequest(NodesStatsRequestParameters nodesStatsRequestParameters */ public NodesStatsRequest all() { this.nodesStatsRequestParameters.indices().all(); - this.nodesStatsRequestParameters.requestedMetrics().addAll(NodesStatsRequestParameters.Metric.allMetrics()); + this.nodesStatsRequestParameters.requestedMetrics().addAll(Metric.ALL); return this; } @@ -100,17 +100,14 @@ public NodesStatsRequest indices(boolean indices) { * Get the names of requested metrics, excluding indices, which are * handled separately. */ - public Set requestedMetrics() { + public Set requestedMetrics() { return Set.copyOf(nodesStatsRequestParameters.requestedMetrics()); } /** * Add metric */ - public NodesStatsRequest addMetric(String metric) { - if (NodesStatsRequestParameters.Metric.allMetrics().contains(metric) == false) { - throw new IllegalStateException("Used an illegal metric: " + metric); - } + public NodesStatsRequest addMetric(Metric metric) { nodesStatsRequestParameters.requestedMetrics().add(metric); return this; } @@ -118,25 +115,22 @@ public NodesStatsRequest addMetric(String metric) { /** * Add an array of metric names */ - public NodesStatsRequest addMetrics(String... metrics) { - // use sorted set for reliable ordering in error messages - SortedSet metricsSet = new TreeSet<>(Set.of(metrics)); - if (NodesStatsRequestParameters.Metric.allMetrics().containsAll(metricsSet) == false) { - metricsSet.removeAll(NodesStatsRequestParameters.Metric.allMetrics()); - String plural = metricsSet.size() == 1 ? "" : "s"; - throw new IllegalStateException("Used illegal metric" + plural + ": " + metricsSet); + public NodesStatsRequest addMetrics(Metric... metrics) { + for (var metric : metrics) { + nodesStatsRequestParameters.requestedMetrics().add(metric); } - nodesStatsRequestParameters.requestedMetrics().addAll(metricsSet); + return this; + } + + public NodesStatsRequest addMetrics(List metrics) { + nodesStatsRequestParameters.requestedMetrics().addAll(metrics); return this; } /** * Remove metric */ - public NodesStatsRequest removeMetric(String metric) { - if (NodesStatsRequestParameters.Metric.allMetrics().contains(metric) == false) { - throw new IllegalStateException("Used an illegal metric: " + metric); - } + public NodesStatsRequest removeMetric(Metric metric) { nodesStatsRequestParameters.requestedMetrics().remove(metric); return this; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java index b412f738f5e4c..3304edcce0831 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.cluster.node.stats; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder; import org.elasticsearch.client.internal.ElasticsearchClient; @@ -46,12 +47,12 @@ public NodesStatsRequestBuilder setIndices(boolean indices) { } public NodesStatsRequestBuilder setBreaker(boolean breaker) { - addOrRemoveMetric(breaker, NodesStatsRequestParameters.Metric.BREAKER); + addOrRemoveMetric(breaker, Metric.BREAKER); return this; } public NodesStatsRequestBuilder setScript(boolean script) { - addOrRemoveMetric(script, NodesStatsRequestParameters.Metric.SCRIPT); + addOrRemoveMetric(script, Metric.SCRIPT); return this; } @@ -67,7 +68,7 @@ public NodesStatsRequestBuilder setIndices(CommonStatsFlags indices) { * Should the node OS stats be returned. */ public NodesStatsRequestBuilder setOs(boolean os) { - addOrRemoveMetric(os, NodesStatsRequestParameters.Metric.OS); + addOrRemoveMetric(os, Metric.OS); return this; } @@ -75,7 +76,7 @@ public NodesStatsRequestBuilder setOs(boolean os) { * Should the node OS stats be returned. */ public NodesStatsRequestBuilder setProcess(boolean process) { - addOrRemoveMetric(process, NodesStatsRequestParameters.Metric.PROCESS); + addOrRemoveMetric(process, Metric.PROCESS); return this; } @@ -83,7 +84,7 @@ public NodesStatsRequestBuilder setProcess(boolean process) { * Should the node JVM stats be returned. */ public NodesStatsRequestBuilder setJvm(boolean jvm) { - addOrRemoveMetric(jvm, NodesStatsRequestParameters.Metric.JVM); + addOrRemoveMetric(jvm, Metric.JVM); return this; } @@ -91,7 +92,7 @@ public NodesStatsRequestBuilder setJvm(boolean jvm) { * Should the node thread pool stats be returned. */ public NodesStatsRequestBuilder setThreadPool(boolean threadPool) { - addOrRemoveMetric(threadPool, NodesStatsRequestParameters.Metric.THREAD_POOL); + addOrRemoveMetric(threadPool, Metric.THREAD_POOL); return this; } @@ -99,7 +100,7 @@ public NodesStatsRequestBuilder setThreadPool(boolean threadPool) { * Should the node file system stats be returned. */ public NodesStatsRequestBuilder setFs(boolean fs) { - addOrRemoveMetric(fs, NodesStatsRequestParameters.Metric.FS); + addOrRemoveMetric(fs, Metric.FS); return this; } @@ -107,7 +108,7 @@ public NodesStatsRequestBuilder setFs(boolean fs) { * Should the node Transport stats be returned. */ public NodesStatsRequestBuilder setTransport(boolean transport) { - addOrRemoveMetric(transport, NodesStatsRequestParameters.Metric.TRANSPORT); + addOrRemoveMetric(transport, Metric.TRANSPORT); return this; } @@ -115,7 +116,7 @@ public NodesStatsRequestBuilder setTransport(boolean transport) { * Should the node HTTP stats be returned. */ public NodesStatsRequestBuilder setHttp(boolean http) { - addOrRemoveMetric(http, NodesStatsRequestParameters.Metric.HTTP); + addOrRemoveMetric(http, Metric.HTTP); return this; } @@ -123,7 +124,7 @@ public NodesStatsRequestBuilder setHttp(boolean http) { * Should the discovery stats be returned. */ public NodesStatsRequestBuilder setDiscovery(boolean discovery) { - addOrRemoveMetric(discovery, NodesStatsRequestParameters.Metric.DISCOVERY); + addOrRemoveMetric(discovery, Metric.DISCOVERY); return this; } @@ -131,12 +132,12 @@ public NodesStatsRequestBuilder setDiscovery(boolean discovery) { * Should ingest statistics be returned. */ public NodesStatsRequestBuilder setIngest(boolean ingest) { - addOrRemoveMetric(ingest, NodesStatsRequestParameters.Metric.INGEST); + addOrRemoveMetric(ingest, Metric.INGEST); return this; } public NodesStatsRequestBuilder setAdaptiveSelection(boolean adaptiveSelection) { - addOrRemoveMetric(adaptiveSelection, NodesStatsRequestParameters.Metric.ADAPTIVE_SELECTION); + addOrRemoveMetric(adaptiveSelection, Metric.ADAPTIVE_SELECTION); return this; } @@ -144,33 +145,33 @@ public NodesStatsRequestBuilder setAdaptiveSelection(boolean adaptiveSelection) * Should script context cache statistics be returned */ public NodesStatsRequestBuilder setScriptCache(boolean scriptCache) { - addOrRemoveMetric(scriptCache, NodesStatsRequestParameters.Metric.SCRIPT_CACHE); + addOrRemoveMetric(scriptCache, Metric.SCRIPT_CACHE); return this; } public NodesStatsRequestBuilder setIndexingPressure(boolean indexingPressure) { - addOrRemoveMetric(indexingPressure, NodesStatsRequestParameters.Metric.INDEXING_PRESSURE); + addOrRemoveMetric(indexingPressure, Metric.INDEXING_PRESSURE); return this; } public NodesStatsRequestBuilder setRepositoryStats(boolean repositoryStats) { - addOrRemoveMetric(repositoryStats, NodesStatsRequestParameters.Metric.REPOSITORIES); + addOrRemoveMetric(repositoryStats, Metric.REPOSITORIES); return this; } public NodesStatsRequestBuilder setAllocationStats(boolean allocationStats) { - addOrRemoveMetric(allocationStats, NodesStatsRequestParameters.Metric.ALLOCATIONS); + addOrRemoveMetric(allocationStats, Metric.ALLOCATIONS); return this; } /** * Helper method for adding metrics to a request */ - private void addOrRemoveMetric(boolean includeMetric, NodesStatsRequestParameters.Metric metric) { + private void addOrRemoveMetric(boolean includeMetric, Metric metric) { if (includeMetric) { - request.addMetric(metric.metricName()); + request.addMetric(metric); } else { - request.removeMetric(metric.metricName()); + request.removeMetric(metric); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParameters.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParameters.java index 9e965fcccb2f3..42d6655d1706b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParameters.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParameters.java @@ -15,25 +15,29 @@ import org.elasticsearch.common.io.stream.Writeable; import java.io.IOException; -import java.util.Arrays; -import java.util.HashSet; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; + +import static java.util.stream.Collectors.toUnmodifiableMap; +import static java.util.stream.Collectors.toUnmodifiableSet; /** * This class encapsulates the metrics and other information needed to define scope when we are requesting node stats. */ public class NodesStatsRequestParameters implements Writeable { private CommonStatsFlags indices = new CommonStatsFlags(); - private final Set requestedMetrics = new HashSet<>(); + private final EnumSet requestedMetrics; private boolean includeShardsStats = true; - public NodesStatsRequestParameters() {} + public NodesStatsRequestParameters() { + this.requestedMetrics = EnumSet.noneOf(Metric.class); + } public NodesStatsRequestParameters(StreamInput in) throws IOException { indices = new CommonStatsFlags(in); - requestedMetrics.clear(); - requestedMetrics.addAll(in.readStringCollectionAsList()); + requestedMetrics = Metric.readSetFrom(in); if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { includeShardsStats = in.readBoolean(); } else { @@ -44,7 +48,7 @@ public NodesStatsRequestParameters(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { indices.writeTo(out); - out.writeStringCollection(requestedMetrics); + Metric.writeSetTo(out, requestedMetrics); if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { out.writeBoolean(includeShardsStats); } @@ -58,7 +62,7 @@ public void setIndices(CommonStatsFlags indices) { this.indices = indices; } - public Set requestedMetrics() { + public EnumSet requestedMetrics() { return requestedMetrics; } @@ -92,22 +96,52 @@ public enum Metric { REPOSITORIES("repositories"), ALLOCATIONS("allocations"); - private String metricName; + public static final Set ALL = Collections.unmodifiableSet(EnumSet.allOf(Metric.class)); + public static final Set ALL_NAMES = ALL.stream().map(Metric::metricName).collect(toUnmodifiableSet()); + public static final Map NAMES_MAP = ALL.stream().collect(toUnmodifiableMap(Metric::metricName, m -> m)); + private final String metricName; - Metric(String name) { - this.metricName = name; + Metric(String metricName) { + this.metricName = metricName; } - public String metricName() { - return this.metricName; + public static boolean isValid(String name) { + return NAMES_MAP.containsKey(name); + } + + public static Metric get(String name) { + var metric = NAMES_MAP.get(name); + assert metric != null; + return metric; } - boolean containedIn(Set metricNames) { - return metricNames.contains(this.metricName()); + public static void writeSetTo(StreamOutput out, EnumSet metrics) throws IOException { + if (out.getTransportVersion().onOrAfter(TransportVersions.NODES_STATS_ENUM_SET)) { + out.writeEnumSet(metrics); + } else { + out.writeCollection(metrics, (output, metric) -> output.writeString(metric.metricName)); + } + } + + public static EnumSet readSetFrom(StreamInput in) throws IOException { + if (in.getTransportVersion().onOrAfter(TransportVersions.NODES_STATS_ENUM_SET)) { + return in.readEnumSet(Metric.class); + } else { + return in.readCollection((i) -> EnumSet.noneOf(Metric.class), (is, out) -> { + var name = is.readString(); + var metric = Metric.get(name); + out.add(metric); + }); + } + } + + public String metricName() { + return metricName; } - static Set allMetrics() { - return Arrays.stream(values()).map(Metric::metricName).collect(Collectors.toSet()); + @Override + public String toString() { + return metricName; } } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java index 1b7ce13333891..3416b77fdd7fd 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.admin.cluster.allocation.TransportGetAllocationStatsAction; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.nodes.TransportNodesAction; import org.elasticsearch.client.internal.node.NodeClient; @@ -39,7 +40,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; public class TransportNodesStatsAction extends TransportNodesAction< NodesStatsRequest, @@ -57,8 +57,8 @@ public TransportNodesStatsAction( ThreadPool threadPool, ClusterService clusterService, TransportService transportService, - NodeService nodeService, ActionFilters actionFilters, + NodeService nodeService, NodeClient client ) { super( @@ -86,21 +86,21 @@ protected void newResponseAsync( List failures, ActionListener listener ) { - Set metrics = request.getNodesStatsRequestParameters().requestedMetrics(); - if (NodesStatsRequestParameters.Metric.ALLOCATIONS.containedIn(metrics) - || NodesStatsRequestParameters.Metric.FS.containedIn(metrics)) { + var metrics = request.getNodesStatsRequestParameters().requestedMetrics(); + if (metrics.contains(Metric.FS) || metrics.contains(Metric.ALLOCATIONS)) { client.execute( TransportGetAllocationStatsAction.TYPE, new TransportGetAllocationStatsAction.Request( Objects.requireNonNullElse(request.timeout(), RestUtils.REST_MASTER_TIMEOUT_DEFAULT), - new TaskId(clusterService.localNode().getId(), task.getId()) + new TaskId(clusterService.localNode().getId(), task.getId()), + metrics ), - listener.delegateFailure((l, r) -> { - ActionListener.respondAndRelease( + listener.delegateFailure( + (l, r) -> ActionListener.respondAndRelease( l, newResponse(request, merge(responses, r.getNodeAllocationStats(), r.getDiskThresholdSettings()), failures) - ); - }) + ) + ) ); } else { ActionListener.run(listener, l -> ActionListener.respondAndRelease(l, newResponse(request, responses, failures))); @@ -132,26 +132,27 @@ protected NodeStats newNodeResponse(StreamInput in, DiscoveryNode node) throws I protected NodeStats nodeOperation(NodeStatsRequest request, Task task) { assert task instanceof CancellableTask; - final NodesStatsRequestParameters nodesStatsRequestParameters = request.getNodesStatsRequestParameters(); - Set metrics = nodesStatsRequestParameters.requestedMetrics(); + final var nodesStatsRequestParameters = request.getNodesStatsRequestParameters(); + final var metrics = nodesStatsRequestParameters.requestedMetrics(); + return nodeService.stats( nodesStatsRequestParameters.indices(), nodesStatsRequestParameters.includeShardsStats(), - NodesStatsRequestParameters.Metric.OS.containedIn(metrics), - NodesStatsRequestParameters.Metric.PROCESS.containedIn(metrics), - NodesStatsRequestParameters.Metric.JVM.containedIn(metrics), - NodesStatsRequestParameters.Metric.THREAD_POOL.containedIn(metrics), - NodesStatsRequestParameters.Metric.FS.containedIn(metrics), - NodesStatsRequestParameters.Metric.TRANSPORT.containedIn(metrics), - NodesStatsRequestParameters.Metric.HTTP.containedIn(metrics), - NodesStatsRequestParameters.Metric.BREAKER.containedIn(metrics), - NodesStatsRequestParameters.Metric.SCRIPT.containedIn(metrics), - NodesStatsRequestParameters.Metric.DISCOVERY.containedIn(metrics), - NodesStatsRequestParameters.Metric.INGEST.containedIn(metrics), - NodesStatsRequestParameters.Metric.ADAPTIVE_SELECTION.containedIn(metrics), - NodesStatsRequestParameters.Metric.SCRIPT_CACHE.containedIn(metrics), - NodesStatsRequestParameters.Metric.INDEXING_PRESSURE.containedIn(metrics), - NodesStatsRequestParameters.Metric.REPOSITORIES.containedIn(metrics) + metrics.contains(Metric.OS), + metrics.contains(Metric.PROCESS), + metrics.contains(Metric.JVM), + metrics.contains(Metric.THREAD_POOL), + metrics.contains(Metric.FS), + metrics.contains(Metric.TRANSPORT), + metrics.contains(Metric.HTTP), + metrics.contains(Metric.BREAKER), + metrics.contains(Metric.SCRIPT), + metrics.contains(Metric.DISCOVERY), + metrics.contains(Metric.INGEST), + metrics.contains(Metric.ADAPTIVE_SELECTION), + metrics.contains(Metric.SCRIPT_CACHE), + metrics.contains(Metric.INDEXING_PRESSURE), + metrics.contains(Metric.REPOSITORIES) ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java index 4b7b91099ff12..5d8ab3daaf85c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java @@ -10,7 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.GroupedActionListener; +import org.elasticsearch.action.support.RefCountingListener; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.ClusterState; @@ -68,21 +68,21 @@ protected void masterOperation( ClusterState state, ActionListener listener ) throws Exception { - if (systemIndices.getFeatures().size() == 0) { - listener.onResponse(new ResetFeatureStateResponse(Collections.emptyList())); - } - - final int features = systemIndices.getFeatures().size(); - GroupedActionListener groupedActionListener = new GroupedActionListener<>( - systemIndices.getFeatures().size(), - listener.map(responses -> { - assert features == responses.size(); - return new ResetFeatureStateResponse(new ArrayList<>(responses)); - }) - ); - - for (SystemIndices.Feature feature : systemIndices.getFeatures()) { - feature.getCleanUpFunction().apply(clusterService, client, groupedActionListener); + final var features = systemIndices.getFeatures(); + final var responses = new ArrayList(features.size()); + try ( + var listeners = new RefCountingListener( + listener.map(ignored -> new ResetFeatureStateResponse(Collections.unmodifiableList(responses))) + ) + ) { + for (final var feature : features) { + feature.getCleanUpFunction().apply(clusterService, client, listeners.acquire(e -> { + assert e != null : feature.getName(); + synchronized (responses) { + responses.add(e); + } + })); + } } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index ff5fdbaa787fe..1a279e3488123 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -53,13 +53,12 @@ import org.elasticsearch.transport.TransportService; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Queue; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiPredicate; @@ -77,6 +76,36 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction> allSnapshotInfos = ConcurrentCollections.newQueue(); + private final List allSnapshotInfos = Collections.synchronizedList(new ArrayList<>()); /** * Accumulates number of snapshots that match the name/fromSortValue/slmPolicy predicates, to be returned in the response. */ private final AtomicInteger totalCount = new AtomicInteger(); - /** - * Accumulates the number of snapshots that match the name/fromSortValue/slmPolicy/after predicates, for sizing the final result - * list. - */ - private final AtomicInteger resultsCount = new AtomicInteger(); - GetSnapshotsOperation( CancellableTask cancellableTask, List repositories, @@ -231,158 +249,61 @@ private class GetSnapshotsOperation { threadPool.info(ThreadPool.Names.SNAPSHOT_META).getMax(), cancellableTask::isCancelled ); - } - - void getMultipleReposSnapshotInfo(ActionListener listener) { - SubscribableListener - - .newForked(repositoriesDoneListener -> { - try (var listeners = new RefCountingListener(repositoriesDoneListener)) { - for (final RepositoryMetadata repository : repositories) { - final String repoName = repository.name(); - if (skipRepository(repoName)) { - continue; - } - - if (listeners.isFailing()) { - return; - } - - SubscribableListener - - .newForked(repositoryDataListener -> { - if (snapshotNamePredicate == SnapshotNamePredicate.MATCH_CURRENT_ONLY) { - repositoryDataListener.onResponse(null); - } else { - repositoriesService.repository(repoName).getRepositoryData(executor, repositoryDataListener); - } - }) - - .andThen((l, repositoryData) -> loadSnapshotInfos(repoName, repositoryData, l)) - - .addListener(listeners.acquire()); - } - } - }) - - .addListener(listener.map(ignored -> buildResponse()), executor, threadPool.getThreadContext()); - } - - private boolean skipRepository(String repositoryName) { - if (sortBy == SnapshotSortKey.REPOSITORY && fromSortValue != null) { - // If we are sorting by repository name with an offset given by fromSortValue, skip earlier repositories - return order == SortOrder.ASC ? fromSortValue.compareTo(repositoryName) > 0 : fromSortValue.compareTo(repositoryName) < 0; - } else { - return false; - } - } - - private void loadSnapshotInfos(String repo, @Nullable RepositoryData repositoryData, ActionListener listener) { - assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT); - - if (cancellableTask.notifyIfCancelled(listener)) { - return; - } - final Set unmatchedRequiredNames = new HashSet<>(snapshotNamePredicate.requiredNames()); - final Set toResolve = new HashSet<>(); - - for (final var snapshotInProgress : snapshotsInProgress.forRepo(repo)) { - final var snapshotName = snapshotInProgress.snapshot().getSnapshotId().getName(); - unmatchedRequiredNames.remove(snapshotName); - if (snapshotNamePredicate.test(snapshotName, true)) { - toResolve.add(snapshotInProgress.snapshot()); - } - } - - if (repositoryData != null) { - for (final var snapshotId : repositoryData.getSnapshotIds()) { - final var snapshotName = snapshotId.getName(); - unmatchedRequiredNames.remove(snapshotName); - if (snapshotNamePredicate.test(snapshotName, false) && matchesPredicates(snapshotId, repositoryData)) { - toResolve.add(new Snapshot(repo, snapshotId)); - } - } - } - - if (unmatchedRequiredNames.isEmpty() == false) { - throw new SnapshotMissingException(repo, unmatchedRequiredNames.iterator().next()); - } - - if (verbose) { - loadSnapshotInfos(repo, toResolve.stream().map(Snapshot::getSnapshotId).toList(), listener); - } else { + if (verbose == false) { assert fromSortValuePredicates.isMatchAll() : "filtering is not supported in non-verbose mode"; assert slmPolicyPredicate == SlmPolicyPredicate.MATCH_ALL_POLICIES : "filtering is not supported in non-verbose mode"; + } + } - addSimpleSnapshotInfos( - toResolve, - repo, - repositoryData, - snapshotsInProgress.forRepo(repo).stream().map(entry -> SnapshotInfo.inProgress(entry).basic()).toList() + /** + * Run the get-snapshots operation and compute the response. + */ + void runOperation(ActionListener listener) { + SubscribableListener.newForked(this::populateResults) + .addListener( + listener.map(ignored -> buildResponse()), + // If we didn't load any SnapshotInfo blobs from the repo (e.g. verbose=false or current-snapshots-only) then this + // listener chain will already be complete, no need to fork again. Otherwise we forked to SNAPSHOT_META so must + // fork back to MANAGEMENT for the final step. + executor, + threadPool.getThreadContext() ); - listener.onResponse(null); - } } - private void loadSnapshotInfos(String repositoryName, Collection snapshotIds, ActionListener listener) { - if (cancellableTask.notifyIfCancelled(listener)) { - return; - } - final AtomicInteger repositoryTotalCount = new AtomicInteger(); - final List snapshots = new ArrayList<>(snapshotIds.size()); - final Set snapshotIdsToIterate = new HashSet<>(snapshotIds); - // first, look at the snapshots in progress - final List entries = SnapshotsService.currentSnapshots( - snapshotsInProgress, - repositoryName, - snapshotIdsToIterate.stream().map(SnapshotId::getName).toList() - ); - for (SnapshotsInProgress.Entry entry : entries) { - if (snapshotIdsToIterate.remove(entry.snapshot().getSnapshotId())) { - final SnapshotInfo snapshotInfo = SnapshotInfo.inProgress(entry); - if (matchesPredicates(snapshotInfo)) { - repositoryTotalCount.incrementAndGet(); - if (afterPredicate.test(snapshotInfo)) { - snapshots.add(snapshotInfo.maybeWithoutIndices(indices)); - } + /** + * Populate the results fields ({@link #allSnapshotInfos} and {@link #totalCount}). + */ + private void populateResults(ActionListener listener) { + try (var listeners = new RefCountingListener(listener)) { + for (final RepositoryMetadata repository : repositories) { + final String repositoryName = repository.name(); + if (skipRepository(repositoryName)) { + continue; } - } - } - // then, look in the repository if there's any matching snapshots left - SubscribableListener - - .newForked(l -> { - try (var listeners = new RefCountingListener(l)) { - if (snapshotIdsToIterate.isEmpty()) { - return; - } - - final Repository repository; - try { - repository = repositoriesService.repository(repositoryName); - } catch (RepositoryMissingException e) { - listeners.acquire().onFailure(e); - return; - } - // only need to synchronize accesses related to reading SnapshotInfo from the repo - final List syncSnapshots = Collections.synchronizedList(snapshots); + if (listeners.isFailing()) { + return; + } + maybeGetRepositoryData(repositoryName, listeners.acquire(repositoryData -> { + assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT); + cancellableTask.ensureNotCancelled(); + ensureRequiredNamesPresent(repositoryName, repositoryData); ThrottledIterator.run( Iterators.failFast( - snapshotIdsToIterate.iterator(), + getAsyncSnapshotInfoIterator(repositoriesService.repository(repositoryName), repositoryData), () -> cancellableTask.isCancelled() || listeners.isFailing() ), - (ref, snapshotId) -> { - final var refListener = ActionListener.runBefore(listeners.acquire(), ref::close); - getSnapshotInfoExecutor.getSnapshotInfo(repository, snapshotId, new ActionListener<>() { + (ref, asyncSnapshotInfo) -> ActionListener.run( + ActionListener.runBefore(listeners.acquire(), ref::close), + refListener -> asyncSnapshotInfo.getSnapshotInfo(new ActionListener<>() { @Override public void onResponse(SnapshotInfo snapshotInfo) { if (matchesPredicates(snapshotInfo)) { - repositoryTotalCount.incrementAndGet(); + totalCount.incrementAndGet(); if (afterPredicate.test(snapshotInfo)) { - syncSnapshots.add(snapshotInfo.maybeWithoutIndices(indices)); + allSnapshotInfos.add(snapshotInfo.maybeWithoutIndices(indices)); } } refListener.onResponse(null); @@ -391,96 +312,201 @@ public void onResponse(SnapshotInfo snapshotInfo) { @Override public void onFailure(Exception e) { if (ignoreUnavailable) { - logger.warn( - Strings.format("failed to fetch snapshot info for [%s:%s]", repository, snapshotId), - e - ); + logger.warn(Strings.format("failed to fetch snapshot info for [%s]", asyncSnapshotInfo), e); refListener.onResponse(null); } else { refListener.onFailure(e); } } - }); - }, + }) + ), getSnapshotInfoExecutor.getMaxRunningTasks(), () -> {}, () -> {} ); - } - }) - - // no need to synchronize access to snapshots: Repository#getSnapshotInfo fails fast but we're on the success path here - .andThenAccept(ignored -> addResults(repositoryTotalCount.get(), snapshots)) + })); + } + } + } - .addListener(listener); + private void maybeGetRepositoryData(String repositoryName, ActionListener listener) { + if (snapshotNamePredicate == SnapshotNamePredicate.MATCH_CURRENT_ONLY) { + listener.onResponse(null); + } else { + repositoriesService.repository(repositoryName).getRepositoryData(executor, listener); + } } - private void addResults(int repositoryTotalCount, List snapshots) { - totalCount.addAndGet(repositoryTotalCount); - resultsCount.addAndGet(snapshots.size()); - allSnapshotInfos.add(snapshots); + private boolean skipRepository(String repositoryName) { + if (sortBy == SnapshotSortKey.REPOSITORY && fromSortValue != null) { + // If we are sorting by repository name with an offset given by fromSortValue, skip earlier repositories + return order == SortOrder.ASC ? fromSortValue.compareTo(repositoryName) > 0 : fromSortValue.compareTo(repositoryName) < 0; + } else { + return false; + } } - private void addSimpleSnapshotInfos( - final Set toResolve, - final String repoName, - final RepositoryData repositoryData, - final List currentSnapshots - ) { - if (repositoryData == null) { - // only want current snapshots - addResults(currentSnapshots.size(), currentSnapshots.stream().filter(afterPredicate).toList()); + /** + * Check that the repository contains every required name according to {@link #snapshotNamePredicate}. + * + * @throws SnapshotMissingException if one or more required names are missing. + */ + private void ensureRequiredNamesPresent(String repositoryName, @Nullable RepositoryData repositoryData) { + if (snapshotNamePredicate.requiredNames().isEmpty()) { return; - } // else want non-current snapshots as well, which are found in the repository data - - List snapshotInfos = new ArrayList<>(currentSnapshots.size() + toResolve.size()); - int repositoryTotalCount = 0; - for (SnapshotInfo snapshotInfo : currentSnapshots) { - assert snapshotInfo.startTime() == 0L && snapshotInfo.endTime() == 0L && snapshotInfo.totalShards() == 0L : snapshotInfo; - if (toResolve.remove(snapshotInfo.snapshot())) { - repositoryTotalCount += 1; - if (afterPredicate.test(snapshotInfo)) { - snapshotInfos.add(snapshotInfo); - } + } + + final var unmatchedRequiredNames = new HashSet<>(snapshotNamePredicate.requiredNames()); + for (final var snapshotInProgress : snapshotsInProgress.forRepo(repositoryName)) { + unmatchedRequiredNames.remove(snapshotInProgress.snapshot().getSnapshotId().getName()); + } + if (unmatchedRequiredNames.isEmpty()) { + return; + } + if (repositoryData != null) { + for (final var snapshotId : repositoryData.getSnapshotIds()) { + unmatchedRequiredNames.remove(snapshotId.getName()); + } + if (unmatchedRequiredNames.isEmpty()) { + return; } } - Map> snapshotsToIndices = new HashMap<>(); - if (indices) { - for (IndexId indexId : repositoryData.getIndices().values()) { - for (SnapshotId snapshotId : repositoryData.getSnapshots(indexId)) { - if (toResolve.contains(new Snapshot(repoName, snapshotId))) { - snapshotsToIndices.computeIfAbsent(snapshotId, (k) -> new ArrayList<>()).add(indexId.getName()); - } + throw new SnapshotMissingException(repositoryName, unmatchedRequiredNames.iterator().next()); + } + + /** + * An asynchronous supplier of a {@link SnapshotInfo}. + */ + private interface AsyncSnapshotInfo { + /** + * @param listener completed, possibly asynchronously, with the appropriate {@link SnapshotInfo}. + */ + void getSnapshotInfo(ActionListener listener); + } + + /** + * @return an {@link AsyncSnapshotInfo} for the given in-progress snapshot entry. + */ + private AsyncSnapshotInfo forSnapshotInProgress(SnapshotsInProgress.Entry snapshotInProgress) { + return new AsyncSnapshotInfo() { + @Override + public void getSnapshotInfo(ActionListener listener) { + assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT); // see [NOTE ON THREADING] + final var snapshotInfo = SnapshotInfo.inProgress(snapshotInProgress); + listener.onResponse(verbose ? snapshotInfo : snapshotInfo.basic()); + } + + @Override + public String toString() { + return snapshotInProgress.snapshot().toString(); + } + }; + } + + /** + * @return an {@link AsyncSnapshotInfo} for the given completed snapshot. + */ + private AsyncSnapshotInfo forCompletedSnapshot( + Repository repository, + SnapshotId snapshotId, + RepositoryData repositoryData, + Map> indicesLookup + ) { + return new AsyncSnapshotInfo() { + @Override + public void getSnapshotInfo(ActionListener listener) { + if (verbose) { + // always forks to SNAPSHOT_META, and may already have done so for an earlier item - see [NOTE ON THREADING] + assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT, ThreadPool.Names.SNAPSHOT_META); + getSnapshotInfoExecutor.getSnapshotInfo(repository, snapshotId, listener); + } else { + assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT); // see [NOTE ON THREADING] + ActionListener.completeWith( + listener, + () -> new SnapshotInfo( + new Snapshot(repository.getMetadata().name(), snapshotId), + indicesLookup.getOrDefault(snapshotId, Collections.emptyList()), + Collections.emptyList(), + Collections.emptyList(), + repositoryData.getSnapshotState(snapshotId) + ) + ); } } + + @Override + public String toString() { + return repository.getMetadata().name() + ":" + snapshotId; + } + }; + } + + /** + * @return an iterator of {@link AsyncSnapshotInfo} instances in the given repository which match {@link #snapshotNamePredicate}. + */ + private Iterator getAsyncSnapshotInfoIterator(Repository repository, @Nullable RepositoryData repositoryData) { + // now iterate through the snapshots again, returning SnapshotInfo suppliers for ones with matching IDs + final Set matchingInProgressSnapshots = new HashSet<>(); + final var indicesLookup = getIndicesLookup(repositoryData); + return Iterators.concat( + // matching in-progress snapshots first + Iterators.map( + Iterators.filter(snapshotsInProgress.forRepo(repository.getMetadata().name()).iterator(), snapshotInProgress -> { + final var snapshotId = snapshotInProgress.snapshot().getSnapshotId(); + if (snapshotNamePredicate.test(snapshotId.getName(), true)) { + matchingInProgressSnapshots.add(snapshotId); + return true; + } else { + return false; + } + }), + this::forSnapshotInProgress + ), + repositoryData == null + // Only returning in-progress snapshots: + ? Collections.emptyIterator() + // Also return matching completed snapshots (except any ones that were also found to be in-progress). + // NB this will fork tasks to SNAPSHOT_META (if verbose=true) which will be used for subsequent items so we mustn't + // follow it with any more non-forking iteration. See [NOTE ON THREADING]. + : Iterators.map( + Iterators.filter( + repositoryData.getSnapshotIds().iterator(), + snapshotId -> matchingInProgressSnapshots.contains(snapshotId) == false + && snapshotNamePredicate.test(snapshotId.getName(), false) + && matchesPredicates(snapshotId, repositoryData) + ), + snapshotId -> forCompletedSnapshot(repository, snapshotId, repositoryData, indicesLookup) + ) + ); + } + + @Nullable + private Map> getIndicesLookup(RepositoryData repositoryData) { + if (repositoryData == null || verbose || indices == false) { + return Map.of(); } - for (Snapshot snapshot : toResolve) { - final var snapshotInfo = new SnapshotInfo( - snapshot, - snapshotsToIndices.getOrDefault(snapshot.getSnapshotId(), Collections.emptyList()), - Collections.emptyList(), - Collections.emptyList(), - repositoryData.getSnapshotState(snapshot.getSnapshotId()) - ); - repositoryTotalCount += 1; - if (afterPredicate.test(snapshotInfo)) { - snapshotInfos.add(snapshotInfo); + + final Map> snapshotsToIndices = new HashMap<>(); + for (IndexId indexId : repositoryData.getIndices().values()) { + for (SnapshotId snapshotId : repositoryData.getSnapshots(indexId)) { + if (snapshotNamePredicate.test(snapshotId.getName(), false) && matchesPredicates(snapshotId, repositoryData)) { + snapshotsToIndices.computeIfAbsent(snapshotId, (k) -> new ArrayList<>()).add(indexId.getName()); + } } } - addResults(repositoryTotalCount, snapshotInfos); + return snapshotsToIndices; } private GetSnapshotsResponse buildResponse() { - assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT); + assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT); // see [NOTE ON THREADING] cancellableTask.ensureNotCancelled(); int remaining = 0; final var resultsStream = allSnapshotInfos.stream() - .flatMap(Collection::stream) .peek(this::assertSatisfiesAllPredicates) .sorted(sortBy.getSnapshotInfoComparator(order)) .skip(offset); final List snapshotInfos; - if (size == GetSnapshotsRequest.NO_LIMIT || resultsCount.get() <= size) { + if (size == GetSnapshotsRequest.NO_LIMIT || allSnapshotInfos.size() <= size) { snapshotInfos = resultsStream.toList(); } else { snapshotInfos = new ArrayList<>(size); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java index 28f970eb8c9fe..caedc3363e9a3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java @@ -127,7 +127,7 @@ protected void masterOperation( Set nodesIds = new HashSet<>(); for (SnapshotsInProgress.Entry entry : currentSnapshots) { - for (SnapshotsInProgress.ShardSnapshotStatus status : entry.shardsByRepoShardId().values()) { + for (SnapshotsInProgress.ShardSnapshotStatus status : entry.shardSnapshotStatusByRepoShardId().values()) { if (status.nodeId() != null) { nodesIds.add(status.nodeId()); } @@ -188,7 +188,8 @@ private void buildResponse( for (SnapshotsInProgress.Entry entry : currentSnapshotEntries) { currentSnapshotNames.add(entry.snapshot().getSnapshotId().getName()); List shardStatusBuilder = new ArrayList<>(); - for (Map.Entry shardEntry : entry.shardsByRepoShardId() + for (Map.Entry shardEntry : entry + .shardSnapshotStatusByRepoShardId() .entrySet()) { SnapshotsInProgress.ShardSnapshotStatus status = shardEntry.getValue(); if (status.nodeId() != null) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesResponse.java index b4f483e6f8161..b8ad70ad8b6f8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesResponse.java @@ -48,7 +48,7 @@ public class IndicesAliasesResponse extends AcknowledgedResponse { protected IndicesAliasesResponse(StreamInput in) throws IOException { super(in); - if (in.getTransportVersion().onOrAfter(TransportVersions.ALIAS_ACTION_RESULTS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { this.errors = in.readBoolean(); this.actionResults = in.readCollectionAsImmutableList(AliasActionResult::new); } else { @@ -91,7 +91,7 @@ public static IndicesAliasesResponse build(final List actionR @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - if (out.getTransportVersion().onOrAfter(TransportVersions.ALIAS_ACTION_RESULTS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeBoolean(errors); out.writeCollection(actionResults); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java index ac2f437f7225a..643f92ec3378f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java @@ -66,7 +66,9 @@ public TransportVerifyShardBeforeCloseAction( actionFilters, ShardRequest::new, ShardRequest::new, - threadPool.executor(ThreadPool.Names.MANAGEMENT) + threadPool.executor(ThreadPool.Names.MANAGEMENT), + SyncGlobalCheckpointAfterOperation.DoNotSync, + PrimaryActionExecution.RejectOnOverload ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java index 74ae53f7ac9de..69e1309b89aef 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java @@ -58,7 +58,9 @@ public TransportShardFlushAction( actionFilters, ShardFlushRequest::new, ShardFlushRequest::new, - threadPool.executor(ThreadPool.Names.FLUSH) + threadPool.executor(ThreadPool.Names.FLUSH), + SyncGlobalCheckpointAfterOperation.DoNotSync, + PrimaryActionExecution.RejectOnOverload ); transportService.registerRequestHandler( PRE_SYNCED_FLUSH_ACTION_NAME, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java index 31e9f959f0fe7..e93b3983ee85b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java @@ -67,7 +67,9 @@ public TransportVerifyShardIndexBlockAction( actionFilters, ShardRequest::new, ShardRequest::new, - threadPool.executor(ThreadPool.Names.MANAGEMENT) + threadPool.executor(ThreadPool.Names.MANAGEMENT), + SyncGlobalCheckpointAfterOperation.DoNotSync, + PrimaryActionExecution.RejectOnOverload ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java index b3e6385e7099d..cc4edcf0efb81 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java @@ -68,7 +68,9 @@ public TransportShardRefreshAction( actionFilters, BasicReplicationRequest::new, ShardRefreshReplicaRequest::new, - threadPool.executor(ThreadPool.Names.REFRESH) + threadPool.executor(ThreadPool.Names.REFRESH), + SyncGlobalCheckpointAfterOperation.DoNotSync, + PrimaryActionExecution.RejectOnOverload ); // registers the unpromotable version of shard refresh action new TransportUnpromotableShardRefreshAction(clusterService, transportService, shardStateAction, actionFilters, indicesService); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/OptimalShardCountCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/OptimalShardCountCondition.java index 93a11b8fe0855..acd26f2984c99 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/OptimalShardCountCondition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/OptimalShardCountCondition.java @@ -65,6 +65,6 @@ public static OptimalShardCountCondition fromXContent(XContentParser parser) thr @Override boolean includedInVersion(TransportVersion version) { - return version.onOrAfter(TransportVersions.AUTO_SHARDING_ROLLOVER_CONDITION); + return version.onOrAfter(TransportVersions.V_8_14_0); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index 6302b1c9ef9fb..db4fad99d4f48 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -116,7 +116,7 @@ public RolloverRequest(StreamInput in) throws IOException { } else { lazy = false; } - if (in.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_ROLLOVER)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { indicesOptions = IndicesOptions.readIndicesOptions(in); } } @@ -168,7 +168,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) { out.writeBoolean(lazy); } - if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_ROLLOVER)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { indicesOptions.writeIndicesOptions(out); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java index 5483097b140da..87d5e39f41a32 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java @@ -132,7 +132,7 @@ public Response(StreamInput in) throws IOException { } else { rolloverConfiguration = null; } - if (in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { globalRetention = in.readOptionalWriteable(DataStreamGlobalRetention::read); } else { globalRetention = null; @@ -171,7 +171,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeOptionalWriteable(rolloverConfiguration); } - if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalWriteable(globalRetention); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java index 5cb35d23c8b7c..f1c7ea95e4fa7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java @@ -133,7 +133,7 @@ public Response(StreamInput in) throws IOException { } else { rolloverConfiguration = null; } - if (in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { globalRetention = in.readOptionalWriteable(DataStreamGlobalRetention::read); } else { globalRetention = null; @@ -168,7 +168,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeOptionalWriteable(rolloverConfiguration); } - if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalWriteable(globalRetention); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java index 5d0a4a293ea4f..a2fe2e5056c4d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java @@ -89,7 +89,7 @@ public SimulateIndexTemplateResponse(StreamInput in) throws IOException { rolloverConfiguration = in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X) ? in.readOptionalWriteable(RolloverConfiguration::new) : null; - globalRetention = in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION) + globalRetention = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) ? in.readOptionalWriteable(DataStreamGlobalRetention::read) : null; } @@ -110,7 +110,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeOptionalWriteable(rolloverConfiguration); } - if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalWriteable(globalRetention); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java new file mode 100644 index 0000000000000..99c2d994a8bd0 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.bulk; + +import org.elasticsearch.features.FeatureSpecification; +import org.elasticsearch.features.NodeFeature; + +import java.util.Set; + +import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION; + +public class BulkFeatures implements FeatureSpecification { + public Set getFeatures() { + return Set.of(SIMULATE_MAPPING_VALIDATION); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java b/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java index 9ac7736815cc3..cc9f9b8ee1ce7 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java @@ -96,7 +96,7 @@ private static XContentBuilder createSource( if (source.routing() != null) { builder.field("routing", source.routing()); } - builder.field("index", source.index()); + builder.field("index", targetIndexName); // Unmapped source field builder.startObject("source"); { diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index 7591ef402847e..fc9df7bbf73b9 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -115,7 +115,7 @@ public TransportShardBulkAction( BulkShardRequest::new, BulkShardRequest::new, ExecutorSelector.getWriteExecutorForShard(threadPool), - false, + PrimaryActionExecution.RejectOnOverload, indexingPressure, systemIndices ); @@ -362,7 +362,8 @@ static boolean executeBulkItemRequest( ); } else { final IndexRequest request = context.getRequestToExecute(); - DocumentSizeObserver documentSizeObserver = getDocumentSizeObserver(documentParsingProvider, request); + + DocumentSizeObserver documentSizeObserver = documentParsingProvider.newDocumentSizeObserver(request); context.setDocumentSizeObserver(documentSizeObserver); final SourceToParse sourceToParse = new SourceToParse( @@ -458,25 +459,6 @@ public void onFailure(Exception e) { return false; } - /** - * Creates a new document size observer - * @param documentParsingProvider a provider to create a new observer. - * @param request an index request to provide information about bytes being already parsed. - * @return a Fixed version of DocumentSizeObserver if parsing already happened (in IngestService, UpdateHelper) - * and there is a value to be reported >0 - * It would be pre-populated with information about how many bytes were already parsed - * or a noop instance if parsed bytes in IngestService/UpdateHelper was 0 (like when empty doc or script in update) - * or return a new DocumentSizeObserver that will be used when parsing. - */ - private static DocumentSizeObserver getDocumentSizeObserver(DocumentParsingProvider documentParsingProvider, IndexRequest request) { - if (request.getNormalisedBytesParsed() > 0) { - return documentParsingProvider.newFixedSizeDocumentObserver(request.getNormalisedBytesParsed()); - } else if (request.getNormalisedBytesParsed() == 0) { - return DocumentSizeObserver.EMPTY_INSTANCE; - } // request.getNormalisedBytesParsed() -1, meaning normalisedBytesParsed isn't set as parsing wasn't done yet - return documentParsingProvider.newDocumentSizeObserver(); - } - private static Engine.Result exceptionToResult(Exception e, IndexShard primary, boolean isDelete, long version, String id) { assert id != null; return isDelete ? primary.getFailedDeleteResult(e, version, id) : primary.getFailedIndexResult(e, version, id); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 95c1c0ce05d89..c08ed6413a7a1 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -13,14 +13,25 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.ingest.SimulateIndexResponse; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexingPressure; +import org.elasticsearch.index.VersionType; +import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.seqno.SequenceNumbers; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.ingest.SimulateIngestService; +import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -33,6 +44,8 @@ * shards are not actually modified). */ public class TransportSimulateBulkAction extends TransportAbstractBulkAction { + public static final NodeFeature SIMULATE_MAPPING_VALIDATION = new NodeFeature("simulate.mapping.validation"); + private final IndicesService indicesService; @Inject public TransportSimulateBulkAction( @@ -42,7 +55,8 @@ public TransportSimulateBulkAction( IngestService ingestService, ActionFilters actionFilters, IndexingPressure indexingPressure, - SystemIndices systemIndices + SystemIndices systemIndices, + IndicesService indicesService ) { super( SimulateBulkAction.INSTANCE, @@ -56,6 +70,7 @@ public TransportSimulateBulkAction( systemIndices, System::nanoTime ); + this.indicesService = indicesService; } @Override @@ -71,7 +86,7 @@ protected void doInternalExecute( DocWriteRequest docRequest = bulkRequest.requests.get(i); assert docRequest instanceof IndexRequest : "TransportSimulateBulkAction should only ever be called with IndexRequests"; IndexRequest request = (IndexRequest) docRequest; - + Exception mappingValidationException = validateMappings(request); responses.set( i, BulkItemResponse.success( @@ -84,7 +99,7 @@ protected void doInternalExecute( request.source(), request.getContentType(), request.getExecutedPipelines(), - null + mappingValidationException ) ) ); @@ -94,6 +109,52 @@ protected void doInternalExecute( ); } + /** + * This creates a temporary index with the mappings of the index in the request, and then attempts to index the source from the request + * into it. If there is a mapping exception, that exception is returned. On success the returned exception is null. + * @param request The IndexRequest whose source will be validated against the mapping (if it exists) of its index + * @return a mapping exception if the source does not match the mappings, otherwise null + */ + private Exception validateMappings(IndexRequest request) { + final SourceToParse sourceToParse = new SourceToParse( + request.id(), + request.source(), + request.getContentType(), + request.routing(), + request.getDynamicTemplates(), + DocumentSizeObserver.EMPTY_INSTANCE + ); + + ClusterState state = clusterService.state(); + Exception mappingValidationException = null; + IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(request.index()); + if (indexAbstraction != null) { + IndexMetadata imd = state.metadata().getIndexSafe(indexAbstraction.getWriteIndex(request, state.metadata())); + try { + indicesService.withTempIndexService(imd, indexService -> { + indexService.mapperService().updateMapping(null, imd); + return IndexShard.prepareIndex( + indexService.mapperService(), + sourceToParse, + SequenceNumbers.UNASSIGNED_SEQ_NO, + -1, + -1, + VersionType.INTERNAL, + Engine.Operation.Origin.PRIMARY, + Long.MIN_VALUE, + false, + request.ifSeqNo(), + request.ifPrimaryTerm(), + 0 + ); + }); + } catch (Exception e) { + mappingValidationException = e; + } + } + return mappingValidationException; + } + /* * This overrides TransportSimulateBulkAction's getIngestService to allow us to provide an IngestService that handles pipeline * substitutions defined in the request. diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index 841a2df5eada6..c08269d2dd6e8 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -511,7 +511,7 @@ public Response(StreamInput in) throws IOException { this( in.readCollectionAsList(DataStreamInfo::new), in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X) ? in.readOptionalWriteable(RolloverConfiguration::new) : null, - in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION) + in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) ? in.readOptionalWriteable(DataStreamGlobalRetention::read) : null ); @@ -537,7 +537,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeOptionalWriteable(rolloverConfiguration); } - if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalWriteable(globalRetention); } } diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java index 579fb83a89f7e..d8fa2ecb727e5 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java @@ -161,7 +161,7 @@ public Response(StreamInput in) throws IOException { super(in); this.indices = in.readCollectionAsList(ExplainIndexDataStreamLifecycle::new); this.rolloverConfiguration = in.readOptionalWriteable(RolloverConfiguration::new); - this.globalRetention = in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION) + this.globalRetention = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) ? in.readOptionalWriteable(DataStreamGlobalRetention::read) : null; } @@ -182,7 +182,7 @@ public DataStreamGlobalRetention getGlobalRetention() { public void writeTo(StreamOutput out) throws IOException { out.writeCollection(indices); out.writeOptionalWriteable(rolloverConfiguration); - if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalWriteable(globalRetention); } } diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java index 9907310a8acbd..a10f95452da81 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java @@ -220,7 +220,7 @@ public Response(StreamInput in) throws IOException { this( in.readCollectionAsList(DataStreamLifecycle::new), in.readOptionalWriteable(RolloverConfiguration::new), - in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION) + in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) ? in.readOptionalWriteable(DataStreamGlobalRetention::read) : null ); @@ -243,7 +243,7 @@ public DataStreamGlobalRetention getGlobalRetention() { public void writeTo(StreamOutput out) throws IOException { out.writeCollection(dataStreamLifecycles); out.writeOptionalWriteable(rolloverConfiguration); - if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalWriteable(globalRetention); } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index 54d6739c4e460..26b9929fd166b 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.fieldcaps; import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.automaton.TooComplexToDeterminizeException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; @@ -111,11 +112,25 @@ public TransportFieldCapabilitiesAction( @Override protected void doExecute(Task task, FieldCapabilitiesRequest request, final ActionListener listener) { + executeRequest(task, request, REMOTE_TYPE, listener); + } + + public void executeRequest( + Task task, + FieldCapabilitiesRequest request, + RemoteClusterActionType remoteAction, + ActionListener listener + ) { // workaround for https://github.com/elastic/elasticsearch/issues/97916 - TODO remove this when we can - searchCoordinationExecutor.execute(ActionRunnable.wrap(listener, l -> doExecuteForked(task, request, l))); + searchCoordinationExecutor.execute(ActionRunnable.wrap(listener, l -> doExecuteForked(task, request, remoteAction, l))); } - private void doExecuteForked(Task task, FieldCapabilitiesRequest request, final ActionListener listener) { + private void doExecuteForked( + Task task, + FieldCapabilitiesRequest request, + RemoteClusterActionType remoteAction, + ActionListener listener + ) { if (ccsCheckCompatibility) { checkCCSVersionCompatibility(request); } @@ -249,7 +264,7 @@ private void doExecuteForked(Task task, FieldCapabilitiesRequest request, final } }); remoteClusterClient.execute( - TransportFieldCapabilitiesAction.REMOTE_TYPE, + remoteAction, remoteRequest, // The underlying transport service may call onFailure with a thread pool other than search_coordinator. // This fork is a workaround to ensure that the merging of field-caps always occurs on the search_coordinator. @@ -265,9 +280,14 @@ private void doExecuteForked(Task task, FieldCapabilitiesRequest request, final } private static void checkIndexBlocks(ClusterState clusterState, String[] concreteIndices) { - clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ); + var blocks = clusterState.blocks(); + if (blocks.global().isEmpty() && blocks.indices().isEmpty()) { + // short circuit optimization because block check below is relatively expensive for many indices + return; + } + blocks.globalBlockedRaiseException(ClusterBlockLevel.READ); for (String index : concreteIndices) { - clusterState.blocks().indexBlockedRaiseException(ClusterBlockLevel.READ, index); + blocks.indexBlockedRaiseException(ClusterBlockLevel.READ, index); } } @@ -538,7 +558,12 @@ public void messageReceived(FieldCapabilitiesNodeRequest request, TransportChann .stream() .collect(Collectors.groupingBy(ShardId::getIndexName)); final FieldCapabilitiesFetcher fetcher = new FieldCapabilitiesFetcher(indicesService, request.includeEmptyFields()); - final Predicate fieldNameFilter = Regex.simpleMatcher(request.fields()); + Predicate fieldNameFilter; + try { + fieldNameFilter = Regex.simpleMatcher(request.fields()); + } catch (TooComplexToDeterminizeException e) { + throw new IllegalArgumentException("The field names are too complex to process. " + e.getMessage()); + } for (List shardIds : groupedShardIds.values()) { final Map failures = new HashMap<>(); final Set unmatched = new HashSet<>(); diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index efe43fdff4efd..5463f9fec4d2a 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -146,6 +146,7 @@ public class IndexRequest extends ReplicatedWriteRequest implement */ private Object rawTimestamp; private long normalisedBytesParsed = -1; + private boolean originatesFromUpdateByScript; public IndexRequest(StreamInput in) throws IOException { this(null, in); @@ -197,6 +198,12 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio } else { requireDataStream = false; } + + if (in.getTransportVersion().onOrAfter(TransportVersions.INDEX_REQUEST_UPDATE_BY_SCRIPT_ORIGIN)) { + originatesFromUpdateByScript = in.readBoolean(); + } else { + originatesFromUpdateByScript = false; + } } public IndexRequest() { @@ -757,6 +764,10 @@ private void writeBody(StreamOutput out) throws IOException { out.writeBoolean(requireDataStream); out.writeZLong(normalisedBytesParsed); } + + if (out.getTransportVersion().onOrAfter(TransportVersions.INDEX_REQUEST_UPDATE_BY_SCRIPT_ORIGIN)) { + out.writeBoolean(originatesFromUpdateByScript); + } } @Override @@ -959,4 +970,13 @@ public List getExecutedPipelines() { return Collections.unmodifiableList(executedPipelines); } } + + public IndexRequest setOriginatesFromUpdateByScript(boolean originatesFromUpdateByScript) { + this.originatesFromUpdateByScript = originatesFromUpdateByScript; + return this; + } + + public boolean originatesFromUpdateByScript() { + return this.originatesFromUpdateByScript; + } } diff --git a/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java b/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java index 4684c990299f9..5a891f33480fa 100644 --- a/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java @@ -71,7 +71,7 @@ public TransportResyncReplicationAction( ResyncReplicationRequest::new, ResyncReplicationRequest::new, ExecutorSelector.getWriteExecutorForShard(threadPool), - true, /* we should never reject resync because of thread pool capacity on primary */ + PrimaryActionExecution.Force, /* we should never reject resync because of thread pool capacity on primary */ indexingPressure, systemIndices ); diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java index ae0a0c40f1267..4fd551994e2a0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java @@ -319,7 +319,7 @@ protected void performPhaseOnShard(final int shardIndex, final SearchShardIterat Runnable r = () -> { final Thread thread = Thread.currentThread(); try { - executePhaseOnShard(shardIt, shard, new SearchActionListener(shard, shardIndex) { + executePhaseOnShard(shardIt, shard, new SearchActionListener<>(shard, shardIndex) { @Override public void innerOnResponse(Result result) { try { diff --git a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java index 52f41179795d6..30460593849c5 100644 --- a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java @@ -9,11 +9,11 @@ package org.elasticsearch.action.search; import org.apache.logging.log4j.Logger; -import org.apache.lucene.util.CollectionUtil; import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.routing.GroupShardsIterator; +import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.CountDown; import org.elasticsearch.core.Nullable; @@ -22,7 +22,6 @@ import org.elasticsearch.search.CanMatchShardResponse; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.SearchShardTarget; -import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.search.sort.FieldSortBuilder; @@ -32,6 +31,7 @@ import org.elasticsearch.transport.Transport; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -42,7 +42,6 @@ import java.util.concurrent.atomic.AtomicReferenceArray; import java.util.function.BiFunction; import java.util.stream.Collectors; -import java.util.stream.IntStream; import java.util.stream.Stream; import static org.elasticsearch.core.Strings.format; @@ -107,18 +106,22 @@ final class CanMatchPreFilterSearchPhase extends SearchPhase { this.requireAtLeastOneMatch = requireAtLeastOneMatch; this.coordinatorRewriteContextProvider = coordinatorRewriteContextProvider; this.executor = executor; - this.shardItIndexMap = new HashMap<>(); results = new CanMatchSearchPhaseResults(shardsIts.size()); // we compute the shard index based on the natural order of the shards // that participate in the search request. This means that this number is // consistent between two requests that target the same shards. - List naturalOrder = new ArrayList<>(); - shardsIts.iterator().forEachRemaining(naturalOrder::add); - CollectionUtil.timSort(naturalOrder); - for (int i = 0; i < naturalOrder.size(); i++) { - shardItIndexMap.put(naturalOrder.get(i), i); + final SearchShardIterator[] naturalOrder = new SearchShardIterator[shardsIts.size()]; + int i = 0; + for (SearchShardIterator shardsIt : shardsIts) { + naturalOrder[i++] = shardsIt; + } + Arrays.sort(naturalOrder); + final Map shardItIndexMap = Maps.newHashMapWithExpectedSize(naturalOrder.length); + for (int j = 0; j < naturalOrder.length; j++) { + shardItIndexMap.put(naturalOrder[j], j); } + this.shardItIndexMap = shardItIndexMap; } private static boolean assertSearchCoordinationThread() { @@ -443,7 +446,11 @@ private static final class CanMatchSearchPhaseResults extends SearchPhaseResults @Override void consumeResult(CanMatchShardResponse result, Runnable next) { try { - consumeResult(result.getShardIndex(), result.canMatch(), result.estimatedMinAndMax()); + final boolean canMatch = result.canMatch(); + final MinAndMax minAndMax = result.estimatedMinAndMax(); + if (canMatch || minAndMax != null) { + consumeResult(result.getShardIndex(), canMatch, minAndMax); + } } finally { next.run(); } @@ -460,7 +467,7 @@ void consumeShardFailure(int shardIndex) { consumeResult(shardIndex, true, null); } - synchronized void consumeResult(int shardIndex, boolean canMatch, MinAndMax minAndMax) { + private synchronized void consumeResult(int shardIndex, boolean canMatch, MinAndMax minAndMax) { if (canMatch) { possibleMatches.set(shardIndex); numPossibleMatches++; @@ -489,10 +496,9 @@ private GroupShardsIterator getIterator( CanMatchSearchPhaseResults results, GroupShardsIterator shardsIts ) { - int cardinality = results.getNumPossibleMatches(); FixedBitSet possibleMatches = results.getPossibleMatches(); // TODO: pick the local shard when possible - if (requireAtLeastOneMatch && cardinality == 0) { + if (requireAtLeastOneMatch && results.getNumPossibleMatches() == 0) { // this is a special case where we have no hit but we need to get at least one search response in order // to produce a valid search result with all the aggs etc. // Since it's possible that some of the shards that we're skipping are @@ -509,7 +515,6 @@ private GroupShardsIterator getIterator( } possibleMatches.set(shardIndexToQuery); } - SearchSourceBuilder source = request.source(); int i = 0; for (SearchShardIterator iter : shardsIts) { iter.reset(); @@ -523,7 +528,7 @@ private GroupShardsIterator getIterator( if (shouldSortShards(results.minAndMaxes) == false) { return shardsIts; } - FieldSortBuilder fieldSort = FieldSortBuilder.getPrimaryFieldSortOrNull(source); + FieldSortBuilder fieldSort = FieldSortBuilder.getPrimaryFieldSortOrNull(request.source()); return new GroupShardsIterator<>(sortShards(shardsIts, results.minAndMaxes, fieldSort.order())); } @@ -532,11 +537,24 @@ private static List sortShards( MinAndMax[] minAndMaxes, SortOrder order ) { - return IntStream.range(0, shardsIts.size()) - .boxed() - .sorted(shardComparator(shardsIts, minAndMaxes, order)) - .map(shardsIts::get) - .toList(); + int bound = shardsIts.size(); + List toSort = new ArrayList<>(bound); + for (int i = 0; i < bound; i++) { + toSort.add(i); + } + Comparator> keyComparator = forciblyCast(MinAndMax.getComparator(order)); + toSort.sort((idx1, idx2) -> { + int res = keyComparator.compare(minAndMaxes[idx1], minAndMaxes[idx2]); + if (res != 0) { + return res; + } + return shardsIts.get(idx1).compareTo(shardsIts.get(idx2)); + }); + List list = new ArrayList<>(bound); + for (Integer integer : toSort) { + list.add(shardsIts.get(integer)); + } + return list; } private static boolean shouldSortShards(MinAndMax[] minAndMaxes) { @@ -554,17 +572,4 @@ private static boolean shouldSortShards(MinAndMax[] minAndMaxes) { return clazz != null; } - private static Comparator shardComparator( - GroupShardsIterator shardsIts, - MinAndMax[] minAndMaxes, - SortOrder order - ) { - final Comparator comparator = Comparator.comparing( - index -> minAndMaxes[index], - forciblyCast(MinAndMax.getComparator(order)) - ); - - return comparator.thenComparing(shardsIts::get); - } - } diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index e8470ba77632f..e2385745149c1 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -100,6 +100,10 @@ private void doRun() { if (hit.getInnerHits() == null) { hit.setInnerHits(Maps.newMapWithExpectedSize(innerHitBuilders.size())); } + if (hit.isPooled() == false) { + // TODO: make this work pooled by forcing the hit itself to become pooled as needed here + innerHits = innerHits.asUnpooled(); + } hit.getInnerHits().put(innerHitBuilder.getName(), innerHits); assert innerHits.isPooled() == false || hit.isPooled() : "pooled inner hits can only be added to a pooled hit"; innerHits.mustIncRef(); diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchExecutionStatsCollector.java b/server/src/main/java/org/elasticsearch/action/search/SearchExecutionStatsCollector.java index efe7c2f9891be..4b5fbe22b28e5 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchExecutionStatsCollector.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchExecutionStatsCollector.java @@ -38,7 +38,7 @@ public final class SearchExecutionStatsCollector extends DelegatingActionListene @SuppressWarnings("unchecked") public static - BiFunction, ActionListener> + BiFunction, ActionListener> makeWrapper(ResponseCollectorService service) { return (connection, originalListener) -> new SearchExecutionStatsCollector( (ActionListener) originalListener, diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java index 62b39dd675387..92c26d2db7a33 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java @@ -147,7 +147,7 @@ private void run(BiFunction clusterNodeLookup, fi // we can't create a SearchShardTarget here since we don't know the index and shard ID we are talking to // we only know the node and the search context ID. Yet, the response will contain the SearchShardTarget // from the target node instead...that's why we pass null here - SearchActionListener searchActionListener = new SearchActionListener(null, shardIndex) { + SearchActionListener searchActionListener = new SearchActionListener<>(null, shardIndex) { @Override protected void setSearchShardTarget(T response) { @@ -215,7 +215,7 @@ private synchronized void addShardFailure(ShardSearchFailure failure) { protected abstract void executeInitialPhase( Transport.Connection connection, InternalScrollSearchRequest internalRequest, - SearchActionListener searchActionListener + ActionListener searchActionListener ); protected abstract SearchPhase moveToNextPhase(BiFunction clusterNodeLookup); diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java index 44a7525b9aef9..c80c62f3387ea 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java @@ -43,7 +43,7 @@ final class SearchScrollQueryAndFetchAsyncAction extends SearchScrollAsyncAction protected void executeInitialPhase( Transport.Connection connection, InternalScrollSearchRequest internalRequest, - SearchActionListener searchActionListener + ActionListener searchActionListener ) { searchTransportService.sendExecuteScrollFetch(connection, internalRequest, task, searchActionListener); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java index 793a5bfe4e9d4..56472e877ed48 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java @@ -55,7 +55,7 @@ protected void onFirstPhaseResult(int shardId, ScrollQuerySearchResult result) { protected void executeInitialPhase( Transport.Connection connection, InternalScrollSearchRequest internalRequest, - SearchActionListener searchActionListener + ActionListener searchActionListener ) { searchTransportService.sendExecuteScrollQuery(connection, internalRequest, task, searchActionListener); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java index 320fd3995bd48..2b1bbffa7e36f 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java @@ -19,7 +19,6 @@ import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.ShardSearchContextId; -import java.util.Comparator; import java.util.List; import java.util.Objects; @@ -174,14 +173,24 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(clusterAlias, shardId); + var clusterAlias = this.clusterAlias; + return 31 * (31 + (clusterAlias == null ? 0 : clusterAlias.hashCode())) + shardId.hashCode(); } - private static final Comparator COMPARATOR = Comparator.comparing(SearchShardIterator::shardId) - .thenComparing(SearchShardIterator::getClusterAlias, Comparator.nullsFirst(String::compareTo)); - @Override public int compareTo(SearchShardIterator o) { - return COMPARATOR.compare(this, o); + int res = shardId.compareTo(o.shardId); + if (res != 0) { + return res; + } + var thisClusterAlias = clusterAlias; + var otherClusterAlias = o.clusterAlias; + if (thisClusterAlias == null) { + return otherClusterAlias == null ? 0 : -1; + } else if (otherClusterAlias == null) { + return 1; + } else { + return thisClusterAlias.compareTo(otherClusterAlias); + } } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java index 9713d804ddc13..fb3c49d83cb93 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java @@ -116,7 +116,7 @@ public class SearchTransportService { private final NodeClient client; private final BiFunction< Transport.Connection, - SearchActionListener, + ActionListener, ActionListener> responseWrapper; private final Map clientConnections = ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency(); @@ -125,7 +125,7 @@ public SearchTransportService( NodeClient client, BiFunction< Transport.Connection, - SearchActionListener, + ActionListener, ActionListener> responseWrapper ) { this.transportService = transportService; @@ -133,6 +133,13 @@ public SearchTransportService( this.responseWrapper = responseWrapper; } + private static final ActionListenerResponseHandler SEND_FREE_CONTEXT_LISTENER = + new ActionListenerResponseHandler<>( + ActionListener.noop(), + SearchFreeContextResponse::readFrom, + TransportResponseHandler.TRANSPORT_WORKER + ); + public void sendFreeContext(Transport.Connection connection, final ShardSearchContextId contextId, OriginalIndices originalIndices) { transportService.sendRequest( connection, @@ -140,11 +147,7 @@ public void sendFreeContext(Transport.Connection connection, final ShardSearchCo new SearchFreeContextRequest(originalIndices, contextId), TransportRequestOptions.EMPTY, // no need to respond if it was freed or not - new ActionListenerResponseHandler<>( - ActionListener.noop(), - SearchFreeContextResponse::new, - TransportResponseHandler.TRANSPORT_WORKER - ) + SEND_FREE_CONTEXT_LISTENER ); } @@ -158,7 +161,7 @@ public void sendFreeContext( FREE_CONTEXT_SCROLL_ACTION_NAME, new ScrollFreeContextRequest(contextId), TransportRequestOptions.EMPTY, - new ActionListenerResponseHandler<>(listener, SearchFreeContextResponse::new, TransportResponseHandler.TRANSPORT_WORKER) + new ActionListenerResponseHandler<>(listener, SearchFreeContextResponse::readFrom, TransportResponseHandler.TRANSPORT_WORKER) ); } @@ -173,7 +176,6 @@ public void sendCanMatch( QUERY_CAN_MATCH_NODE_NAME, request, task, - TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>(listener, CanMatchNodeResponse::new, TransportResponseHandler.TRANSPORT_WORKER) ); } @@ -184,11 +186,7 @@ public void sendClearAllScrollContexts(Transport.Connection connection, final Ac CLEAR_SCROLL_CONTEXTS_ACTION_NAME, new ClearScrollContextsRequest(), TransportRequestOptions.EMPTY, - new ActionListenerResponseHandler<>( - listener, - (in) -> TransportResponse.Empty.INSTANCE, - TransportResponseHandler.TRANSPORT_WORKER - ) + new ActionListenerResponseHandler<>(listener, in -> TransportResponse.Empty.INSTANCE, TransportResponseHandler.TRANSPORT_WORKER) ); } @@ -196,7 +194,7 @@ public void sendExecuteDfs( Transport.Connection connection, final ShardSearchRequest request, SearchTask task, - final SearchActionListener listener + final ActionListener listener ) { transportService.sendChildRequest( connection, @@ -211,7 +209,7 @@ public void sendExecuteQuery( Transport.Connection connection, final ShardSearchRequest request, SearchTask task, - final SearchActionListener listener + final ActionListener listener ) { // we optimize this and expect a QueryFetchSearchResult if we only have a single shard in the search request // this used to be the QUERY_AND_FETCH which doesn't exist anymore. @@ -233,7 +231,7 @@ public void sendExecuteQuery( Transport.Connection connection, final QuerySearchRequest request, SearchTask task, - final SearchActionListener listener + final ActionListener listener ) { transportService.sendChildRequest( connection, @@ -248,7 +246,7 @@ public void sendExecuteScrollQuery( Transport.Connection connection, final InternalScrollSearchRequest request, SearchTask task, - final SearchActionListener listener + final ActionListener listener ) { transportService.sendChildRequest( connection, @@ -263,7 +261,7 @@ public void sendExecuteRankFeature( Transport.Connection connection, final RankFeatureShardRequest request, SearchTask task, - final SearchActionListener listener + final ActionListener listener ) { transportService.sendChildRequest( connection, @@ -278,7 +276,7 @@ public void sendExecuteScrollFetch( Transport.Connection connection, final InternalScrollSearchRequest request, SearchTask task, - final SearchActionListener listener + final ActionListener listener ) { transportService.sendChildRequest( connection, @@ -293,7 +291,7 @@ public void sendExecuteFetch( Transport.Connection connection, final ShardFetchSearchRequest request, SearchTask task, - final SearchActionListener listener + final ActionListener listener ) { sendExecuteFetch(connection, FETCH_ID_ACTION_NAME, request, task, listener); } @@ -302,7 +300,7 @@ public void sendExecuteFetchScroll( Transport.Connection connection, final ShardFetchRequest request, SearchTask task, - final SearchActionListener listener + final ActionListener listener ) { sendExecuteFetch(connection, FETCH_ID_SCROLL_ACTION_NAME, request, task, listener); } @@ -312,7 +310,7 @@ private void sendExecuteFetch( String action, final ShardFetchRequest request, SearchTask task, - final SearchActionListener listener + final ActionListener listener ) { transportService.sendChildRequest( connection, @@ -420,13 +418,20 @@ public IndicesOptions indicesOptions() { public static class SearchFreeContextResponse extends TransportResponse { + private static final SearchFreeContextResponse FREED = new SearchFreeContextResponse(true); + private static final SearchFreeContextResponse NOT_FREED = new SearchFreeContextResponse(false); + private final boolean freed; - SearchFreeContextResponse(StreamInput in) throws IOException { - freed = in.readBoolean(); + static SearchFreeContextResponse readFrom(StreamInput in) throws IOException { + return of(in.readBoolean()); } - SearchFreeContextResponse(boolean freed) { + static SearchFreeContextResponse of(boolean freed) { + return freed ? FREED : NOT_FREED; + } + + private SearchFreeContextResponse(boolean freed) { this.freed = freed; } @@ -448,7 +453,7 @@ public static void registerRequestHandler( final TransportRequestHandler freeContextHandler = (request, channel, task) -> { logger.trace("releasing search context [{}]", request.id()); boolean freed = searchService.freeReaderContext(request.id()); - channel.sendResponse(new SearchFreeContextResponse(freed)); + channel.sendResponse(SearchFreeContextResponse.of(freed)); }; transportService.registerRequestHandler( FREE_CONTEXT_SCROLL_ACTION_NAME, @@ -456,7 +461,12 @@ public static void registerRequestHandler( ScrollFreeContextRequest::new, instrumentedHandler(FREE_CONTEXT_SCROLL_ACTION_METRIC, transportService, searchTransportMetrics, freeContextHandler) ); - TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_SCROLL_ACTION_NAME, false, SearchFreeContextResponse::new); + TransportActionProxy.registerProxyAction( + transportService, + FREE_CONTEXT_SCROLL_ACTION_NAME, + false, + SearchFreeContextResponse::readFrom + ); transportService.registerRequestHandler( FREE_CONTEXT_ACTION_NAME, @@ -464,7 +474,7 @@ public static void registerRequestHandler( SearchFreeContextRequest::new, instrumentedHandler(FREE_CONTEXT_ACTION_METRIC, transportService, searchTransportMetrics, freeContextHandler) ); - TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_ACTION_NAME, false, SearchFreeContextResponse::new); + TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_ACTION_NAME, false, SearchFreeContextResponse::readFrom); transportService.registerRequestHandler( CLEAR_SCROLL_CONTEXTS_ACTION_NAME, diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 370bd49c0523b..60a69dd6a7450 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -48,6 +48,7 @@ import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.util.ArrayUtils; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.util.Maps; @@ -101,10 +102,8 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.BiFunction; -import java.util.function.BooleanSupplier; import java.util.function.Function; import java.util.function.LongSupplier; -import java.util.stream.StreamSupport; import static org.elasticsearch.action.search.SearchType.DFS_QUERY_THEN_FETCH; import static org.elasticsearch.action.search.SearchType.QUERY_THEN_FETCH; @@ -199,9 +198,14 @@ private Map buildPerIndexOriginalIndices( String[] indices, IndicesOptions indicesOptions ) { - Map res = new HashMap<>(); + Map res = Maps.newMapWithExpectedSize(indices.length); + var blocks = clusterState.blocks(); + // optimization: mostly we do not have any blocks so there's no point in the expensive per-index checking + boolean hasBlocks = blocks.global().isEmpty() == false || blocks.indices().isEmpty() == false; for (String index : indices) { - clusterState.blocks().indexBlockedRaiseException(ClusterBlockLevel.READ, index); + if (hasBlocks) { + blocks.indexBlockedRaiseException(ClusterBlockLevel.READ, index); + } String[] aliases = indexNameExpressionResolver.indexAliases( clusterState, @@ -211,23 +215,27 @@ private Map buildPerIndexOriginalIndices( true, indicesAndAliases ); - BooleanSupplier hasDataStreamRef = () -> { - IndexAbstraction ret = clusterState.getMetadata().getIndicesLookup().get(index); - if (ret == null || ret.getParentDataStream() == null) { - return false; - } - return indicesAndAliases.contains(ret.getParentDataStream().getName()); - }; - List finalIndices = new ArrayList<>(); - if (aliases == null || aliases.length == 0 || indicesAndAliases.contains(index) || hasDataStreamRef.getAsBoolean()) { - finalIndices.add(index); + String[] finalIndices = Strings.EMPTY_ARRAY; + if (aliases == null + || aliases.length == 0 + || indicesAndAliases.contains(index) + || hasDataStreamRef(clusterState, indicesAndAliases, index)) { + finalIndices = new String[] { index }; } if (aliases != null) { - finalIndices.addAll(Arrays.asList(aliases)); + finalIndices = finalIndices.length == 0 ? aliases : ArrayUtils.concat(finalIndices, aliases); } - res.put(index, new OriginalIndices(finalIndices.toArray(String[]::new), indicesOptions)); + res.put(index, new OriginalIndices(finalIndices, indicesOptions)); } - return Collections.unmodifiableMap(res); + return res; + } + + private static boolean hasDataStreamRef(ClusterState clusterState, Set indicesAndAliases, String index) { + IndexAbstraction ret = clusterState.getMetadata().getIndicesLookup().get(index); + if (ret == null || ret.getParentDataStream() == null) { + return false; + } + return indicesAndAliases.contains(ret.getParentDataStream().getName()); } Map buildIndexAliasFilters(ClusterState clusterState, Set indicesAndAliases, Index[] concreteIndices) { @@ -773,6 +781,7 @@ static SearchResponseMerger createSearchResponseMerger( } /** + * Collect remote search shards that we need to search for potential matches. * Used for ccs_minimize_roundtrips=false */ static void collectSearchShards( @@ -1050,6 +1059,10 @@ static BiFunction getRemoteClusterNodeLookup(Map< }; } + /** + * Produce a list of {@link SearchShardIterator}s from the set of responses from remote clusters. + * Used for ccs_minimize_roundtrips=false. + */ static List getRemoteShardsIterator( Map searchShardsResponses, Map remoteIndicesByCluster, @@ -1147,6 +1160,9 @@ private static boolean checkAllRemotePITShardsWereReturnedBySearchShards( .allMatch(searchContextIdForNode -> searchContextIdForNode.getClusterAlias() == null); } + /** + * If any of the indices we are searching are frozen, issue deprecation warning. + */ void frozenIndexCheck(ResolvedIndices resolvedIndices) { List frozenIndices = new ArrayList<>(); Map indexMetadataMap = resolvedIndices.getConcreteLocalIndicesMetadata(); @@ -1166,6 +1182,10 @@ void frozenIndexCheck(ResolvedIndices resolvedIndices) { } } + /** + * Execute search locally and for all given remote shards. + * Used when minimize_roundtrips=false or for local search. + */ private void executeSearch( SearchTask task, SearchTimeProvider timeProvider, @@ -1307,21 +1327,30 @@ static boolean shouldPreFilterSearchShards( int numShards, int defaultPreFilterShardSize ) { + if (searchRequest.searchType() != QUERY_THEN_FETCH) { + // we can't do this for DFS it needs to fan out to all shards all the time + return false; + } SearchSourceBuilder source = searchRequest.source(); Integer preFilterShardSize = searchRequest.getPreFilterShardSize(); - if (preFilterShardSize == null && (hasReadOnlyIndices(indices, clusterState) || hasPrimaryFieldSort(source))) { - preFilterShardSize = 1; - } else if (preFilterShardSize == null) { - preFilterShardSize = defaultPreFilterShardSize; + if (preFilterShardSize == null) { + if (hasReadOnlyIndices(indices, clusterState) || hasPrimaryFieldSort(source)) { + preFilterShardSize = 1; + } else { + preFilterShardSize = defaultPreFilterShardSize; + } } - return searchRequest.searchType() == QUERY_THEN_FETCH // we can't do this for DFS it needs to fan out to all shards all the time - && (SearchService.canRewriteToMatchNone(source) || hasPrimaryFieldSort(source)) - && preFilterShardSize < numShards; + return preFilterShardSize < numShards && (SearchService.canRewriteToMatchNone(source) || hasPrimaryFieldSort(source)); } private static boolean hasReadOnlyIndices(String[] indices, ClusterState clusterState) { + var blocks = clusterState.blocks(); + if (blocks.global().isEmpty() && blocks.indices().isEmpty()) { + // short circuit optimization because block check below is relatively expensive for many indices + return false; + } for (String index : indices) { - ClusterBlockException writeBlock = clusterState.blocks().indexBlockedException(ClusterBlockLevel.WRITE, index); + ClusterBlockException writeBlock = blocks.indexBlockedException(ClusterBlockLevel.WRITE, index); if (writeBlock != null) { return true; } @@ -1329,12 +1358,17 @@ private static boolean hasReadOnlyIndices(String[] indices, ClusterState cluster return false; } + // package private for testing static GroupShardsIterator mergeShardsIterators( List localShardIterators, List remoteShardIterators ) { - List shards = new ArrayList<>(remoteShardIterators); - shards.addAll(localShardIterators); + final List shards; + if (remoteShardIterators.isEmpty()) { + shards = localShardIterators; + } else { + shards = CollectionUtils.concatLists(remoteShardIterators, localShardIterators); + } return GroupShardsIterator.sortAndCreate(shards); } @@ -1562,6 +1596,11 @@ private static void failIfOverShardCountLimit(ClusterService clusterService, int } } + /** + * {@link ActionListener} suitable for collecting cross-cluster responses. + * @param Response type we're getting as intermediate per-cluster results. + * @param Response type that the final listener expects. + */ abstract static class CCSActionListener implements ActionListener { protected final String clusterAlias; protected final boolean skipUnavailable; @@ -1595,6 +1634,9 @@ public final void onResponse(Response response) { maybeFinish(); } + /** + * Specific listener type will implement this method to process its specific partial response. + */ abstract void innerOnResponse(Response response); @Override @@ -1732,6 +1774,10 @@ static List getLocalLocalShardsIteratorFromPointInTime( return iterators; } + /** + * Create a list of {@link SearchShardIterator}s for the local indices we are searching. + * This resolves aliases and index expressions. + */ List getLocalShardsIterator( ClusterState clusterState, SearchRequest searchRequest, @@ -1755,10 +1801,15 @@ List getLocalShardsIterator( concreteIndices, searchRequest.indicesOptions() ); - return StreamSupport.stream(shardRoutings.spliterator(), false).map(it -> { - OriginalIndices finalIndices = originalIndices.get(it.shardId().getIndex().getName()); + SearchShardIterator[] list = new SearchShardIterator[shardRoutings.size()]; + int i = 0; + for (ShardIterator shardRouting : shardRoutings) { + final ShardId shardId = shardRouting.shardId(); + OriginalIndices finalIndices = originalIndices.get(shardId.getIndex().getName()); assert finalIndices != null; - return new SearchShardIterator(clusterAlias, it.shardId(), it.getShardRoutings(), finalIndices); - }).toList(); + list[i++] = new SearchShardIterator(clusterAlias, shardId, shardRouting.getShardRoutings(), finalIndices); + } + // the returned list must support in-place sorting, so this is the most memory efficient we can do here + return Arrays.asList(list); } } diff --git a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java index 69bdfdea31ae4..b9200a0e32736 100644 --- a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java @@ -40,14 +40,14 @@ protected HandledTransportAction( Writeable.Reader requestReader, Executor executor ) { - super(actionName, actionFilters, transportService.getTaskManager()); + super(actionName, actionFilters, transportService.getTaskManager(), executor); transportService.registerRequestHandler( actionName, executor, false, canTripCircuitBreaker, requestReader, - (request, channel, task) -> execute(task, request, new ChannelActionListener<>(channel)) + (request, channel, task) -> executeDirect(task, request, new ChannelActionListener<>(channel)) ); } } diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java index 33b64a9388c00..7c9dcb608ec84 100644 --- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java +++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java @@ -958,7 +958,7 @@ public void writeIndicesOptions(StreamOutput out) throws IOException { if (ignoreUnavailable()) { backwardsCompatibleOptions.add(Option.ALLOW_UNAVAILABLE_CONCRETE_TARGETS); } - if (out.getTransportVersion().onOrAfter(TransportVersions.ADD_FAILURE_STORE_INDICES_OPTIONS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { if (allowFailureIndices()) { backwardsCompatibleOptions.add(Option.ALLOW_FAILURE_INDICES); } @@ -976,7 +976,7 @@ public void writeIndicesOptions(StreamOutput out) throws IOException { states.add(WildcardStates.HIDDEN); } out.writeEnumSet(states); - if (out.getTransportVersion().onOrAfter(TransportVersions.ADD_FAILURE_STORE_INDICES_OPTIONS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { failureStoreOptions.writeTo(out); } } @@ -989,7 +989,7 @@ public static IndicesOptions readIndicesOptions(StreamInput in) throws IOExcepti options.contains(Option.EXCLUDE_ALIASES) ); boolean allowFailureIndices = true; - if (in.getTransportVersion().onOrAfter(TransportVersions.ADD_FAILURE_STORE_INDICES_OPTIONS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { allowFailureIndices = options.contains(Option.ALLOW_FAILURE_INDICES); } GatekeeperOptions gatekeeperOptions = GatekeeperOptions.builder() @@ -998,7 +998,7 @@ public static IndicesOptions readIndicesOptions(StreamInput in) throws IOExcepti .allowFailureIndices(allowFailureIndices) .ignoreThrottled(options.contains(Option.IGNORE_THROTTLED)) .build(); - FailureStoreOptions failureStoreOptions = in.getTransportVersion().onOrAfter(TransportVersions.ADD_FAILURE_STORE_INDICES_OPTIONS) + FailureStoreOptions failureStoreOptions = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) ? FailureStoreOptions.read(in) : FailureStoreOptions.DEFAULT; return new IndicesOptions( diff --git a/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java b/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java index 47fcd43f0d238..ee4433369f689 100644 --- a/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java +++ b/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.FutureUtils; import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException; -import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.threadpool.ThreadPool; @@ -369,18 +368,6 @@ private static RuntimeException unwrapEsException(ElasticsearchException esEx) { return new UncategorizedExecutionException("Failed execution", root); } - public static T get(CheckedConsumer, E> e) throws E { - PlainActionFuture fut = new PlainActionFuture<>(); - e.accept(fut); - return fut.actionGet(); - } - - public static T get(CheckedConsumer, E> e, long timeout, TimeUnit unit) throws E { - PlainActionFuture fut = new PlainActionFuture<>(); - e.accept(fut); - return fut.actionGet(timeout, unit); - } - private boolean assertCompleteAllowed() { Thread waiter = sync.getFirstQueuedThread(); assert waiter == null || allowedExecutors(waiter, Thread.currentThread()) diff --git a/server/src/main/java/org/elasticsearch/action/support/TransportAction.java b/server/src/main/java/org/elasticsearch/action/support/TransportAction.java index 222941981f05a..65a7e2302b9ae 100644 --- a/server/src/main/java/org/elasticsearch/action/support/TransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/TransportAction.java @@ -14,11 +14,14 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionRunnable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskManager; +import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicInteger; public abstract class TransportAction { @@ -26,22 +29,46 @@ public abstract class TransportAction { + void execute(Task task, Request request, ActionListener listener); + } + + protected TransportAction(String actionName, ActionFilters actionFilters, TaskManager taskManager, Executor executor) { this.actionName = actionName; this.filters = actionFilters.filters(); this.taskManager = taskManager; + this.executor = executor; } /** * Use this method when the transport action should continue to run in the context of the current task */ + protected final void executeDirect(Task task, Request request, ActionListener listener) { + handleExecution(task, request, listener, this::doExecute); + } + public final void execute(Task task, Request request, ActionListener listener) { + handleExecution( + task, + request, + listener, + executor == EsExecutors.DIRECT_EXECUTOR_SERVICE ? this::doExecute : this::doExecuteForking + ); + } + + private void handleExecution( + Task task, + Request request, + ActionListener listener, + TransportActionHandler handler + ) { final ActionRequestValidationException validationException; try { validationException = request.validate(); @@ -64,10 +91,14 @@ public final void execute(Task task, Request request, ActionListener l // Releasables#releaseOnce to avoid a double-release. request.mustIncRef(); final var releaseRef = Releasables.releaseOnce(request::decRef); - RequestFilterChain requestFilterChain = new RequestFilterChain<>(this, logger, releaseRef); + RequestFilterChain requestFilterChain = new RequestFilterChain<>(this, logger, handler, releaseRef); requestFilterChain.proceed(task, actionName, request, ActionListener.runBefore(listener, releaseRef::close)); } + private void doExecuteForking(Task task, Request request, ActionListener listener) { + executor.execute(ActionRunnable.wrap(listener, l -> doExecute(task, request, listener))); + } + protected abstract void doExecute(Task task, Request request, ActionListener listener); private static class RequestFilterChain @@ -75,13 +106,20 @@ private static class RequestFilterChain { private final TransportAction action; + private final TransportActionHandler handler; private final AtomicInteger index = new AtomicInteger(); private final Logger logger; private final Releasable releaseRef; - private RequestFilterChain(TransportAction action, Logger logger, Releasable releaseRef) { + private RequestFilterChain( + TransportAction action, + Logger logger, + TransportActionHandler handler, + Releasable releaseRef + ) { this.action = action; this.logger = logger; + this.handler = handler; this.releaseRef = releaseRef; } @@ -93,7 +131,7 @@ public void proceed(Task task, String actionName, Request request, ActionListene this.action.filters[i].apply(task, actionName, request, listener, this); } else if (i == this.action.filters.length) { try (releaseRef) { - this.action.doExecute(task, request, listener); + handler.execute(task, request, listener); } } else { listener.onFailure(new IllegalStateException("proceed was called too many times")); @@ -103,7 +141,6 @@ public void proceed(Task task, String actionName, Request request, ActionListene listener.onFailure(e); } } - } /** diff --git a/server/src/main/java/org/elasticsearch/action/support/UnsafePlainActionFuture.java b/server/src/main/java/org/elasticsearch/action/support/UnsafePlainActionFuture.java index 2d9585bd26b5f..8aa6bc4de109a 100644 --- a/server/src/main/java/org/elasticsearch/action/support/UnsafePlainActionFuture.java +++ b/server/src/main/java/org/elasticsearch/action/support/UnsafePlainActionFuture.java @@ -9,7 +9,6 @@ package org.elasticsearch.action.support; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.core.CheckedConsumer; import java.util.Objects; @@ -43,10 +42,4 @@ boolean allowedExecutors(Thread thread1, Thread thread2) { || unsafeExecutor2 == null || unsafeExecutor2.equals(EsExecutors.executorName(thread1)); } - - public static T get(CheckedConsumer, E> e, String allowedExecutor) throws E { - PlainActionFuture fut = new UnsafePlainActionFuture<>(allowedExecutor); - e.accept(fut); - return fut.actionGet(); - } } diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java index fcd513b175bb1..347edd0916fc5 100644 --- a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java @@ -78,7 +78,9 @@ protected TransportNodesAction( Writeable.Reader nodeRequest, Executor executor ) { - super(actionName, actionFilters, transportService.getTaskManager()); + // Only part of this action execution needs to be forked off - coordination can run on SAME because it's only O(#nodes) work. + // Hence the separate "finalExecutor", and why we run the whole TransportAction.execute on SAME. + super(actionName, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); assert executor.equals(EsExecutors.DIRECT_EXECUTOR_SERVICE) == false : "TransportNodesAction must always fork off the transport thread"; this.clusterService = Objects.requireNonNull(clusterService); diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index ac5b004886319..3c97bda2ef8d0 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -89,6 +89,34 @@ public abstract class TransportReplicationAction< ReplicaRequest extends ReplicationRequest, Response extends ReplicationResponse> extends TransportAction { + /** + * Execution of the primary action + */ + protected enum PrimaryActionExecution { + /** + * Is subject to usual queue length and indexing pressure checks + */ + RejectOnOverload, + /** + * Will be "forced" (bypassing queue length and indexing pressure checks) + */ + Force + } + + /** + * Global checkpoint behaviour + */ + protected enum SyncGlobalCheckpointAfterOperation { + /** + * Do not sync as part of this action + */ + DoNotSync, + /** + * Attempt to sync the global checkpoint to the replica(s) after success + */ + AttemptAfterSuccess + } + /** * The timeout for retrying replication requests. */ @@ -128,36 +156,6 @@ public abstract class TransportReplicationAction< private volatile TimeValue initialRetryBackoffBound; private volatile TimeValue retryTimeout; - protected TransportReplicationAction( - Settings settings, - String actionName, - TransportService transportService, - ClusterService clusterService, - IndicesService indicesService, - ThreadPool threadPool, - ShardStateAction shardStateAction, - ActionFilters actionFilters, - Writeable.Reader requestReader, - Writeable.Reader replicaRequestReader, - Executor executor - ) { - this( - settings, - actionName, - transportService, - clusterService, - indicesService, - threadPool, - shardStateAction, - actionFilters, - requestReader, - replicaRequestReader, - executor, - false, - false - ); - } - @SuppressWarnings("this-escape") protected TransportReplicationAction( Settings settings, @@ -171,10 +169,13 @@ protected TransportReplicationAction( Writeable.Reader requestReader, Writeable.Reader replicaRequestReader, Executor executor, - boolean syncGlobalCheckpointAfterOperation, - boolean forceExecutionOnPrimary + SyncGlobalCheckpointAfterOperation syncGlobalCheckpointAfterOperation, + PrimaryActionExecution primaryActionExecution ) { - super(actionName, actionFilters, transportService.getTaskManager()); + // TODO: consider passing the executor, investigate doExecute and let InboundHandler/TransportAction handle concurrency. + super(actionName, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); + assert syncGlobalCheckpointAfterOperation != null : "Must specify global checkpoint sync behaviour"; + assert primaryActionExecution != null : "Must specify primary action execution behaviour"; this.threadPool = threadPool; this.transportService = transportService; this.clusterService = clusterService; @@ -187,7 +188,10 @@ protected TransportReplicationAction( this.initialRetryBackoffBound = REPLICATION_INITIAL_RETRY_BACKOFF_BOUND.get(settings); this.retryTimeout = REPLICATION_RETRY_TIMEOUT.get(settings); - this.forceExecutionOnPrimary = forceExecutionOnPrimary; + this.forceExecutionOnPrimary = switch (primaryActionExecution) { + case Force -> true; + case RejectOnOverload -> false; + }; transportService.registerRequestHandler( actionName, @@ -217,7 +221,10 @@ protected TransportReplicationAction( this.transportOptions = transportOptions(); - this.syncGlobalCheckpointAfterOperation = syncGlobalCheckpointAfterOperation; + this.syncGlobalCheckpointAfterOperation = switch (syncGlobalCheckpointAfterOperation) { + case AttemptAfterSuccess -> true; + case DoNotSync -> false; + }; ClusterSettings clusterSettings = clusterService.getClusterSettings(); clusterSettings.addSettingsUpdateConsumer(REPLICATION_INITIAL_RETRY_BACKOFF_BOUND, (v) -> initialRetryBackoffBound = v); diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java index 8994b428adcbe..f380710cc0794 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java @@ -76,7 +76,7 @@ protected TransportWriteAction( Writeable.Reader request, Writeable.Reader replicaRequest, BiFunction executorFunction, - boolean forceExecutionOnPrimary, + PrimaryActionExecution primaryActionExecution, IndexingPressure indexingPressure, SystemIndices systemIndices ) { @@ -94,8 +94,8 @@ protected TransportWriteAction( request, replicaRequest, EsExecutors.DIRECT_EXECUTOR_SERVICE, - true, - forceExecutionOnPrimary + SyncGlobalCheckpointAfterOperation.AttemptAfterSuccess, + primaryActionExecution ); this.executorFunction = executorFunction; this.indexingPressure = indexingPressure; diff --git a/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java b/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java index a6a6b2c332a0a..180aa3b336149 100644 --- a/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java @@ -72,7 +72,8 @@ protected TransportSingleShardAction( Writeable.Reader request, Executor executor ) { - super(actionName, actionFilters, transportService.getTaskManager()); + // TODO: consider passing the executor, remove it from doExecute and let InboundHandler/TransportAction handle concurrency. + super(actionName, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.threadPool = threadPool; this.clusterService = clusterService; this.transportService = transportService; @@ -250,7 +251,7 @@ private class TransportHandler implements TransportRequestHandler { @Override public void messageReceived(Request request, final TransportChannel channel, Task task) throws Exception { // if we have a local operation, execute it on a thread since we don't spawn - execute(task, request, new ChannelActionListener<>(channel)); + executeDirect(task, request, new ChannelActionListener<>(channel)); } } diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java index 0738e2cb111bb..6b54654d7fbe9 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java @@ -181,7 +181,7 @@ static String calculateRouting(GetResult getResult, @Nullable IndexRequest updat Result prepareUpdateIndexRequest(ShardId shardId, UpdateRequest request, GetResult getResult, boolean detectNoop) { final IndexRequest currentRequest = request.doc(); final String routing = calculateRouting(getResult, currentRequest); - final DocumentSizeObserver documentSizeObserver = documentParsingProvider.newDocumentSizeObserver(); + final DocumentSizeObserver documentSizeObserver = documentParsingProvider.newDocumentSizeObserver(request); final Tuple> sourceAndContent = XContentHelper.convertToMap(getResult.internalSourceRef(), true); final XContentType updateSourceContentType = sourceAndContent.v1(); final Map updatedSourceAsMap = sourceAndContent.v2(); @@ -218,7 +218,7 @@ Result prepareUpdateIndexRequest(ShardId shardId, UpdateRequest request, GetResu return new Result(update, DocWriteResponse.Result.NOOP, updatedSourceAsMap, updateSourceContentType); } else { String index = request.index(); - final IndexRequest finalIndexRequest = new IndexRequest(index).id(request.id()) + IndexRequest finalIndexRequest = new IndexRequest(index).id(request.id()) .routing(routing) .source(updatedSourceAsMap, updateSourceContentType) .setIfSeqNo(getResult.getSeqNo()) @@ -227,6 +227,7 @@ Result prepareUpdateIndexRequest(ShardId shardId, UpdateRequest request, GetResu .timeout(request.timeout()) .setRefreshPolicy(request.getRefreshPolicy()) .setNormalisedBytesParsed(documentSizeObserver.normalisedBytesParsed()); + return new Result(finalIndexRequest, DocWriteResponse.Result.UPDATED, updatedSourceAsMap, updateSourceContentType); } } @@ -261,7 +262,7 @@ Result prepareUpdateScriptRequest(ShardId shardId, UpdateRequest request, GetRes switch (operation) { case INDEX -> { String index = request.index(); - final IndexRequest indexRequest = new IndexRequest(index).id(request.id()) + IndexRequest indexRequest = new IndexRequest(index).id(request.id()) .routing(routing) .source(updatedSourceAsMap, updateSourceContentType) .setIfSeqNo(getResult.getSeqNo()) @@ -269,7 +270,7 @@ Result prepareUpdateScriptRequest(ShardId shardId, UpdateRequest request, GetRes .waitForActiveShards(request.waitForActiveShards()) .timeout(request.timeout()) .setRefreshPolicy(request.getRefreshPolicy()) - .noParsedBytesToReport(); + .setOriginatesFromUpdateByScript(true); return new Result(indexRequest, DocWriteResponse.Result.UPDATED, updatedSourceAsMap, updateSourceContentType); } case DELETE -> { diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index 3fc659cb8065d..be1220da6b1c4 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -19,7 +19,6 @@ import org.elasticsearch.ReleaseVersions; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.ReferenceDocs; -import org.elasticsearch.common.filesystem.FileSystemNatives; import org.elasticsearch.common.io.stream.InputStreamStreamInput; import org.elasticsearch.common.logging.LogConfigurator; import org.elasticsearch.common.network.IfConfig; @@ -318,9 +317,6 @@ static void initializeNatives(final Path tmpFile, final boolean mlockAll, final // init lucene random seed. it will use /dev/urandom where available: StringHelper.randomId(); - - // init filesystem natives - FileSystemNatives.init(); } static void initializeProbes() { diff --git a/server/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java b/server/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java deleted file mode 100644 index 03b106407f45f..0000000000000 --- a/server/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.bootstrap; - -import com.sun.jna.Native; -import com.sun.jna.NativeLong; -import com.sun.jna.Structure; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.lucene.util.Constants; - -import java.util.Arrays; -import java.util.List; - -/** - * java mapping to some libc functions - */ -final class JNACLibrary { - - private static final Logger logger = LogManager.getLogger(JNACLibrary.class); - - public static final int MCL_CURRENT = 1; - public static final int ENOMEM = 12; - public static final int RLIMIT_MEMLOCK = Constants.MAC_OS_X ? 6 : 8; - public static final int RLIMIT_AS = Constants.MAC_OS_X ? 5 : 9; - public static final int RLIMIT_FSIZE = Constants.MAC_OS_X ? 1 : 1; - public static final long RLIM_INFINITY = Constants.MAC_OS_X ? 9223372036854775807L : -1L; - - static { - try { - Native.register("c"); - } catch (UnsatisfiedLinkError e) { - logger.warn("unable to link C library. native methods (mlockall) will be disabled.", e); - } - } - - static native int mlockall(int flags); - - /** corresponds to struct rlimit */ - public static final class Rlimit extends Structure implements Structure.ByReference { - public NativeLong rlim_cur = new NativeLong(0); - public NativeLong rlim_max = new NativeLong(0); - - @Override - protected List getFieldOrder() { - return Arrays.asList("rlim_cur", "rlim_max"); - } - } - - static native int getrlimit(int resource, Rlimit rlimit); - - static native int setrlimit(int resource, Rlimit rlimit); - - static native String strerror(int errno); - - private JNACLibrary() {} -} diff --git a/server/src/main/java/org/elasticsearch/bootstrap/PolicyUtil.java b/server/src/main/java/org/elasticsearch/bootstrap/PolicyUtil.java index b9574f1a29ae8..444ec49c70407 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/PolicyUtil.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/PolicyUtil.java @@ -294,8 +294,8 @@ public String getProperty(String key) { + " in policy file [" + policyFile + "]" - + "\nAvailable codebases: " - + codebaseProperties.keySet() + + "\nAvailable codebases: \n " + + String.join("\n ", codebaseProperties.keySet().stream().sorted().toList()) ); } return policy; diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index f9294210e0a6a..c54269da68507 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -884,6 +884,11 @@ public Map> nodeFeatures() { return Collections.unmodifiableMap(this.nodeFeatures); } + public Builder putNodeFeatures(String node, Set features) { + this.nodeFeatures.put(node, features); + return this; + } + public Builder routingTable(RoutingTable.Builder routingTableBuilder) { return routingTable(routingTableBuilder.build()); } diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java index 26c453d419f4c..f4440688ea307 100644 --- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java +++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java @@ -283,7 +283,7 @@ private void fetchNodeStats() { final NodesStatsRequest nodesStatsRequest = new NodesStatsRequest("data:true"); nodesStatsRequest.setIncludeShardsStats(false); nodesStatsRequest.clear(); - nodesStatsRequest.addMetric(NodesStatsRequestParameters.Metric.FS.metricName()); + nodesStatsRequest.addMetric(NodesStatsRequestParameters.Metric.FS); nodesStatsRequest.timeout(fetchTimeout); client.admin().cluster().nodesStats(nodesStatsRequest, ActionListener.releaseAfter(new ActionListener<>() { @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/RepositoryCleanupInProgress.java b/server/src/main/java/org/elasticsearch/cluster/RepositoryCleanupInProgress.java index 2dba73a3ec68f..cc5e71b38ecb2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/RepositoryCleanupInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/RepositoryCleanupInProgress.java @@ -21,6 +21,9 @@ import java.util.Iterator; import java.util.List; +/** + * A repository cleanup request entry. Part of the cluster state. + */ public final class RepositoryCleanupInProgress extends AbstractNamedDiffable implements ClusterState.Custom { public static final RepositoryCleanupInProgress EMPTY = new RepositoryCleanupInProgress(List.of()); diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java index eea89c6ff3714..914bf2d0cdb3e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java @@ -32,7 +32,7 @@ import java.util.Set; /** - * A class that represents the snapshot deletions that are in progress in the cluster. + * Represents the in-progress snapshot deletions in the cluster state. */ public class SnapshotDeletionsInProgress extends AbstractNamedDiffable implements Custom { diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index b6fb370991a93..7b0ab346501f3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -68,7 +68,7 @@ public class SnapshotsInProgress extends AbstractNamedDiffable implement public static final String ABORTED_FAILURE_TEXT = "Snapshot was aborted by deletion"; - // keyed by repository name + /** Maps repository name to list of snapshots in that repository */ private final Map entries; /** @@ -86,6 +86,9 @@ public class SnapshotsInProgress extends AbstractNamedDiffable implement // INIT state. private final Set nodesIdsForRemoval; + /** + * Returns the SnapshotInProgress metadata present within the given cluster state. + */ public static SnapshotsInProgress get(ClusterState state) { return state.custom(TYPE, EMPTY); } @@ -145,6 +148,9 @@ public SnapshotsInProgress withAddedEntry(Entry entry) { return withUpdatedEntriesForRepo(entry.repository(), forRepo); } + /** + * Returns the list of snapshots in the specified repository. + */ public List forRepo(String repository) { return entries.getOrDefault(repository, ByRepo.EMPTY).entries; } @@ -171,14 +177,18 @@ public Stream asStream() { @Nullable public Entry snapshot(final Snapshot snapshot) { - return findInList(snapshot, forRepo(snapshot.getRepository())); + return findSnapshotInList(snapshot, forRepo(snapshot.getRepository())); } + /** + * Searches for a particular {@code snapshotToFind} in the given snapshot list. + * @return a matching snapshot entry or null. + */ @Nullable - private static Entry findInList(Snapshot snapshot, List forRepo) { + private static Entry findSnapshotInList(Snapshot snapshotToFind, List forRepo) { for (Entry entry : forRepo) { - final Snapshot curr = entry.snapshot(); - if (curr.equals(snapshot)) { + final Snapshot snapshot = entry.snapshot(); + if (snapshot.equals(snapshotToFind)) { return entry; } } @@ -186,30 +196,41 @@ private static Entry findInList(Snapshot snapshot, List forRepo) { } /** - * Computes a map of repository shard id to set of generations, containing all shard generations that became obsolete and may be - * deleted from the repository as the cluster state moved from the given {@code old} value of {@link SnapshotsInProgress} to this - * instance. + * Computes a map of repository shard id to set of shard generations, containing all shard generations that became obsolete and may be + * deleted from the repository as the cluster state moves from the given old value of {@link SnapshotsInProgress} to this instance. + *

+ * An unique shard generation is created for every in-progress shard snapshot. The shard generation file contains information about all + * the files needed by pre-existing and any new shard snapshots that were in-progress. When a shard snapshot is finalized, its file list + * is promoted to the official shard snapshot list for the index shard. This final list will contain metadata about any other + * in-progress shard snapshots that were not yet finalized when it began. All these other in-progress shard snapshot lists are scheduled + * for deletion now. */ - public Map> obsoleteGenerations(String repository, SnapshotsInProgress old) { + public Map> obsoleteGenerations( + String repository, + SnapshotsInProgress oldClusterStateSnapshots + ) { final Map> obsoleteGenerations = new HashMap<>(); - final List updatedSnapshots = forRepo(repository); - for (Entry entry : old.forRepo(repository)) { - final Entry updatedEntry = findInList(entry.snapshot(), updatedSnapshots); - if (updatedEntry == null || updatedEntry == entry) { + final List latestSnapshots = forRepo(repository); + + for (Entry oldEntry : oldClusterStateSnapshots.forRepo(repository)) { + final Entry matchingLatestEntry = findSnapshotInList(oldEntry.snapshot(), latestSnapshots); + if (matchingLatestEntry == null || matchingLatestEntry == oldEntry) { + // The snapshot progress has not changed. continue; } - for (Map.Entry oldShardAssignment : entry.shardsByRepoShardId().entrySet()) { + for (Map.Entry oldShardAssignment : oldEntry.shardSnapshotStatusByRepoShardId() + .entrySet()) { final RepositoryShardId repositoryShardId = oldShardAssignment.getKey(); final ShardSnapshotStatus oldStatus = oldShardAssignment.getValue(); - final ShardSnapshotStatus newStatus = updatedEntry.shardsByRepoShardId().get(repositoryShardId); + final ShardSnapshotStatus newStatus = matchingLatestEntry.shardSnapshotStatusByRepoShardId().get(repositoryShardId); if (oldStatus.state == ShardState.SUCCESS && oldStatus.generation() != null && newStatus != null && newStatus.state() == ShardState.SUCCESS && newStatus.generation() != null && oldStatus.generation().equals(newStatus.generation()) == false) { - // We moved from a non-null generation successful generation to a different non-null successful generation - // so the original generation is clearly obsolete because it was in-flight before and is now unreferenced everywhere. + // We moved from a non-null successful generation to a different non-null successful generation + // so the original generation is obsolete because it was in-flight before and is now unreferenced. obsoleteGenerations.computeIfAbsent(repositoryShardId, ignored -> new HashSet<>()).add(oldStatus.generation()); logger.debug( """ @@ -218,7 +239,7 @@ public Map> obsoleteGenerations(String r """, oldStatus.generation(), newStatus.generation(), - entry.snapshot(), + oldEntry.snapshot(), repositoryShardId.shardId(), oldStatus.nodeId() ); @@ -399,7 +420,7 @@ private static boolean assertConsistentEntries(Map entries) { assert entriesForRepository.isEmpty() == false : "found empty list of snapshots for " + repository + " in " + entries; for (Entry entry : entriesForRepository) { assert entry.repository().equals(repository) : "mismatched repository " + entry + " tracked under " + repository; - for (Map.Entry shard : entry.shardsByRepoShardId().entrySet()) { + for (Map.Entry shard : entry.shardSnapshotStatusByRepoShardId().entrySet()) { final RepositoryShardId sid = shard.getKey(); final ShardSnapshotStatus shardSnapshotStatus = shard.getValue(); assert assertShardStateConsistent( @@ -520,11 +541,17 @@ public boolean nodeIdsForRemovalChanged(SnapshotsInProgress other) { return nodesIdsForRemoval.equals(other.nodesIdsForRemoval) == false; } + /** + * The current stage/phase of the shard snapshot, and whether it has completed or failed. + */ public enum ShardState { INIT((byte) 0, false, false), SUCCESS((byte) 2, true, false), FAILED((byte) 3, true, true), ABORTED((byte) 4, false, true), + /** + * Shard primary is unassigned and shard cannot be snapshotted. + */ MISSING((byte) 5, true, true), /** * Shard snapshot is waiting for the primary to snapshot to become available. @@ -611,6 +638,13 @@ public static State fromValue(byte value) { } } + /** + * @param nodeId node snapshotting the shard + * @param state the current phase of the snapshot + * @param generation shard generation ID identifying a particular snapshot of a shard + * @param reason what initiated the shard snapshot + * @param shardSnapshotResult only set if the snapshot has been successful, contains information for the shard finalization phase + */ public record ShardSnapshotStatus( @Nullable String nodeId, ShardState state, @@ -779,7 +813,7 @@ public static class Entry implements Writeable, ToXContentObject, RepositoryOper private final SnapshotId source; /** - * Map of {@link RepositoryShardId} to {@link ShardSnapshotStatus} tracking the state of each shard operation in this entry. + * Map of {@link RepositoryShardId} to {@link ShardSnapshotStatus} tracking the state of each shard operation in this snapshot. */ private final Map shardStatusByRepoShardId; @@ -1201,7 +1235,7 @@ public Entry withStartedShards(Map shards) { userMetadata, version ); - assert updated.state().completed() == false && completed(updated.shardsByRepoShardId().values()) == false + assert updated.state().completed() == false && completed(updated.shardSnapshotStatusByRepoShardId().values()) == false : "Only running snapshots allowed but saw [" + updated + "]"; return updated; } @@ -1215,7 +1249,10 @@ public Snapshot snapshot() { return this.snapshot; } - public Map shardsByRepoShardId() { + /** + * Returns a map of shards to their snapshot status. + */ + public Map shardSnapshotStatusByRepoShardId() { return shardStatusByRepoShardId; } diff --git a/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlockException.java b/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlockException.java index 6e4968c8f359a..c496ccccd9c10 100644 --- a/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlockException.java +++ b/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlockException.java @@ -38,6 +38,11 @@ public ClusterBlockException(StreamInput in) throws IOException { this.blocks = in.readCollectionAsImmutableSet(ClusterBlock::new); } + @Override + public Throwable fillInStackTrace() { + return this; // this exception doesn't imply a bug, no need for a stack trace + } + @Override protected void writeTo(StreamOutput out, Writer nestedExceptionsWriter) throws IOException { super.writeTo(out, nestedExceptionsWriter); diff --git a/server/src/main/java/org/elasticsearch/cluster/features/NodeFeaturesFixupListener.java b/server/src/main/java/org/elasticsearch/cluster/features/NodeFeaturesFixupListener.java new file mode 100644 index 0000000000000..c8b2555c0f15d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/features/NodeFeaturesFixupListener.java @@ -0,0 +1,217 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.cluster.features; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.admin.cluster.node.features.NodeFeatures; +import org.elasticsearch.action.admin.cluster.node.features.NodesFeaturesRequest; +import org.elasticsearch.action.admin.cluster.node.features.NodesFeaturesResponse; +import org.elasticsearch.action.admin.cluster.node.features.TransportNodesFeaturesAction; +import org.elasticsearch.client.internal.ClusterAdminClient; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterFeatures; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.ClusterStateTaskExecutor; +import org.elasticsearch.cluster.ClusterStateTaskListener; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.service.MasterServiceTaskQueue; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.UpdateForV9; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; +import org.elasticsearch.threadpool.Scheduler; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executor; +import java.util.stream.Collectors; + +@UpdateForV9 // this can be removed in v9 +public class NodeFeaturesFixupListener implements ClusterStateListener { + + private static final Logger logger = LogManager.getLogger(NodeFeaturesFixupListener.class); + + private static final TimeValue RETRY_TIME = TimeValue.timeValueSeconds(30); + + private final MasterServiceTaskQueue taskQueue; + private final ClusterAdminClient client; + private final Scheduler scheduler; + private final Executor executor; + private final Set pendingNodes = Collections.synchronizedSet(new HashSet<>()); + + public NodeFeaturesFixupListener(ClusterService service, ClusterAdminClient client, ThreadPool threadPool) { + // there tends to be a lot of state operations on an upgrade - this one is not time-critical, + // so use LOW priority. It just needs to be run at some point after upgrade. + this( + service.createTaskQueue("fix-node-features", Priority.LOW, new NodesFeaturesUpdater()), + client, + threadPool, + threadPool.executor(ThreadPool.Names.CLUSTER_COORDINATION) + ); + } + + NodeFeaturesFixupListener( + MasterServiceTaskQueue taskQueue, + ClusterAdminClient client, + Scheduler scheduler, + Executor executor + ) { + this.taskQueue = taskQueue; + this.client = client; + this.scheduler = scheduler; + this.executor = executor; + } + + class NodesFeaturesTask implements ClusterStateTaskListener { + private final Map> results; + private final int retryNum; + + NodesFeaturesTask(Map> results, int retryNum) { + this.results = results; + this.retryNum = retryNum; + } + + @Override + public void onFailure(Exception e) { + logger.error("Could not apply features for nodes {} to cluster state", results.keySet(), e); + scheduleRetry(results.keySet(), retryNum); + } + + public Map> results() { + return results; + } + } + + static class NodesFeaturesUpdater implements ClusterStateTaskExecutor { + @Override + public ClusterState execute(BatchExecutionContext context) { + ClusterState.Builder builder = ClusterState.builder(context.initialState()); + var existingFeatures = builder.nodeFeatures(); + + boolean modified = false; + for (var c : context.taskContexts()) { + for (var e : c.getTask().results().entrySet()) { + // double check there are still no features for the node + if (existingFeatures.getOrDefault(e.getKey(), Set.of()).isEmpty()) { + builder.putNodeFeatures(e.getKey(), e.getValue()); + modified = true; + } + } + c.success(() -> {}); + } + return modified ? builder.build() : context.initialState(); + } + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + if (event.nodesDelta().masterNodeChanged() && event.localNodeMaster()) { + /* + * Execute this if we have just become master. + * Check if there are any nodes that should have features in cluster state, but don't. + * This can happen if the master was upgraded from before 8.13, and one or more non-master nodes + * were already upgraded. They don't re-join the cluster with the new master, so never get their features + * (which the master now understands) added to cluster state. + * So we need to do a separate transport call to get the node features and add them to cluster state. + * We can't use features to determine when this should happen, as the features are incorrect. + * We also can't use transport version, as that is unreliable for upgrades + * from versions before 8.8 (see TransportVersionFixupListener). + * So the only thing we can use is release version. + * This is ok here, as Serverless will never hit this case, so the node feature fetch action will never be called on Serverless. + * This whole class will be removed in ES v9. + */ + ClusterFeatures nodeFeatures = event.state().clusterFeatures(); + Set queryNodes = event.state() + .nodes() + .stream() + .filter(n -> n.getVersion().onOrAfter(Version.V_8_15_0)) + .map(DiscoveryNode::getId) + .filter(n -> getNodeFeatures(nodeFeatures, n).isEmpty()) + .collect(Collectors.toSet()); + + if (queryNodes.isEmpty() == false) { + logger.debug("Fetching actual node features for nodes {}", queryNodes); + queryNodesFeatures(queryNodes, 0); + } + } + } + + @SuppressForbidden(reason = "Need to access a specific node's features") + private static Set getNodeFeatures(ClusterFeatures features, String nodeId) { + return features.nodeFeatures().getOrDefault(nodeId, Set.of()); + } + + private void scheduleRetry(Set nodes, int thisRetryNum) { + // just keep retrying until this succeeds + logger.debug("Scheduling retry {} for nodes {}", thisRetryNum + 1, nodes); + scheduler.schedule(() -> queryNodesFeatures(nodes, thisRetryNum + 1), RETRY_TIME, executor); + } + + private void queryNodesFeatures(Set nodes, int retryNum) { + // some might already be in-progress + Set outstandingNodes = Sets.newHashSetWithExpectedSize(nodes.size()); + synchronized (pendingNodes) { + for (String n : nodes) { + if (pendingNodes.add(n)) { + outstandingNodes.add(n); + } + } + } + if (outstandingNodes.isEmpty()) { + // all nodes already have in-progress requests + return; + } + + NodesFeaturesRequest request = new NodesFeaturesRequest(outstandingNodes.toArray(String[]::new)); + client.execute(TransportNodesFeaturesAction.TYPE, request, new ActionListener<>() { + @Override + public void onResponse(NodesFeaturesResponse response) { + pendingNodes.removeAll(outstandingNodes); + handleResponse(response, retryNum); + } + + @Override + public void onFailure(Exception e) { + pendingNodes.removeAll(outstandingNodes); + logger.warn("Could not read features for nodes {}", outstandingNodes, e); + scheduleRetry(outstandingNodes, retryNum); + } + }); + } + + private void handleResponse(NodesFeaturesResponse response, int retryNum) { + if (response.hasFailures()) { + Set failedNodes = new HashSet<>(); + for (FailedNodeException fne : response.failures()) { + logger.warn("Failed to read features from node {}", fne.nodeId(), fne); + failedNodes.add(fne.nodeId()); + } + scheduleRetry(failedNodes, retryNum); + } + // carry on and read what we can + + Map> results = response.getNodes() + .stream() + .collect(Collectors.toUnmodifiableMap(n -> n.getNode().getId(), NodeFeatures::nodeFeatures)); + + if (results.isEmpty() == false) { + taskQueue.submitTask("fix-node-features", new NodesFeaturesTask(results, retryNum), null); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 03b23c462ecec..6b20399a1bc59 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -69,7 +69,7 @@ public final class DataStream implements SimpleDiffable, ToXContentO public static final FeatureFlag FAILURE_STORE_FEATURE_FLAG = new FeatureFlag("failure_store"); public static final TransportVersion ADDED_FAILURE_STORE_TRANSPORT_VERSION = TransportVersions.V_8_12_0; - public static final TransportVersion ADDED_AUTO_SHARDING_EVENT_VERSION = TransportVersions.DATA_STREAM_AUTO_SHARDING_EVENT; + public static final TransportVersion ADDED_AUTO_SHARDING_EVENT_VERSION = TransportVersions.V_8_14_0; public static boolean isFailureStoreFeatureFlagEnabled() { return FAILURE_STORE_FEATURE_FLAG.isEnabled(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAction.java index f260b48cd7b7a..32bf46ce45919 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAction.java @@ -88,7 +88,7 @@ public DataStreamAction(StreamInput in) throws IOException { this.type = Type.fromValue(in.readByte()); this.dataStream = in.readString(); this.index = in.readString(); - this.failureStore = in.getTransportVersion().onOrAfter(TransportVersions.MODIFY_DATA_STREAM_FAILURE_STORES) && in.readBoolean(); + this.failureStore = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) && in.readBoolean(); } private DataStreamAction(Type type, String dataStream, String index, boolean failureStore) { @@ -155,7 +155,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeByte(type.value()); out.writeString(dataStream); out.writeString(index); - if (out.getTransportVersion().onOrAfter(TransportVersions.MODIFY_DATA_STREAM_FAILURE_STORES)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeBoolean(failureStore); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java index f691151eee95e..1e5847ac0a639 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java @@ -86,7 +86,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ADD_DATA_STREAM_GLOBAL_RETENTION; + return TransportVersions.V_8_14_0; } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 2b65a68e8d43c..32a3af0c341e5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -1623,7 +1623,7 @@ private static class IndexMetadataDiff implements Diff { } primaryTerms = in.readVLongArray(); mappings = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), MAPPING_DIFF_VALUE_READER); - if (in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_FIELDS_METADATA)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { inferenceFields = DiffableUtils.readImmutableOpenMapDiff( in, DiffableUtils.getStringKeySerializer(), @@ -1691,7 +1691,7 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeVLongArray(primaryTerms); mappings.writeTo(out); - if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_FIELDS_METADATA)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { inferenceFields.writeTo(out); } aliases.writeTo(out); @@ -1784,7 +1784,7 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function builder.putInferenceField(f)); } @@ -1856,7 +1856,7 @@ public void writeTo(StreamOutput out, boolean mappingsAsHash) throws IOException mapping.writeTo(out); } } - if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_FIELDS_METADATA)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeCollection(inferenceFields.values()); } out.writeCollection(aliases.values()); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java index 523dc0efd450b..8abb1c76da142 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java @@ -240,7 +240,7 @@ public boolean started() { } /** - * Returns true iff the this shard is currently relocating to + * Returns true iff this shard is currently relocating to * another node. Otherwise false * * @see ShardRoutingState#RELOCATING diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/ShardChangesObserver.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/ShardChangesObserver.java index f265ab7f62db2..e051f7b88edd3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/ShardChangesObserver.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/ShardChangesObserver.java @@ -18,9 +18,24 @@ public class ShardChangesObserver implements RoutingChangesObserver { private static final Logger logger = LogManager.getLogger(ShardChangesObserver.class); + @Override + public void shardInitialized(ShardRouting unassignedShard, ShardRouting initializedShard) { + logger.trace( + "{} initializing from {} on node [{}]", + shardIdentifier(initializedShard), + initializedShard.recoverySource().getType(), + initializedShard.currentNodeId() + ); + } + @Override public void shardStarted(ShardRouting initializingShard, ShardRouting startedShard) { - logger.debug("{} started on node [{}]", shardIdentifier(startedShard), startedShard.currentNodeId()); + logger.debug( + "{} started from {} on node [{}]", + shardIdentifier(startedShard), + initializingShard.recoverySource().getType(), + startedShard.currentNodeId() + ); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java index 88d4a652a5a39..7289b218b6be4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; @@ -44,7 +45,9 @@ public class ClusterRebalanceAllocationDecider extends AllocationDecider { private static final String CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE = "cluster.routing.allocation.allow_rebalance"; public static final Setting CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING = new Setting<>( CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, - ClusterRebalanceType.INDICES_ALL_ACTIVE.toString(), + settings -> ClusterModule.DESIRED_BALANCE_ALLOCATOR.equals(ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING.get(settings)) + ? ClusterRebalanceType.ALWAYS.toString() + : ClusterRebalanceType.INDICES_ALL_ACTIVE.toString(), ClusterRebalanceType::parseString, Property.Dynamic, Property.NodeScope diff --git a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java index 770ed4d213c55..a87f3b3d4bda0 100644 --- a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java +++ b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java @@ -77,6 +77,8 @@ public enum ReferenceDocs { NETWORK_BINDING_AND_PUBLISHING, SNAPSHOT_REPOSITORY_ANALYSIS, S3_COMPATIBLE_REPOSITORIES, + LUCENE_MAX_DOCS_LIMIT, + MAX_SHARDS_PER_NODE, // this comment keeps the ';' on the next line so every entry above has a trailing ',' which makes the diff for adding new links cleaner ; diff --git a/server/src/main/java/org/elasticsearch/common/collect/Iterators.java b/server/src/main/java/org/elasticsearch/common/collect/Iterators.java index 165280e370025..d029f8e3becc0 100644 --- a/server/src/main/java/org/elasticsearch/common/collect/Iterators.java +++ b/server/src/main/java/org/elasticsearch/common/collect/Iterators.java @@ -21,6 +21,7 @@ import java.util.function.Consumer; import java.util.function.Function; import java.util.function.IntFunction; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.function.ToIntFunction; @@ -179,6 +180,59 @@ public void forEachRemaining(Consumer action) { } } + /** + * @param input An iterator over non-null values. + * @param predicate The predicate with which to filter the input. + * @return an iterator which returns the values from {@code input} which match {@code predicate}. + */ + public static Iterator filter(Iterator input, Predicate predicate) { + while (input.hasNext()) { + final var value = input.next(); + assert value != null; + if (predicate.test(value)) { + return new FilterIterator<>(value, input, predicate); + } + } + return Collections.emptyIterator(); + } + + private static final class FilterIterator implements Iterator { + private final Iterator input; + private final Predicate predicate; + private T next; + + FilterIterator(T value, Iterator input, Predicate predicate) { + this.next = value; + this.input = input; + this.predicate = predicate; + assert next != null; + assert predicate.test(next); + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public T next() { + if (hasNext() == false) { + throw new NoSuchElementException(); + } + final var value = next; + while (input.hasNext()) { + final var laterValue = input.next(); + assert laterValue != null; + if (predicate.test(laterValue)) { + next = laterValue; + return value; + } + } + next = null; + return value; + } + } + public static Iterator flatMap(Iterator input, Function> fn) { while (input.hasNext()) { final var value = fn.apply(input.next()); diff --git a/server/src/main/java/org/elasticsearch/common/filesystem/FileSystemNatives.java b/server/src/main/java/org/elasticsearch/common/filesystem/FileSystemNatives.java deleted file mode 100644 index 00502d64b3896..0000000000000 --- a/server/src/main/java/org/elasticsearch/common/filesystem/FileSystemNatives.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.common.filesystem; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.lucene.util.Constants; - -import java.nio.file.Path; -import java.util.OptionalLong; - -/** - * This class provides utility methods for calling some native methods related to filesystems. - */ -public final class FileSystemNatives { - - private static final Logger logger = LogManager.getLogger(FileSystemNatives.class); - - @FunctionalInterface - interface Provider { - OptionalLong allocatedSizeInBytes(Path path); - } - - private static final Provider NOOP_FILE_SYSTEM_NATIVES_PROVIDER = path -> OptionalLong.empty(); - private static final Provider JNA_PROVIDER = loadJnaProvider(); - - private static Provider loadJnaProvider() { - try { - // load one of the main JNA classes to see if the classes are available. this does not ensure that all native - // libraries are available, only the ones necessary by JNA to function - Class.forName("com.sun.jna.Native"); - if (Constants.WINDOWS) { - return WindowsFileSystemNatives.getInstance(); - } else if (Constants.LINUX && Constants.JRE_IS_64BIT) { - return LinuxFileSystemNatives.getInstance(); - } - } catch (ClassNotFoundException e) { - logger.warn("JNA not found. FileSystemNatives methods will be disabled.", e); - } catch (LinkageError e) { - logger.warn("unable to load JNA native support library, FileSystemNatives methods will be disabled.", e); - } - return NOOP_FILE_SYSTEM_NATIVES_PROVIDER; - } - - private FileSystemNatives() {} - - public static void init() { - assert JNA_PROVIDER != null; - } - - /** - * Returns the number of allocated bytes on disk for a given file. - * - * @param path the path to the file - * @return an {@link OptionalLong} that contains the number of allocated bytes on disk for the file. The optional is empty is the - * allocated size of the file failed be retrieved using native methods - */ - public static OptionalLong allocatedSizeInBytes(Path path) { - return JNA_PROVIDER.allocatedSizeInBytes(path); - } - -} diff --git a/server/src/main/java/org/elasticsearch/common/filesystem/LinuxFileSystemNatives.java b/server/src/main/java/org/elasticsearch/common/filesystem/LinuxFileSystemNatives.java deleted file mode 100644 index b40fb5c2e145b..0000000000000 --- a/server/src/main/java/org/elasticsearch/common/filesystem/LinuxFileSystemNatives.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.common.filesystem; - -import com.sun.jna.LastErrorException; -import com.sun.jna.Native; -import com.sun.jna.Platform; -import com.sun.jna.Structure; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.lucene.util.Constants; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Instant; -import java.util.OptionalLong; - -import static org.elasticsearch.core.Strings.format; - -/** - * {@link FileSystemNatives.Provider} implementation for Linux x86-64bits - */ -final class LinuxFileSystemNatives implements FileSystemNatives.Provider { - - private static final Logger logger = LogManager.getLogger(LinuxFileSystemNatives.class); - - private static final LinuxFileSystemNatives INSTANCE = new LinuxFileSystemNatives(); - - /** st_blocks field indicates the number of blocks allocated to the file, 512-byte units **/ - private static final long ST_BLOCKS_UNIT = 512L; - - /** - * Version of the `struct stat' data structure. - * - * To allow the `struct stat' structure bits to vary without changing shared library major version number, the `stat' function is often - * an inline wrapper around `xstat' which takes a leading version-number argument designating the data structure and bits used. - * - * In glibc this version is defined in bits/stat.h (or bits/struct_stat.h in glibc 2.33, or bits/xstatver.h in more recent versions). - * - * For x86-64 the _STAT_VER used is: - * # define _STAT_VER_LINUX 1 - * # define _STAT_VER _STAT_VER_LINUX - * - * For other architectures the _STAT_VER used is: - * # define _STAT_VER_LINUX 0 - * # define _STAT_VER _STAT_VER_LINUX - **/ - private static int loadStatVersion() { - return "aarch64".equalsIgnoreCase(Constants.OS_ARCH) ? 0 : 1; - } - - private static final int STAT_VER = loadStatVersion(); - - private LinuxFileSystemNatives() { - assert Constants.LINUX : Constants.OS_NAME; - assert Constants.JRE_IS_64BIT : Constants.OS_ARCH; - try { - Native.register(XStatLibrary.class, Platform.C_LIBRARY_NAME); - logger.debug("C library loaded"); - } catch (LinkageError e) { - logger.warn("unable to link C library. native methods and handlers will be disabled.", e); - throw e; - } - } - - static LinuxFileSystemNatives getInstance() { - return INSTANCE; - } - - public static class XStatLibrary { - public static native int __xstat(int version, String path, Stat stats) throws LastErrorException; - } - - /** - * Retrieves the actual number of bytes of disk storage used to store a specified file. - * - * @param path the path to the file - * @return an {@link OptionalLong} that contains the number of allocated bytes on disk for the file, or empty if the size is invalid - */ - @Override - public OptionalLong allocatedSizeInBytes(Path path) { - assert Files.isRegularFile(path) : path; - try { - final Stat stats = new Stat(); - final int rc = XStatLibrary.__xstat(STAT_VER, path.toString(), stats); - if (logger.isTraceEnabled()) { - logger.trace("executing native method __xstat() returned {} with error code [{}] for file [{}]", stats, rc, path); - } - return OptionalLong.of(stats.st_blocks * ST_BLOCKS_UNIT); - } catch (LastErrorException e) { - logger.warn( - () -> format( - "error when executing native method __xstat(int vers, const char *name, struct stat *buf) for file [%s]", - path - ), - e - ); - } - return OptionalLong.empty(); - } - - @Structure.FieldOrder( - { - "st_dev", - "st_ino", - "st_nlink", - "st_mode", - "st_uid", - "st_gid", - "__pad0", - "st_rdev", - "st_size", - "st_blksize", - "st_blocks", - "st_atim", - "st_mtim", - "st_ctim", - "__glibc_reserved0", - "__glibc_reserved1", - "__glibc_reserved2" } - ) - public static class Stat extends Structure { - - /** - * The stat structure varies across architectures in the glibc and kernel source codes. For example some fields might be ordered - * differently and/or some padding bytes might be present between some fields. - * - * The struct implemented here refers to the Linux x86 architecture in the glibc source files: - * - glibc version 2.23: sysdeps/unix/sysv/linux/x86/bits/stat.h - * - glibc version 2.33: sysdeps/unix/sysv/linux/x86/bits/struct_stat.h - * - * The following command is useful to compile the stat struct on a given system: - * echo "#include <sys/stat.h>" | gcc -xc - -E -dD | grep -ve '^$' | grep -A23 '^struct stat' - */ - public long st_dev; // __dev_t st_dev; /* Device. */ - public long st_ino; // __ino_t st_ino; /* File serial number. */ - public long st_nlink; // __nlink_t st_nlink; /* Link count. */ - public int st_mode; // __mode_t st_mode; /* File mode. */ - public int st_uid; // __uid_t st_uid; /* User ID of the file's owner. */ - public int st_gid; // __gid_t st_gid; /* Group ID of the file's group. */ - public int __pad0; - public long st_rdev; // __dev_t st_rdev; /* Device number, if device. */ - public long st_size; // __off_t st_size; /* Size of file, in bytes. */ - public long st_blksize; // __blksize_t st_blksize; /* Optimal block size for I/O. */ - public long st_blocks; // __blkcnt_t st_blocks; /* Number 512-byte blocks allocated. */ - public Time st_atim; // struct timespec st_atim; /* Time of last access. */ - public Time st_mtim; // struct timespec st_mtim; /* Time of last modification. */ - public Time st_ctim; // struct timespec st_ctim; /* Time of last status change. */ - public long __glibc_reserved0; // __syscall_slong_t - public long __glibc_reserved1; // __syscall_slong_t - public long __glibc_reserved2; // __syscall_slong_t - - @Override - public String toString() { - return "[st_dev=" - + st_dev - + ", st_ino=" - + st_ino - + ", st_nlink=" - + st_nlink - + ", st_mode=" - + st_mode - + ", st_uid=" - + st_uid - + ", st_gid=" - + st_gid - + ", st_rdev=" - + st_rdev - + ", st_size=" - + st_size - + ", st_blksize=" - + st_blksize - + ", st_blocks=" - + st_blocks - + ", st_atim=" - + Instant.ofEpochSecond(st_atim.tv_sec, st_atim.tv_nsec) - + ", st_mtim=" - + Instant.ofEpochSecond(st_mtim.tv_sec, st_mtim.tv_nsec) - + ", st_ctim=" - + Instant.ofEpochSecond(st_ctim.tv_sec, st_ctim.tv_nsec) - + ']'; - } - } - - @Structure.FieldOrder({ "tv_sec", "tv_nsec" }) - public static class Time extends Structure { - public long tv_sec; - public long tv_nsec; - } -} diff --git a/server/src/main/java/org/elasticsearch/common/filesystem/WindowsFileSystemNatives.java b/server/src/main/java/org/elasticsearch/common/filesystem/WindowsFileSystemNatives.java deleted file mode 100644 index 4fe219bfc774d..0000000000000 --- a/server/src/main/java/org/elasticsearch/common/filesystem/WindowsFileSystemNatives.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.common.filesystem; - -import com.sun.jna.Native; -import com.sun.jna.WString; -import com.sun.jna.ptr.IntByReference; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.lucene.util.Constants; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.OptionalLong; - -/** - * {@link FileSystemNatives.Provider} implementation for Windows/Kernel32 - */ -final class WindowsFileSystemNatives implements FileSystemNatives.Provider { - - private static final Logger logger = LogManager.getLogger(WindowsFileSystemNatives.class); - - private static final WindowsFileSystemNatives INSTANCE = new WindowsFileSystemNatives(); - - private static final int INVALID_FILE_SIZE = -1; - private static final int NO_ERROR = 0; - - private WindowsFileSystemNatives() { - assert Constants.WINDOWS : Constants.OS_NAME; - try { - Native.register("kernel32"); - logger.debug("windows/Kernel32 library loaded"); - } catch (LinkageError e) { - logger.warn("unable to link Windows/Kernel32 library. native methods and handlers will be disabled.", e); - throw e; - } - } - - static WindowsFileSystemNatives getInstance() { - return INSTANCE; - } - - /** - * Retrieves the actual number of bytes of disk storage used to store a specified file. - * - * https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getcompressedfilesizew - * - * @param lpFileName the path string - * @param lpFileSizeHigh pointer to high-order DWORD for compressed file size (or null if not needed) - * @return the low-order DWORD for compressed file siz - */ - private native int GetCompressedFileSizeW(WString lpFileName, IntByReference lpFileSizeHigh); - - /** - * Retrieves the actual number of bytes of disk storage used to store a specified file. If the file is located on a volume that supports - * compression and the file is compressed, the value obtained is the compressed size of the specified file. If the file is located on a - * volume that supports sparse files and the file is a sparse file, the value obtained is the sparse size of the specified file. - * - * This method uses Win32 DLL native method {@link #GetCompressedFileSizeW(WString, IntByReference)}. - * - * @param path the path to the file - * @return an {@link OptionalLong} that contains the number of allocated bytes on disk for the file, or empty if the size is invalid - */ - public OptionalLong allocatedSizeInBytes(Path path) { - assert Files.isRegularFile(path) : path; - final WString fileName = new WString("\\\\?\\" + path); - final IntByReference lpFileSizeHigh = new IntByReference(); - - final int lpFileSizeLow = GetCompressedFileSizeW(fileName, lpFileSizeHigh); - if (lpFileSizeLow == INVALID_FILE_SIZE) { - final int err = Native.getLastError(); - if (err != NO_ERROR) { - logger.warn("error [{}] when executing native method GetCompressedFileSizeW for file [{}]", err, path); - return OptionalLong.empty(); - } - } - - // convert lpFileSizeLow to unsigned long and combine with signed/shifted lpFileSizeHigh - final long allocatedSize = (((long) lpFileSizeHigh.getValue()) << Integer.SIZE) | Integer.toUnsignedLong(lpFileSizeLow); - if (logger.isTraceEnabled()) { - logger.trace( - "executing native method GetCompressedFileSizeW returned [high={}, low={}, allocated={}] for file [{}]", - lpFileSizeHigh, - lpFileSizeLow, - allocatedSize, - path - ); - } - return OptionalLong.of(allocatedSize); - } -} diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/CountingFilterInputStream.java b/server/src/main/java/org/elasticsearch/common/io/stream/CountingFilterInputStream.java new file mode 100644 index 0000000000000..2f5fd01e77dab --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/io/stream/CountingFilterInputStream.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.io.stream; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +public class CountingFilterInputStream extends FilterInputStream { + + private int bytesRead = 0; + + public CountingFilterInputStream(InputStream in) { + super(in); + } + + @Override + public int read() throws IOException { + assert assertInvariant(); + final int result = super.read(); + if (result != -1) { + bytesRead += 1; + } + return result; + } + + // Not overriding read(byte[]) because FilterInputStream delegates to read(byte[], int, int) + + @Override + public int read(byte[] b, int off, int len) throws IOException { + assert assertInvariant(); + final int n = super.read(b, off, len); + if (n != -1) { + bytesRead += n; + } + return n; + } + + @Override + public long skip(long n) throws IOException { + assert assertInvariant(); + final long skipped = super.skip(n); + bytesRead += Math.toIntExact(skipped); + return skipped; + } + + public int getBytesRead() { + return bytesRead; + } + + protected boolean assertInvariant() { + return true; + } +} diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/RecyclerBytesStreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/RecyclerBytesStreamOutput.java index c4857a8b85ea3..b83ebc6a8c64f 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/RecyclerBytesStreamOutput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/RecyclerBytesStreamOutput.java @@ -56,10 +56,14 @@ public long position() { @Override public void writeByte(byte b) { - ensureCapacity(1); + int currentPageOffset = this.currentPageOffset; + if (1 > (pageSize - currentPageOffset)) { + ensureCapacity(1); + currentPageOffset = 0; + } BytesRef currentPage = pages.get(pageIndex).v(); currentPage.bytes[currentPage.offset + currentPageOffset] = b; - currentPageOffset++; + this.currentPageOffset = currentPageOffset + 1; } @Override @@ -72,7 +76,12 @@ public void writeBytes(byte[] b, int offset, int length) { Objects.checkFromIndexSize(offset, length, b.length); // get enough pages for new size - ensureCapacity(length); + final int pageSize = this.pageSize; + int currentPageOffset = this.currentPageOffset; + if (length > pageSize - currentPageOffset) { + ensureCapacity(length); + currentPageOffset = this.currentPageOffset; + } // bulk copy int bytesToCopy = length; @@ -92,6 +101,7 @@ public void writeBytes(byte[] b, int offset, int length) { } j++; } + this.currentPageOffset = currentPageOffset; // advance pageIndex += j; @@ -99,12 +109,13 @@ public void writeBytes(byte[] b, int offset, int length) { @Override public void writeInt(int i) throws IOException { + final int currentPageOffset = this.currentPageOffset; if (4 > (pageSize - currentPageOffset)) { super.writeInt(i); } else { BytesRef currentPage = pages.get(pageIndex).v(); VH_BE_INT.set(currentPage.bytes, currentPage.offset + currentPageOffset, i); - currentPageOffset += 4; + this.currentPageOffset = currentPageOffset + 4; } } @@ -121,12 +132,13 @@ public void writeIntLE(int i) throws IOException { @Override public void writeLong(long i) throws IOException { + final int currentPageOffset = this.currentPageOffset; if (8 > (pageSize - currentPageOffset)) { super.writeLong(i); } else { BytesRef currentPage = pages.get(pageIndex).v(); VH_BE_LONG.set(currentPage.bytes, currentPage.offset + currentPageOffset, i); - currentPageOffset += 8; + this.currentPageOffset = currentPageOffset + 8; } } @@ -242,9 +254,8 @@ public BytesReference bytes() { } private void ensureCapacity(int bytesNeeded) { - if (bytesNeeded > pageSize - currentPageOffset) { - ensureCapacityFromPosition(position() + bytesNeeded); - } + assert bytesNeeded > pageSize - currentPageOffset; + ensureCapacityFromPosition(position() + bytesNeeded); } private void ensureCapacityFromPosition(long newPosition) { diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index 86560b8a58963..60322ea89cbe8 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -1295,14 +1295,16 @@ private > C readCollection(Writeable.Reader> E readEnum(Class enumClass) throws IOException { return readEnum(enumClass, enumClass.getEnumConstants()); } /** - * Reads an optional enum with type E that was serialized based on the value of its ordinal + * Reads an optional enum with type {@code E} that was serialized based on the value of its ordinal. Enums serialized like this must + * have a corresponding test which uses {@code EnumSerializationTestUtils#assertEnumSerialization} to fix the wire protocol. */ @Nullable public > E readOptionalEnum(Class enumClass) throws IOException { @@ -1322,7 +1324,8 @@ private > E readEnum(Class enumClass, E[] values) throws IO } /** - * Reads an enum with type E that was serialized based on the value of it's ordinal + * Reads a set of enums with type {@code E} that were serialized based on the value of their ordinals. Enums serialized like this must + * have a corresponding test which uses {@code EnumSerializationTestUtils#assertEnumSerialization} to fix the wire protocol. */ public > EnumSet readEnumSet(Class enumClass) throws IOException { int size = readVInt(); diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index c245498333c94..f17e3ee8018a2 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -1135,7 +1135,8 @@ public void writeNamedWriteableCollection(Collection l } /** - * Writes an enum with type E based on its ordinal value + * Writes an enum with type {@code E} in terms of the value of its ordinal. Enums serialized like this must have a corresponding test + * which uses {@code EnumSerializationTestUtils#assertEnumSerialization} to fix the wire protocol. */ public > void writeEnum(E enumValue) throws IOException { assert enumValue instanceof XContentType == false : "XContentHelper#writeTo should be used for XContentType serialisation"; @@ -1143,7 +1144,8 @@ public > void writeEnum(E enumValue) throws IOException { } /** - * Writes an optional enum with type E based on its ordinal value + * Writes an optional enum with type {@code E} in terms of the value of its ordinal. Enums serialized like this must have a + * corresponding test which uses {@code EnumSerializationTestUtils#assertEnumSerialization} to fix the wire protocol. */ public > void writeOptionalEnum(@Nullable E enumValue) throws IOException { if (enumValue == null) { @@ -1156,7 +1158,8 @@ public > void writeOptionalEnum(@Nullable E enumValue) throws } /** - * Writes an EnumSet with type E that by serialized it based on it's ordinal value + * Writes a set of enum with type {@code E} in terms of the value of its ordinal. Enums serialized like this must have a corresponding + * test which uses {@code EnumSerializationTestUtils#assertEnumSerialization} to fix the wire protocol. */ public > void writeEnumSet(EnumSet enumSet) throws IOException { writeVInt(enumSet.size()); diff --git a/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java b/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java index a1ccfd30cbac3..79ee3c109c52d 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java +++ b/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java @@ -128,6 +128,21 @@ public static void configure(final Environment environment, boolean useConsole) configureESLogging(); configure(environment.settings(), environment.configFile(), environment.logsFile(), useConsole); initializeStatics(); + // creates a permanent status logger that can watch for StatusLogger events and forward to a real logger + configureStatusLoggerForwarder(); + } + + private static void configureStatusLoggerForwarder() { + // the real logger is lazily retrieved here since logging won't yet be setup during clinit of this class + var logger = LogManager.getLogger("StatusLogger"); + var listener = new StatusConsoleListener(Level.WARN) { + @Override + public void log(StatusData data) { + logger.log(data.getLevel(), data.getMessage(), data.getThrowable()); + super.log(data); + } + }; + StatusLogger.getLogger().registerListener(listener); } public static void configureESLogging() { diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java b/server/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java index 1765e598c30a2..f8d0c81466dcc 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java @@ -23,6 +23,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute; import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.PostingsEnum; @@ -592,6 +593,7 @@ private void addTermFrequencies(Reader r, Map termFreqMap, String f int tokenCount = 0; // for every token CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); + TermFrequencyAttribute tfAtt = ts.addAttribute(TermFrequencyAttribute.class); ts.reset(); while (ts.incrementToken()) { String word = termAtt.toString(); @@ -612,9 +614,9 @@ private void addTermFrequencies(Reader r, Map termFreqMap, String f // increment frequency Int cnt = termFreqMap.get(word); if (cnt == null) { - termFreqMap.put(word, new Int()); + termFreqMap.put(word, new Int(tfAtt.getTermFrequency())); } else { - cnt.x++; + cnt.x += tfAtt.getTermFrequency(); } } ts.end(); @@ -681,10 +683,14 @@ void update(String word, String topField, float score) { * Use for frequencies and to avoid renewing Integers. */ private static class Int { - int x; + private int x; - Int() { - x = 1; + private Int(int initialValue) { + x = initialValue; + } + + private Int() { + this(1); } } } diff --git a/server/src/main/java/org/elasticsearch/common/regex/Regex.java b/server/src/main/java/org/elasticsearch/common/regex/Regex.java index 983144c7cee89..f292ac5032f7e 100644 --- a/server/src/main/java/org/elasticsearch/common/regex/Regex.java +++ b/server/src/main/java/org/elasticsearch/common/regex/Regex.java @@ -44,10 +44,20 @@ public static boolean isMatchAllPattern(String str) { return str.equals("*"); } + /** + * Returns true if the str ends with "*". + */ public static boolean isSuffixMatchPattern(String str) { return str.length() > 1 && str.indexOf('*') == str.length() - 1; } + /** + * Returns true if the str ends with ".*". + */ + public static boolean isSuffixWildcard(String str) { + return isSuffixMatchPattern(str) && str.endsWith(".*"); + } + /** Return an {@link Automaton} that matches the given pattern. */ public static Automaton simpleMatchToAutomaton(String pattern) { List automata = new ArrayList<>(); @@ -71,9 +81,12 @@ public static Automaton simpleMatchToAutomaton(String... patterns) { List simpleStrings = new ArrayList<>(); List automata = new ArrayList<>(); + List prefixes = new ArrayList<>(); for (String pattern : patterns) { // Strings longer than 1000 characters aren't supported by makeStringUnion - if (isSimpleMatchPattern(pattern) || pattern.length() >= 1000) { + if (isSuffixWildcard(pattern) && pattern.length() < 1000) { + prefixes.add(new BytesRef(pattern.substring(0, pattern.length() - 1))); + } else if (isSimpleMatchPattern(pattern) || pattern.length() >= 1000) { automata.add(simpleMatchToAutomaton(pattern)); } else { simpleStrings.add(new BytesRef(pattern)); @@ -87,11 +100,18 @@ public static Automaton simpleMatchToAutomaton(String... patterns) { } else { simpleStringsAutomaton = Automata.makeString(simpleStrings.get(0).utf8ToString()); } - if (automata.isEmpty()) { + if (automata.isEmpty() && prefixes.isEmpty()) { return simpleStringsAutomaton; } automata.add(simpleStringsAutomaton); } + if (false == prefixes.isEmpty()) { + List prefixAutomaton = new ArrayList<>(); + Collections.sort(prefixes); + prefixAutomaton.add(Automata.makeStringUnion(prefixes)); + prefixAutomaton.add(Automata.makeAnyString()); + automata.add(Operations.concatenate(prefixAutomaton)); + } return Operations.union(automata); } diff --git a/server/src/main/java/org/elasticsearch/common/time/DateFormatter.java b/server/src/main/java/org/elasticsearch/common/time/DateFormatter.java index 41f44dfbdedbc..45550c13174ce 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateFormatter.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateFormatter.java @@ -71,6 +71,14 @@ default String formatMillis(long millis) { return format(Instant.ofEpochMilli(millis).atZone(zone)); } + /** + * Return the given nanoseconds-since-epoch formatted with this format. + */ + default String formatNanos(long nanos) { + ZoneId zone = zone() != null ? zone() : ZoneOffset.UTC; + return format(Instant.ofEpochMilli(nanos / 1_000_000).plusNanos(nanos % 1_000_000).atZone(zone)); + } + /** * A name based format for this formatter. Can be one of the registered formatters like epoch_millis or * a configured format like HH:mm:ss diff --git a/server/src/main/java/org/elasticsearch/common/util/BitArray.java b/server/src/main/java/org/elasticsearch/common/util/BitArray.java index 041111840056d..c2c0829530ec5 100644 --- a/server/src/main/java/org/elasticsearch/common/util/BitArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/BitArray.java @@ -181,8 +181,9 @@ public void fill(long fromIndex, long toIndex, boolean value) { // There's no need to grow the array just to clear bits. toIndex = Math.min(toIndex, currentSize); } - if (fromIndex == toIndex) { - return; // Empty range + if (fromIndex >= toIndex) { + // Empty range or false values after the end of the array. + return; } if (toIndex > currentSize) { diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsRejectedExecutionException.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsRejectedExecutionException.java index 6a67f41ab8004..3a21ea486ce39 100644 --- a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsRejectedExecutionException.java +++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsRejectedExecutionException.java @@ -27,6 +27,11 @@ public EsRejectedExecutionException() { this(null, false); } + @Override + public Throwable fillInStackTrace() { + return this; // this exception doesn't imply a bug, no need for a stack trace + } + /** * Checks if the thread pool that rejected the execution was terminated * shortly after the rejection. Its possible that this returns false and the diff --git a/server/src/main/java/org/elasticsearch/discovery/MasterNotDiscoveredException.java b/server/src/main/java/org/elasticsearch/discovery/MasterNotDiscoveredException.java index 0bd62627cb061..0f982e0e0ec65 100644 --- a/server/src/main/java/org/elasticsearch/discovery/MasterNotDiscoveredException.java +++ b/server/src/main/java/org/elasticsearch/discovery/MasterNotDiscoveredException.java @@ -32,4 +32,9 @@ public RestStatus status() { public MasterNotDiscoveredException(StreamInput in) throws IOException { super(in); } + + @Override + public Throwable fillInStackTrace() { + return this; // this exception doesn't imply a bug, no need for a stack trace + } } diff --git a/server/src/main/java/org/elasticsearch/health/GetHealthAction.java b/server/src/main/java/org/elasticsearch/health/GetHealthAction.java index eeff060c174da..5a5f8f6d93c47 100644 --- a/server/src/main/java/org/elasticsearch/health/GetHealthAction.java +++ b/server/src/main/java/org/elasticsearch/health/GetHealthAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.core.Nullable; import org.elasticsearch.health.stats.HealthApiStats; @@ -199,7 +200,7 @@ public LocalAction( NodeClient client, HealthApiStats healthApiStats ) { - super(NAME, actionFilters, transportService.getTaskManager()); + super(NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.clusterService = clusterService; this.healthService = healthService; this.client = client; diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 3df5b3fe288a2..49eb6d84f0b1e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -20,6 +20,7 @@ import org.elasticsearch.index.mapper.DocumentDimensions; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MetadataFieldMapper; @@ -222,7 +223,7 @@ public boolean isSyntheticSourceEnabled() { return true; } }, - LOGS("logs") { + LOGSDB("logsdb") { @Override void validateWithOtherSettings(Map, Object> settings) { IndexMode.validateTimeSeriesSettings(settings); @@ -286,7 +287,11 @@ public boolean shouldValidateTimestamp() { } @Override - public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {} + public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) { + if (sourceFieldMapper.isSynthetic() == false) { + throw new IllegalArgumentException("Indices with with index mode [" + IndexMode.LOGSDB + "] only support synthetic source"); + } + } @Override public boolean isSyntheticSourceEnabled() { @@ -345,6 +350,10 @@ protected static String tsdbMode() { .startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH) .field("type", DateFieldMapper.CONTENT_TYPE) .endObject() + .startObject("host.name") + .field("type", KeywordFieldMapper.CONTENT_TYPE) + .field("ignore_above", 1024) + .endObject() .endObject() .endObject()) ); @@ -464,7 +473,7 @@ public static IndexMode fromString(String value) { return switch (value) { case "standard" -> IndexMode.STANDARD; case "time_series" -> IndexMode.TIME_SERIES; - case "logs" -> IndexMode.LOGS; + case "logsdb" -> IndexMode.LOGSDB; default -> throw new IllegalArgumentException( "[" + value diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 2b470eea0a8bf..cbe94526819d9 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -332,10 +332,18 @@ public NodeMappingStats getNodeMappingStats() { if (mapperService == null) { return null; } - long totalCount = mapperService().mappingLookup().getTotalMapperCount(); - long totalEstimatedOverhead = totalCount * 1024L; // 1KiB estimated per mapping - NodeMappingStats indexNodeMappingStats = new NodeMappingStats(totalCount, totalEstimatedOverhead); - return indexNodeMappingStats; + long numFields = mapperService().mappingLookup().getTotalMapperCount(); + long totalEstimatedOverhead = numFields * 1024L; // 1KiB estimated per mapping + // Assume all segments have the same mapping; otherwise, we need to acquire searchers to count the actual fields. + int numLeaves = 0; + for (IndexShard shard : shards.values()) { + try { + numLeaves += shard.commitStats().getNumLeaves(); + } catch (AlreadyClosedException ignored) { + + } + } + return new NodeMappingStats(numFields, totalEstimatedOverhead, numLeaves, numLeaves * numFields); } public Set shardIds() { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index f190462d6d1e9..a11a51ef7ad62 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -153,14 +153,14 @@ public IndexSortConfig(IndexSettings indexSettings) { } List fields = INDEX_SORT_FIELD_SETTING.get(settings); - if (this.indexMode == IndexMode.LOGS && fields.isEmpty()) { + if (this.indexMode == IndexMode.LOGSDB && fields.isEmpty()) { fields = List.of("host.name", DataStream.TIMESTAMP_FIELD_NAME); } this.sortSpecs = fields.stream().map(FieldSortSpec::new).toArray(FieldSortSpec[]::new); if (INDEX_SORT_ORDER_SETTING.exists(settings)) { List orders = INDEX_SORT_ORDER_SETTING.get(settings); - if (this.indexMode == IndexMode.LOGS && orders.isEmpty()) { + if (this.indexMode == IndexMode.LOGSDB && orders.isEmpty()) { orders = List.of(SortOrder.DESC, SortOrder.DESC); } if (orders.size() != sortSpecs.length) { @@ -175,7 +175,7 @@ public IndexSortConfig(IndexSettings indexSettings) { if (INDEX_SORT_MODE_SETTING.exists(settings)) { List modes = INDEX_SORT_MODE_SETTING.get(settings); - if (this.indexMode == IndexMode.LOGS && modes.isEmpty()) { + if (this.indexMode == IndexMode.LOGSDB && modes.isEmpty()) { modes = List.of(MultiValueMode.MIN, MultiValueMode.MIN); } if (modes.size() != sortSpecs.length) { @@ -188,7 +188,7 @@ public IndexSortConfig(IndexSettings indexSettings) { if (INDEX_SORT_MISSING_SETTING.exists(settings)) { List missingValues = INDEX_SORT_MISSING_SETTING.get(settings); - if (this.indexMode == IndexMode.LOGS && missingValues.isEmpty()) { + if (this.indexMode == IndexMode.LOGSDB && missingValues.isEmpty()) { missingValues = List.of("_first", "_first"); } if (missingValues.size() != sortSpecs.length) { diff --git a/server/src/main/java/org/elasticsearch/index/codec/CodecProvider.java b/server/src/main/java/org/elasticsearch/index/codec/CodecProvider.java new file mode 100644 index 0000000000000..277c2a578fa2c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/CodecProvider.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.codec; + +import org.apache.lucene.codecs.Codec; + +/** + * Abstracts codec lookup by name, to make CodecService extensible. + */ +@FunctionalInterface +public interface CodecProvider { + Codec codec(String name); +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java index 3ebcd1cb5b420..f95aeada762f1 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java +++ b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java @@ -9,6 +9,8 @@ package org.elasticsearch.index.codec; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.FeatureFlag; @@ -18,6 +20,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; /** * Since Lucene 4.0 low level index segments are read and written through a @@ -25,7 +28,7 @@ * data-structures per field. Elasticsearch exposes the full * {@link Codec} capabilities through this {@link CodecService}. */ -public class CodecService { +public class CodecService implements CodecProvider { public static final FeatureFlag ZSTD_STORED_FIELDS_FEATURE_FLAG = new FeatureFlag("zstd_stored_fields"); @@ -65,7 +68,13 @@ public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays) for (String codec : Codec.availableCodecs()) { codecs.put(codec, Codec.forName(codec)); } - this.codecs = Map.copyOf(codecs); + this.codecs = codecs.entrySet().stream().collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e -> { + var codec = e.getValue(); + if (codec instanceof DeduplicateFieldInfosCodec) { + return codec; + } + return new DeduplicateFieldInfosCodec(codec.getName(), codec); + })); } public Codec codec(String name) { @@ -77,9 +86,30 @@ public Codec codec(String name) { } /** - * Returns all registered available codec names + * Returns all registered available codec names. + * Public visibility for tests. */ public String[] availableCodecs() { return codecs.keySet().toArray(new String[0]); } + + public static class DeduplicateFieldInfosCodec extends FilterCodec { + + private final DeduplicatingFieldInfosFormat deduplicatingFieldInfosFormat; + + protected DeduplicateFieldInfosCodec(String name, Codec delegate) { + super(name, delegate); + this.deduplicatingFieldInfosFormat = new DeduplicatingFieldInfosFormat(super.fieldInfosFormat()); + } + + @Override + public final FieldInfosFormat fieldInfosFormat() { + return deduplicatingFieldInfosFormat; + } + + public final Codec delegate() { + return delegate; + } + + } } diff --git a/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java index dd7a668605e57..301d3129f7c2a 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java +++ b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java @@ -9,8 +9,6 @@ package org.elasticsearch.index.codec; import org.apache.lucene.codecs.DocValuesFormat; -import org.apache.lucene.codecs.FieldInfosFormat; -import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.StoredFieldsFormat; @@ -27,12 +25,10 @@ * Elasticsearch codec as of 8.14. This extends the Lucene 9.9 codec to compressed stored fields with ZSTD instead of LZ4/DEFLATE. See * {@link Zstd814StoredFieldsFormat}. */ -public class Elasticsearch814Codec extends FilterCodec { +public class Elasticsearch814Codec extends CodecService.DeduplicateFieldInfosCodec { private final StoredFieldsFormat storedFieldsFormat; - private final FieldInfosFormat fieldInfosFormat; - private final PostingsFormat defaultPostingsFormat; private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { @Override @@ -72,7 +68,6 @@ public Elasticsearch814Codec(Zstd814StoredFieldsFormat.Mode mode) { this.defaultPostingsFormat = new Lucene99PostingsFormat(); this.defaultDVFormat = new Lucene90DocValuesFormat(); this.defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat(); - this.fieldInfosFormat = new DeduplicatingFieldInfosFormat(delegate.fieldInfosFormat()); } @Override @@ -132,8 +127,4 @@ public KnnVectorsFormat getKnnVectorsFormatForField(String field) { return defaultKnnVectorsFormat; } - @Override - public FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; - } } diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java index 0b4bb9dfc10ae..1228c908f7c18 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java +++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java @@ -123,7 +123,7 @@ private boolean isTimeSeriesModeIndex() { } private boolean isLogsModeIndex() { - return mapperService != null && IndexMode.LOGS == mapperService.getIndexSettings().getMode(); + return mapperService != null && IndexMode.LOGSDB == mapperService.getIndexSettings().getMode(); } } diff --git a/server/src/main/java/org/elasticsearch/index/engine/CommitStats.java b/server/src/main/java/org/elasticsearch/index/engine/CommitStats.java index 16b58e001dcbe..1a03cf6c7bf98 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/CommitStats.java +++ b/server/src/main/java/org/elasticsearch/index/engine/CommitStats.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.engine; import org.apache.lucene.index.SegmentInfos; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -27,6 +28,7 @@ public final class CommitStats implements Writeable, ToXContentFragment { private final long generation; private final String id; // lucene commit id in base 64; private final int numDocs; + private final int numLeaves; public CommitStats(SegmentInfos segmentInfos) { // clone the map to protect against concurrent changes @@ -35,6 +37,7 @@ public CommitStats(SegmentInfos segmentInfos) { generation = segmentInfos.getLastGeneration(); id = Base64.getEncoder().encodeToString(segmentInfos.getId()); numDocs = Lucene.getNumDocs(segmentInfos); + numLeaves = segmentInfos.size(); } CommitStats(StreamInput in) throws IOException { @@ -42,6 +45,7 @@ public CommitStats(SegmentInfos segmentInfos) { generation = in.readLong(); id = in.readOptionalString(); numDocs = in.readInt(); + numLeaves = in.getTransportVersion().onOrAfter(TransportVersions.SEGMENT_LEVEL_FIELDS_STATS) ? in.readVInt() : 0; } @Override @@ -49,12 +53,16 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; CommitStats that = (CommitStats) o; - return userData.equals(that.userData) && generation == that.generation && Objects.equals(id, that.id) && numDocs == that.numDocs; + return userData.equals(that.userData) + && generation == that.generation + && Objects.equals(id, that.id) + && numDocs == that.numDocs + && numLeaves == that.numLeaves; } @Override public int hashCode() { - return Objects.hash(userData, generation, id, numDocs); + return Objects.hash(userData, generation, id, numDocs, numLeaves); } public static CommitStats readOptionalCommitStatsFrom(StreamInput in) throws IOException { @@ -81,12 +89,19 @@ public int getNumDocs() { return numDocs; } + public int getNumLeaves() { + return numLeaves; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeMap(userData, StreamOutput::writeString); out.writeLong(generation); out.writeOptionalString(id); out.writeInt(numDocs); + if (out.getTransportVersion().onOrAfter(TransportVersions.SEGMENT_LEVEL_FIELDS_STATS)) { + out.writeVInt(numLeaves); + } } static final class Fields { diff --git a/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java index 51840d2fbfcdd..f82ac04207604 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java @@ -23,7 +23,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.codec.CodecService; +import org.elasticsearch.index.codec.CodecProvider; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.seqno.RetentionLeases; import org.elasticsearch.index.shard.ShardId; @@ -60,7 +60,7 @@ public final class EngineConfig { private final MergePolicy mergePolicy; private final Analyzer analyzer; private final Similarity similarity; - private final CodecService codecService; + private final CodecProvider codecProvider; private final Engine.EventListener eventListener; private final QueryCache queryCache; private final QueryCachingPolicy queryCachingPolicy; @@ -148,7 +148,7 @@ public EngineConfig( MergePolicy mergePolicy, Analyzer analyzer, Similarity similarity, - CodecService codecService, + CodecProvider codecProvider, Engine.EventListener eventListener, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, @@ -176,7 +176,7 @@ public EngineConfig( this.mergePolicy = mergePolicy; this.analyzer = analyzer; this.similarity = similarity; - this.codecService = codecService; + this.codecProvider = codecProvider; this.eventListener = eventListener; codecName = indexSettings.getValue(INDEX_CODEC_SETTING); this.mapperService = mapperService; @@ -252,14 +252,22 @@ public boolean isEnableGcDeletes() { *

*/ public Codec getCodec() { - return codecService.codec(codecName); + return codecProvider.codec(codecName); } /** - * @return the {@link CodecService} + * @return the {@link CodecProvider} */ - public CodecService getCodecService() { - return codecService; + public CodecProvider getCodecProvider() { + return codecProvider; + } + + /** + * @return the {@link CodecProvider} + */ + @Deprecated // to avoid breaking serverless, just temporary + public CodecProvider getCodecService() { + return codecProvider; } /** diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 03d244cd8e4ef..70c593b590f5a 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -45,6 +45,8 @@ import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.service.ClusterApplierService; +import org.elasticsearch.common.ReferenceDocs; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.lucene.LoggerInfoStream; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; @@ -1688,7 +1690,13 @@ private Exception tryAcquireInFlightDocs(Operation operation, int addingDocs) { final long totalDocs = indexWriter.getPendingNumDocs() + inFlightDocCount.addAndGet(addingDocs); if (totalDocs > maxDocs) { releaseInFlightDocs(addingDocs); - return new IllegalArgumentException("Number of documents in the shard cannot exceed [" + maxDocs + "]"); + return new IllegalArgumentException( + Strings.format( + "Number of documents in the shard cannot exceed [%d]; for more information, see [%s]", + maxDocs, + ReferenceDocs.LUCENE_MAX_DOCS_LIMIT + ) + ); } else { return null; } @@ -2668,8 +2676,8 @@ private IndexWriter createWriter() throws IOException { } } - // pkg-private for testing - IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOException { + // protected for testing + protected IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOException { if (Assertions.ENABLED) { return new AssertingIndexWriter(directory, iwc); } else { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java b/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java index 3cfab4e599015..dd3d8c9ffda4b 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.fielddata; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.search.lookup.SearchLookup; @@ -25,6 +26,7 @@ */ public record FieldDataContext( String fullyQualifiedIndexName, + IndexSettings indexSettings, Supplier lookupSupplier, Function> sourcePathsLookup, MappedFieldType.FielddataOperation fielddataOperation @@ -38,11 +40,8 @@ public record FieldDataContext( * @param reason the reason that runtime fields are not supported */ public static FieldDataContext noRuntimeFields(String reason) { - return new FieldDataContext( - "", - () -> { throw new UnsupportedOperationException("Runtime fields not supported for [" + reason + "]"); }, - Set::of, - MappedFieldType.FielddataOperation.SEARCH - ); + return new FieldDataContext("", null, () -> { + throw new UnsupportedOperationException("Runtime fields not supported for [" + reason + "]"); + }, Set::of, MappedFieldType.FielddataOperation.SEARCH); } } diff --git a/server/src/main/java/org/elasticsearch/index/flush/FlushStats.java b/server/src/main/java/org/elasticsearch/index/flush/FlushStats.java index e514a6d2adac0..1f8caa3cc236e 100644 --- a/server/src/main/java/org/elasticsearch/index/flush/FlushStats.java +++ b/server/src/main/java/org/elasticsearch/index/flush/FlushStats.java @@ -34,8 +34,7 @@ public FlushStats(StreamInput in) throws IOException { total = in.readVLong(); totalTimeInMillis = in.readVLong(); periodic = in.readVLong(); - totalTimeExcludingWaitingOnLockInMillis = in.getTransportVersion() - .onOrAfter(TransportVersions.TRACK_FLUSH_TIME_EXCLUDING_WAITING_ON_LOCKS) ? in.readVLong() : 0L; + totalTimeExcludingWaitingOnLockInMillis = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) ? in.readVLong() : 0L; } public FlushStats(long total, long periodic, long totalTimeInMillis, long totalTimeExcludingWaitingOnLockInMillis) { @@ -131,7 +130,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(total); out.writeVLong(totalTimeInMillis); out.writeVLong(periodic); - if (out.getTransportVersion().onOrAfter(TransportVersions.TRACK_FLUSH_TIME_EXCLUDING_WAITING_ON_LOCKS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeVLong(totalTimeExcludingWaitingOnLockInMillis); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java index ece0b082a3ccf..6f4a8ffa92cbe 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java @@ -8,7 +8,8 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.LongField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; import org.elasticsearch.common.bytes.BytesReference; @@ -22,7 +23,6 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.time.Instant; -import java.util.List; import java.util.Map; import static org.elasticsearch.core.TimeValue.NSEC_PER_MSEC; @@ -36,6 +36,7 @@ public class DataStreamTimestampFieldMapper extends MetadataFieldMapper { public static final String NAME = "_data_stream_timestamp"; public static final String DEFAULT_PATH = "@timestamp"; + public static final String TIMESTAMP_VALUE_KEY = "@timestamp._value"; public static final DataStreamTimestampFieldMapper ENABLED_INSTANCE = new DataStreamTimestampFieldMapper(true); private static final DataStreamTimestampFieldMapper DISABLED_INSTANCE = new DataStreamTimestampFieldMapper(false); @@ -189,43 +190,44 @@ public void doValidate(MappingLookup lookup) { } } + public static void storeTimestampValueForReuse(LuceneDocument document, long timestamp) { + var existingField = document.getByKey(DataStreamTimestampFieldMapper.TIMESTAMP_VALUE_KEY); + if (existingField != null) { + throw new IllegalArgumentException("data stream timestamp field [" + DEFAULT_PATH + "] encountered multiple values"); + } + + document.onlyAddKey( + DataStreamTimestampFieldMapper.TIMESTAMP_VALUE_KEY, + new LongField(DataStreamTimestampFieldMapper.TIMESTAMP_VALUE_KEY, timestamp, Field.Store.NO) + ); + } + + public static long extractTimestampValue(LuceneDocument document) { + IndexableField timestampValueField = document.getByKey(TIMESTAMP_VALUE_KEY); + if (timestampValueField == null) { + throw new IllegalArgumentException("data stream timestamp field [" + DEFAULT_PATH + "] is missing"); + } + + return timestampValueField.numericValue().longValue(); + } + @Override public void postParse(DocumentParserContext context) throws IOException { if (enabled == false) { // not configured, so skip the validation return; } - boolean foundFsTimestampField = false; - IndexableField first = null; - final List fields = context.rootDoc().getFields(); - for (int i = 0; i < fields.size(); i++) { - IndexableField indexableField = fields.get(i); - if (DEFAULT_PATH.equals(indexableField.name()) == false) { - continue; - } - if (first == null) { - first = indexableField; - } - if (indexableField.fieldType().docValuesType() == DocValuesType.SORTED_NUMERIC) { - if (foundFsTimestampField) { - throw new IllegalArgumentException("data stream timestamp field [" + DEFAULT_PATH + "] encountered multiple values"); - } - foundFsTimestampField = true; - } - } - if (first == null) { - throw new IllegalArgumentException("data stream timestamp field [" + DEFAULT_PATH + "] is missing"); - } + long timestamp = extractTimestampValue(context.doc()); + var indexMode = context.indexSettings().getMode(); if (indexMode.shouldValidateTimestamp()) { TimestampBounds bounds = context.indexSettings().getTimestampBounds(); - validateTimestamp(bounds, first, context); + validateTimestamp(bounds, timestamp, context); } } - private static void validateTimestamp(TimestampBounds bounds, IndexableField field, DocumentParserContext context) { - long originValue = field.numericValue().longValue(); + private static void validateTimestamp(TimestampBounds bounds, long originValue, DocumentParserContext context) { long value = originValue; Resolution resolution; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 501d31547ded1..c70414807cdce 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -364,6 +364,7 @@ public DateFieldMapper build(MapperBuilderContext context) { && ignoreMalformed.isConfigured() == false) { ignoreMalformed.setValue(false); } + return new DateFieldMapper( leafName(), ft, @@ -868,8 +869,10 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { private final ScriptCompiler scriptCompiler; private final FieldValues scriptValues; + private final boolean isDataStreamTimestampField; + private DateFieldMapper( - String simpleName, + String leafName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, @@ -878,7 +881,7 @@ private DateFieldMapper( boolean isSourceSynthetic, Builder builder ) { - super(simpleName, mappedFieldType, multiFields, copyTo, builder.script.get() != null, builder.onScriptError.get()); + super(leafName, mappedFieldType, multiFields, copyTo, builder.script.get() != null, builder.onScriptError.get()); this.store = builder.store.getValue(); this.indexed = builder.index.getValue(); this.hasDocValues = builder.docValues.getValue(); @@ -894,6 +897,7 @@ private DateFieldMapper( this.script = builder.script.get(); this.scriptCompiler = builder.scriptCompiler; this.scriptValues = builder.scriptValues(); + this.isDataStreamTimestampField = mappedFieldType.name().equals(DataStreamTimestampFieldMapper.DEFAULT_PATH); } @Override @@ -942,6 +946,16 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio } private void indexValue(DocumentParserContext context, long timestamp) { + // DataStreamTimestampFieldMapper and TsidExtractingFieldMapper need to use timestamp value, + // so when this is true we store it in a well-known place + // instead of forcing them to iterate over all fields. + // + // DataStreamTimestampFieldMapper is present and enabled both + // in data streams and standalone indices in time_series mode + if (isDataStreamTimestampField && context.mappingLookup().isDataStreamTimestampFieldEnabled()) { + DataStreamTimestampFieldMapper.storeTimestampValueForReuse(context.doc(), timestamp); + } + if (indexed && hasDocValues) { context.doc().add(new LongField(fieldType().name(), timestamp)); } else if (hasDocValues) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index f0b0142101a7f..8bf7f3f4e72a3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -161,7 +161,13 @@ private static void executeIndexTimeScripts(DocumentParserContext context) { SearchLookup searchLookup = new SearchLookup( context.mappingLookup().indexTimeLookup()::get, (ft, lookup, fto) -> ft.fielddataBuilder( - new FieldDataContext(context.indexSettings().getIndex().getName(), lookup, context.mappingLookup()::sourcePaths, fto) + new FieldDataContext( + context.indexSettings().getIndex().getName(), + context.indexSettings(), + lookup, + context.mappingLookup()::sourcePaths, + fto + ) ).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), (ctx, doc) -> Source.fromBytes(context.sourceToParse().source()) ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java new file mode 100644 index 0000000000000..8fa1ff3040563 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.fielddata.FieldData; +import org.elasticsearch.index.fielddata.FieldDataContext; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData; +import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.script.field.DelegateDocValuesField; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.fetch.StoredFieldsSpec; +import org.elasticsearch.search.lookup.Source; + +import java.util.Collections; +import java.util.List; + +public class IndexModeFieldMapper extends MetadataFieldMapper { + + static final NodeFeature QUERYING_INDEX_MODE = new NodeFeature("mapper.query_index_mode"); + + public static final String NAME = "_index_mode"; + + public static final String CONTENT_TYPE = "_index_mode"; + + private static final IndexModeFieldMapper INSTANCE = new IndexModeFieldMapper(); + + public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE); + + static final class IndexModeFieldType extends ConstantFieldType { + + static final IndexModeFieldType INSTANCE = new IndexModeFieldType(); + + private IndexModeFieldType() { + super(NAME, Collections.emptyMap()); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + protected boolean matches(String pattern, boolean caseInsensitive, QueryRewriteContext context) { + final String indexMode = context.getIndexSettings().getMode().getName(); + return Regex.simpleMatch(pattern, indexMode, caseInsensitive); + } + + @Override + public Query existsQuery(SearchExecutionContext context) { + return new MatchAllDocsQuery(); + } + + @Override + public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { + final String indexMode = fieldDataContext.indexSettings().getMode().getName(); + return new ConstantIndexFieldData.Builder( + indexMode, + name(), + CoreValuesSourceType.KEYWORD, + (dv, n) -> new DelegateDocValuesField( + new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))), + n + ) + ); + } + + @Override + public BlockLoader blockLoader(BlockLoaderContext blContext) { + final String indexMode = blContext.indexSettings().getMode().getName(); + return BlockLoader.constantBytes(new BytesRef(indexMode)); + } + + @Override + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + return new ValueFetcher() { + private final List indexMode = List.of(context.getIndexSettings().getMode().getName()); + + @Override + public List fetchValues(Source source, int doc, List ignoredValues) { + return indexMode; + } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + }; + } + + } + + public IndexModeFieldMapper() { + super(IndexModeFieldType.INSTANCE); + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index a554e6e44a8e8..aec0c580f1c51 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.query.DistanceFeatureQueryBuilder; @@ -693,6 +694,11 @@ public interface BlockLoaderContext { */ String indexName(); + /** + * The index settings of the index + */ + IndexSettings indexSettings(); + /** * How the field should be extracted into the BlockLoader. The default is {@link FieldExtractPreference#NONE}, which means * that the field type can choose where to load the field from. However, in some cases, the caller may have a preference. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 755c2d94571d3..ac3275c5e7119 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -28,7 +28,9 @@ public Set getFeatures() { DenseVectorFieldMapper.INT4_QUANTIZATION, DenseVectorFieldMapper.BIT_VECTORS, DocumentMapper.INDEX_SORTING_ON_NESTED, - KeywordFieldMapper.KEYWORD_DIMENSION_IGNORE_ABOVE + KeywordFieldMapper.KEYWORD_DIMENSION_IGNORE_ABOVE, + IndexModeFieldMapper.QUERYING_INDEX_MODE, + NodeMappingStats.SEGMENT_LEVEL_FIELDS_STATS ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java b/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java index 87a4f1367e108..77f86816d6fb7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java @@ -8,11 +8,13 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Nullable; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; @@ -25,15 +27,22 @@ */ public class NodeMappingStats implements Writeable, ToXContentFragment { + public static final NodeFeature SEGMENT_LEVEL_FIELDS_STATS = new NodeFeature("mapper.segment_level_fields_stats"); + private static final class Fields { static final String MAPPINGS = "mappings"; static final String TOTAL_COUNT = "total_count"; static final String TOTAL_ESTIMATED_OVERHEAD = "total_estimated_overhead"; static final String TOTAL_ESTIMATED_OVERHEAD_IN_BYTES = "total_estimated_overhead_in_bytes"; + static final String TOTAL_SEGMENTS = "total_segments"; + static final String TOTAL_SEGMENT_FIELDS = "total_segment_fields"; + static final String AVERAGE_FIELDS_PER_SEGMENT = "average_fields_per_segment"; } private long totalCount; private long totalEstimatedOverhead; + private long totalSegments; + private long totalSegmentFields; public NodeMappingStats() { @@ -42,17 +51,25 @@ public NodeMappingStats() { public NodeMappingStats(StreamInput in) throws IOException { totalCount = in.readVLong(); totalEstimatedOverhead = in.readVLong(); + if (in.getTransportVersion().onOrAfter(TransportVersions.SEGMENT_LEVEL_FIELDS_STATS)) { + totalSegments = in.readVLong(); + totalSegmentFields = in.readVLong(); + } } - public NodeMappingStats(long totalCount, long totalEstimatedOverhead) { + public NodeMappingStats(long totalCount, long totalEstimatedOverhead, long totalSegments, long totalSegmentFields) { this.totalCount = totalCount; this.totalEstimatedOverhead = totalEstimatedOverhead; + this.totalSegments = totalSegments; + this.totalSegmentFields = totalSegmentFields; } public void add(@Nullable NodeMappingStats other) { if (other == null) return; this.totalCount += other.totalCount; this.totalEstimatedOverhead += other.totalEstimatedOverhead; + this.totalSegments += other.totalSegments; + this.totalSegmentFields += other.totalSegmentFields; } public long getTotalCount() { @@ -63,10 +80,22 @@ public ByteSizeValue getTotalEstimatedOverhead() { return ByteSizeValue.ofBytes(totalEstimatedOverhead); } + public long getTotalSegments() { + return totalSegments; + } + + public long getTotalSegmentFields() { + return totalSegmentFields; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeVLong(totalCount); out.writeVLong(totalEstimatedOverhead); + if (out.getTransportVersion().onOrAfter(TransportVersions.SEGMENT_LEVEL_FIELDS_STATS)) { + out.writeVLong(totalSegments); + out.writeVLong(totalSegmentFields); + } } @Override @@ -74,6 +103,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(Fields.MAPPINGS); builder.field(Fields.TOTAL_COUNT, getTotalCount()); builder.humanReadableField(Fields.TOTAL_ESTIMATED_OVERHEAD_IN_BYTES, Fields.TOTAL_ESTIMATED_OVERHEAD, getTotalEstimatedOverhead()); + builder.field(Fields.TOTAL_SEGMENTS, totalSegments); + builder.field(Fields.TOTAL_SEGMENT_FIELDS, totalSegmentFields); + builder.field(Fields.AVERAGE_FIELDS_PER_SEGMENT, totalSegments == 0 ? 0 : totalSegmentFields / totalSegments); builder.endObject(); return builder; } @@ -83,11 +115,14 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; NodeMappingStats that = (NodeMappingStats) o; - return totalCount == that.totalCount && totalEstimatedOverhead == that.totalEstimatedOverhead; + return totalCount == that.totalCount + && totalEstimatedOverhead == that.totalEstimatedOverhead + && totalSegments == that.totalSegments + && totalSegmentFields == that.totalSegmentFields; } @Override public int hashCode() { - return Objects.hash(totalCount, totalEstimatedOverhead); + return Objects.hash(totalCount, totalEstimatedOverhead, totalSegments, totalSegmentFields); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 67e457907f8cc..908108bce31da 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -69,12 +69,12 @@ private enum Mode { IndexMode.TIME_SERIES ); - private static final SourceFieldMapper LOGS_DEFAULT = new SourceFieldMapper( + private static final SourceFieldMapper LOGSDB_DEFAULT = new SourceFieldMapper( Mode.SYNTHETIC, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, - IndexMode.LOGS + IndexMode.LOGSDB ); /* @@ -184,7 +184,7 @@ public SourceFieldMapper build() { if (isDefault()) { return switch (indexMode) { case TIME_SERIES -> TSDB_DEFAULT; - case LOGS -> LOGS_DEFAULT; + case LOGSDB -> LOGSDB_DEFAULT; default -> DEFAULT; }; } @@ -234,8 +234,8 @@ public SourceFieldMapper build() { } else { return TSDB_LEGACY_DEFAULT; } - } else if (indexMode == IndexMode.LOGS) { - return LOGS_DEFAULT; + } else if (indexMode == IndexMode.LOGSDB) { + return LOGSDB_DEFAULT; } } return DEFAULT; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java index 80e6d04d967d5..f574e509df9b9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java @@ -45,7 +45,13 @@ protected BytesRef indexedValueForSearch(Object value) { @Override public Query termQueryCaseInsensitive(Object value, SearchExecutionContext context) { failIfNotIndexed(); - return AutomatonQueries.caseInsensitiveTermQuery(new Term(name(), indexedValueForSearch(value))); + final BytesRef valueForSearch = indexedValueForSearch(value); + // check if valueForSearch is the same as an empty string + // if we have a length of zero, just do a regular term query + if (valueForSearch.length == 0) { + return termQuery(value, context); + } + return AutomatonQueries.caseInsensitiveTermQuery(new Term(name(), valueForSearch)); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 8101b5be1b60e..5bb8145a090a1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -46,13 +46,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext private static final long SEED = 0; public static void createField(DocumentParserContext context, IndexRouting.ExtractFromSource.Builder routingBuilder, BytesRef tsid) { - final IndexableField timestampField = context.rootDoc().getField(DataStreamTimestampFieldMapper.DEFAULT_PATH); - if (timestampField == null) { - throw new IllegalArgumentException( - "data stream timestamp field [" + DataStreamTimestampFieldMapper.DEFAULT_PATH + "] is missing" - ); - } - long timestamp = timestampField.numericValue().longValue(); + final long timestamp = DataStreamTimestampFieldMapper.extractTimestampValue(context.doc()); String id; if (routingBuilder != null) { byte[] suffix = new byte[16]; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index d27c0acdb6b2e..8ffe4b4cc4a66 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -226,14 +226,17 @@ public Builder(String name, IndexVersion indexVersionCreated) { if (v != null && dims.isConfigured() && dims.get() != null) { v.validateDimension(dims.get()); } - if (v != null && v.supportsElementType(elementType.getValue()) == false) { - throw new IllegalArgumentException( - "[element_type] cannot be [" + elementType.getValue().toString() + "] when using index type [" + v.type + "]" - ); + if (v != null) { + v.validateElementType(elementType.getValue()); } }) .acceptsNull() - .setMergeValidator((previous, current, c) -> previous == null || current == null || previous.updatableTo(current)); + .setMergeValidator( + (previous, current, c) -> previous == null + || current == null + || Objects.equals(previous, current) + || previous.updatableTo(current) + ); if (defaultInt8Hnsw) { this.indexOptions.alwaysSerialize(); } @@ -1146,22 +1149,50 @@ public final String toString() { } abstract static class IndexOptions implements ToXContent { - final String type; + final VectorIndexType type; - IndexOptions(String type) { + IndexOptions(VectorIndexType type) { this.type = type; } abstract KnnVectorsFormat getVectorsFormat(ElementType elementType); - boolean supportsElementType(ElementType elementType) { - return true; + final void validateElementType(ElementType elementType) { + if (type.supportsElementType(elementType) == false) { + throw new IllegalArgumentException( + "[element_type] cannot be [" + elementType.toString() + "] when using index type [" + type + "]" + ); + } } abstract boolean updatableTo(IndexOptions update); - void validateDimension(int dim) { - // no-op + public final void validateDimension(int dim) { + if (type.supportsDimension(dim)) { + return; + } + throw new IllegalArgumentException(type.name + " only supports even dimensions; provided=" + dim); + } + + abstract boolean doEquals(IndexOptions other); + + abstract int doHashCode(); + + @Override + public final boolean equals(Object other) { + if (other == this) { + return true; + } + if (other == null || other.getClass() != getClass()) { + return false; + } + IndexOptions otherOptions = (IndexOptions) other; + return Objects.equals(type, otherOptions.type) && doEquals(otherOptions); + } + + @Override + public final int hashCode() { + return Objects.hash(type, doHashCode()); } } @@ -1182,6 +1213,16 @@ public IndexOptions parseIndexOptions(String fieldName, Map indexOpti MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); return new HnswIndexOptions(m, efConstruction); } + + @Override + public boolean supportsElementType(ElementType elementType) { + return true; + } + + @Override + public boolean supportsDimension(int dims) { + return true; + } }, INT8_HNSW("int8_hnsw") { @Override @@ -1204,6 +1245,16 @@ public IndexOptions parseIndexOptions(String fieldName, Map indexOpti MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); return new Int8HnswIndexOptions(m, efConstruction, confidenceInterval); } + + @Override + public boolean supportsElementType(ElementType elementType) { + return elementType == ElementType.FLOAT; + } + + @Override + public boolean supportsDimension(int dims) { + return true; + } }, INT4_HNSW("int4_hnsw") { public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap) { @@ -1225,6 +1276,16 @@ public IndexOptions parseIndexOptions(String fieldName, Map indexOpti MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); return new Int4HnswIndexOptions(m, efConstruction, confidenceInterval); } + + @Override + public boolean supportsElementType(ElementType elementType) { + return elementType == ElementType.FLOAT; + } + + @Override + public boolean supportsDimension(int dims) { + return dims % 2 == 0; + } }, FLAT("flat") { @Override @@ -1232,6 +1293,16 @@ public IndexOptions parseIndexOptions(String fieldName, Map indexOpti MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); return new FlatIndexOptions(); } + + @Override + public boolean supportsElementType(ElementType elementType) { + return true; + } + + @Override + public boolean supportsDimension(int dims) { + return true; + } }, INT8_FLAT("int8_flat") { @Override @@ -1244,6 +1315,16 @@ public IndexOptions parseIndexOptions(String fieldName, Map indexOpti MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); return new Int8FlatIndexOptions(confidenceInterval); } + + @Override + public boolean supportsElementType(ElementType elementType) { + return elementType == ElementType.FLOAT; + } + + @Override + public boolean supportsDimension(int dims) { + return true; + } }, INT4_FLAT("int4_flat") { @Override @@ -1256,6 +1337,16 @@ public IndexOptions parseIndexOptions(String fieldName, Map indexOpti MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); return new Int4FlatIndexOptions(confidenceInterval); } + + @Override + public boolean supportsElementType(ElementType elementType) { + return elementType == ElementType.FLOAT; + } + + @Override + public boolean supportsDimension(int dims) { + return dims % 2 == 0; + } }; static Optional fromString(String type) { @@ -1269,13 +1360,22 @@ static Optional fromString(String type) { } abstract IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap); + + public abstract boolean supportsElementType(ElementType elementType); + + public abstract boolean supportsDimension(int dims); + + @Override + public String toString() { + return name; + } } static class Int8FlatIndexOptions extends IndexOptions { private final Float confidenceInterval; Int8FlatIndexOptions(Float confidenceInterval) { - super("int8_flat"); + super(VectorIndexType.INT8_FLAT); this.confidenceInterval = confidenceInterval; } @@ -1297,35 +1397,30 @@ KnnVectorsFormat getVectorsFormat(ElementType elementType) { } @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + boolean doEquals(IndexOptions o) { Int8FlatIndexOptions that = (Int8FlatIndexOptions) o; return Objects.equals(confidenceInterval, that.confidenceInterval); } @Override - public int hashCode() { + int doHashCode() { return Objects.hash(confidenceInterval); } - @Override - boolean supportsElementType(ElementType elementType) { - return elementType == ElementType.FLOAT; - } - @Override boolean updatableTo(IndexOptions update) { return update.type.equals(this.type) - || update.type.equals(VectorIndexType.HNSW.name) - || update.type.equals(VectorIndexType.INT8_HNSW.name); + || update.type.equals(VectorIndexType.HNSW) + || update.type.equals(VectorIndexType.INT8_HNSW) + || update.type.equals(VectorIndexType.INT4_HNSW) + || update.type.equals(VectorIndexType.INT4_FLAT); } } static class FlatIndexOptions extends IndexOptions { FlatIndexOptions() { - super("flat"); + super(VectorIndexType.FLAT); } @Override @@ -1350,13 +1445,12 @@ boolean updatableTo(IndexOptions update) { } @Override - public boolean equals(Object o) { - if (this == o) return true; - return o != null && getClass() == o.getClass(); + public boolean doEquals(IndexOptions o) { + return o instanceof FlatIndexOptions; } @Override - public int hashCode() { + public int doHashCode() { return Objects.hash(type); } } @@ -1367,7 +1461,7 @@ static class Int4HnswIndexOptions extends IndexOptions { private final float confidenceInterval; Int4HnswIndexOptions(int m, int efConstruction, Float confidenceInterval) { - super("int4_hnsw"); + super(VectorIndexType.INT4_HNSW); this.m = m; this.efConstruction = efConstruction; // The default confidence interval for int4 is dynamic quantiles, this provides the best relevancy and is @@ -1393,15 +1487,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + public boolean doEquals(IndexOptions o) { Int4HnswIndexOptions that = (Int4HnswIndexOptions) o; return m == that.m && efConstruction == that.efConstruction && Objects.equals(confidenceInterval, that.confidenceInterval); } @Override - public int hashCode() { + public int doHashCode() { return Objects.hash(m, efConstruction, confidenceInterval); } @@ -1418,21 +1510,16 @@ public String toString() { + "}"; } - @Override - boolean supportsElementType(ElementType elementType) { - return elementType == ElementType.FLOAT; - } - @Override boolean updatableTo(IndexOptions update) { - return Objects.equals(this, update); - } - - @Override - void validateDimension(int dim) { - if (dim % 2 != 0) { - throw new IllegalArgumentException("int4_hnsw only supports even dimensions; provided=" + dim); + boolean updatable = update.type.equals(this.type); + if (updatable) { + Int4HnswIndexOptions int4HnswIndexOptions = (Int4HnswIndexOptions) update; + // fewer connections would break assumptions on max number of connections (based on largest previous graph) during merge + // quantization could not behave as expected with different confidence intervals (and quantiles) to be created + updatable = int4HnswIndexOptions.m >= this.m && confidenceInterval == int4HnswIndexOptions.confidenceInterval; } + return updatable; } } @@ -1440,7 +1527,7 @@ static class Int4FlatIndexOptions extends IndexOptions { private final float confidenceInterval; Int4FlatIndexOptions(Float confidenceInterval) { - super("int4_flat"); + super(VectorIndexType.INT4_FLAT); // The default confidence interval for int4 is dynamic quantiles, this provides the best relevancy and is // effectively required for int4 to behave well across a wide range of data. this.confidenceInterval = confidenceInterval == null ? 0f : confidenceInterval; @@ -1462,7 +1549,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } @Override - public boolean equals(Object o) { + public boolean doEquals(IndexOptions o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Int4FlatIndexOptions that = (Int4FlatIndexOptions) o; @@ -1470,7 +1557,7 @@ public boolean equals(Object o) { } @Override - public int hashCode() { + public int doHashCode() { return Objects.hash(confidenceInterval); } @@ -1479,23 +1566,15 @@ public String toString() { return "{type=" + type + ", confidence_interval=" + confidenceInterval + "}"; } - @Override - boolean supportsElementType(ElementType elementType) { - return elementType == ElementType.FLOAT; - } - @Override boolean updatableTo(IndexOptions update) { // TODO: add support for updating from flat, hnsw, and int8_hnsw and updating params - return Objects.equals(this, update); + return update.type.equals(this.type) + || update.type.equals(VectorIndexType.HNSW) + || update.type.equals(VectorIndexType.INT8_HNSW) + || update.type.equals(VectorIndexType.INT4_HNSW); } - @Override - void validateDimension(int dim) { - if (dim % 2 != 0) { - throw new IllegalArgumentException("int4_flat only supports even dimensions; provided=" + dim); - } - } } static class Int8HnswIndexOptions extends IndexOptions { @@ -1504,7 +1583,7 @@ static class Int8HnswIndexOptions extends IndexOptions { private final Float confidenceInterval; Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval) { - super("int8_hnsw"); + super(VectorIndexType.INT8_HNSW); this.m = m; this.efConstruction = efConstruction; this.confidenceInterval = confidenceInterval; @@ -1530,7 +1609,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } @Override - public boolean equals(Object o) { + public boolean doEquals(IndexOptions o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Int8HnswIndexOptions that = (Int8HnswIndexOptions) o; @@ -1538,7 +1617,7 @@ public boolean equals(Object o) { } @Override - public int hashCode() { + public int doHashCode() { return Objects.hash(m, efConstruction, confidenceInterval); } @@ -1555,15 +1634,10 @@ public String toString() { + "}"; } - @Override - boolean supportsElementType(ElementType elementType) { - return elementType == ElementType.FLOAT; - } - @Override boolean updatableTo(IndexOptions update) { - boolean updatable = update.type.equals(this.type); - if (updatable) { + boolean updatable; + if (update.type.equals(this.type)) { Int8HnswIndexOptions int8HnswIndexOptions = (Int8HnswIndexOptions) update; // fewer connections would break assumptions on max number of connections (based on largest previous graph) during merge // quantization could not behave as expected with different confidence intervals (and quantiles) to be created @@ -1571,6 +1645,8 @@ boolean updatableTo(IndexOptions update) { updatable &= confidenceInterval == null || int8HnswIndexOptions.confidenceInterval != null && confidenceInterval.equals(int8HnswIndexOptions.confidenceInterval); + } else { + updatable = update.type.equals(VectorIndexType.INT4_HNSW) && ((Int4HnswIndexOptions) update).m >= this.m; } return updatable; } @@ -1581,7 +1657,7 @@ static class HnswIndexOptions extends IndexOptions { private final int efConstruction; HnswIndexOptions(int m, int efConstruction) { - super("hnsw"); + super(VectorIndexType.HNSW); this.m = m; this.efConstruction = efConstruction; } @@ -1602,7 +1678,9 @@ boolean updatableTo(IndexOptions update) { HnswIndexOptions hnswIndexOptions = (HnswIndexOptions) update; updatable = hnswIndexOptions.m >= this.m; } - return updatable || (update.type.equals(VectorIndexType.INT8_HNSW.name) && ((Int8HnswIndexOptions) update).m >= m); + return updatable + || (update.type.equals(VectorIndexType.INT8_HNSW) && ((Int8HnswIndexOptions) update).m >= m) + || (update.type.equals(VectorIndexType.INT4_HNSW) && ((Int4HnswIndexOptions) update).m >= m); } @Override @@ -1616,7 +1694,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } @Override - public boolean equals(Object o) { + public boolean doEquals(IndexOptions o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; HnswIndexOptions that = (HnswIndexOptions) o; @@ -1624,8 +1702,8 @@ public boolean equals(Object o) { } @Override - public int hashCode() { - return Objects.hash(type, m, efConstruction); + public int doHashCode() { + return Objects.hash(m, efConstruction); } @Override @@ -1757,17 +1835,6 @@ && isNotUnitVector(squaredMagnitude)) { return new DenseVectorQuery.Floats(queryVector, name()); } - Query createKnnQuery( - float[] queryVector, - Integer k, - int numCands, - Query filter, - Float similarityThreshold, - BitSetProducer parentFilter - ) { - return createKnnQuery(VectorData.fromFloats(queryVector), k, numCands, filter, similarityThreshold, parentFilter); - } - public Query createKnnQuery( VectorData queryVector, Integer k, @@ -1884,10 +1951,6 @@ int getVectorDimensions() { ElementType getElementType() { return elementType; } - - IndexOptions getIndexOptions() { - return indexOptions; - } } private final IndexOptions indexOptions; diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 315ce37244219..f25a0c73ac25d 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -339,6 +339,7 @@ public > IFD getForField(MappedFieldType fieldType fieldType, new FieldDataContext( getFullyQualifiedIndex().getName(), + getIndexSettings(), () -> this.lookup().forkAndTrackFieldReferences(fieldType.name()), this::sourcePath, fielddataOperation @@ -515,7 +516,13 @@ public void setLookupProviders( this::getFieldType, (fieldType, searchLookup, fielddataOperation) -> indexFieldDataLookup.apply( fieldType, - new FieldDataContext(getFullyQualifiedIndex().getName(), searchLookup, this::sourcePath, fielddataOperation) + new FieldDataContext( + getFullyQualifiedIndex().getName(), + getIndexSettings(), + searchLookup, + this::sourcePath, + fielddataOperation + ) ), sourceProvider, fieldLookupProvider diff --git a/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java index 7d3df2c174a83..a051d9c2df430 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java @@ -63,8 +63,8 @@ public GlobalCheckpointSyncAction( Request::new, Request::new, threadPool.executor(ThreadPool.Names.WRITE), - false, - true + SyncGlobalCheckpointAfterOperation.DoNotSync, + PrimaryActionExecution.Force ); } diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java index 541e279d4cfbb..0aa0f0b8d1556 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java @@ -81,7 +81,9 @@ public RetentionLeaseBackgroundSyncAction( actionFilters, Request::new, Request::new, - threadPool.executor(ThreadPool.Names.MANAGEMENT) + threadPool.executor(ThreadPool.Names.MANAGEMENT), + SyncGlobalCheckpointAfterOperation.DoNotSync, + PrimaryActionExecution.RejectOnOverload ); } diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java index b5fe27fb20bc3..0efcf8ac9298b 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java @@ -91,7 +91,7 @@ public RetentionLeaseSyncAction( RetentionLeaseSyncAction.Request::new, RetentionLeaseSyncAction.Request::new, new ManagementOnlyExecutorFunction(threadPool), - false, + PrimaryActionExecution.RejectOnOverload, indexingPressure, systemIndices ); diff --git a/server/src/main/java/org/elasticsearch/index/shard/ShardId.java b/server/src/main/java/org/elasticsearch/index/shard/ShardId.java index 30e3a13d54224..aaefb469b7bc0 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/ShardId.java +++ b/server/src/main/java/org/elasticsearch/index/shard/ShardId.java @@ -108,14 +108,17 @@ private int computeHashCode() { @Override public int compareTo(ShardId o) { - if (o.getId() == shardId) { - int compare = index.getName().compareTo(o.getIndex().getName()); - if (compare != 0) { - return compare; - } - return index.getUUID().compareTo(o.getIndex().getUUID()); + final int res = Integer.compare(shardId, o.shardId); + if (res != 0) { + return res; } - return Integer.compare(shardId, o.getId()); + final Index index = this.index; + final Index otherIndex = o.index; + int compare = index.getName().compareTo(otherIndex.getName()); + if (compare != 0) { + return compare; + } + return index.getUUID().compareTo(otherIndex.getUUID()); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java index 113d3c8f28a19..b17545a4cbeb6 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java @@ -33,8 +33,8 @@ /** * Contains information about all snapshots for the given shard in repository *

- * This class is used to find files that were already snapshotted and clear out files that no longer referenced by any - * snapshots. + * This class is used to find shard files that were already snapshotted and clear out shard files that are no longer referenced by any + * snapshots of the shard. */ public class BlobStoreIndexShardSnapshots implements Iterable, ToXContentFragment { @@ -48,6 +48,10 @@ private BlobStoreIndexShardSnapshots(Map files, List retainedSnapshots) { if (retainedSnapshots.isEmpty()) { return EMPTY; @@ -68,6 +72,10 @@ public BlobStoreIndexShardSnapshots withRetainedSnapshots(Set retain return new BlobStoreIndexShardSnapshots(newFiles, updatedSnapshots); } + /** + * Creates a new list of the shard's snapshots ({@link BlobStoreIndexShardSnapshots}) adding a new shard snapshot + * ({@link SnapshotFiles}). + */ public BlobStoreIndexShardSnapshots withAddedSnapshot(SnapshotFiles snapshotFiles) { Map updatedFiles = null; for (FileInfo fileInfo : snapshotFiles.indexFiles()) { diff --git a/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java b/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java index 1316776ec39b2..21ccdd56900f8 100644 --- a/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java +++ b/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java @@ -73,7 +73,7 @@ public IndexingPressureStats(StreamInput in) throws IOException { this.currentPrimaryOps = 0; this.currentReplicaOps = 0; - if (in.getTransportVersion().onOrAfter(TransportVersions.INDEXING_PRESSURE_DOCUMENT_REJECTIONS_COUNT)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { primaryDocumentRejections = in.readVLong(); } else { primaryDocumentRejections = -1L; @@ -152,7 +152,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(memoryLimit); } - if (out.getTransportVersion().onOrAfter(TransportVersions.INDEXING_PRESSURE_DOCUMENT_REJECTIONS_COUNT)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeVLong(primaryDocumentRejections); } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 17e0105d59d8c..03df21531d4cc 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -41,6 +41,7 @@ import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; +import org.elasticsearch.index.mapper.IndexModeFieldMapper; import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.IpScriptFieldType; import org.elasticsearch.index.mapper.KeywordFieldMapper; @@ -258,6 +259,7 @@ private static Map initBuiltInMetadataMa builtInMetadataMappers.put(TimeSeriesIdFieldMapper.NAME, TimeSeriesIdFieldMapper.PARSER); builtInMetadataMappers.put(TimeSeriesRoutingHashFieldMapper.NAME, TimeSeriesRoutingHashFieldMapper.PARSER); builtInMetadataMappers.put(IndexFieldMapper.NAME, IndexFieldMapper.PARSER); + builtInMetadataMappers.put(IndexModeFieldMapper.NAME, IndexModeFieldMapper.PARSER); builtInMetadataMappers.put(SourceFieldMapper.NAME, SourceFieldMapper.PARSER); builtInMetadataMappers.put(IgnoredSourceFieldMapper.NAME, IgnoredSourceFieldMapper.PARSER); builtInMetadataMappers.put(NestedPathFieldMapper.NAME, NestedPathFieldMapper.PARSER); diff --git a/server/src/main/java/org/elasticsearch/indices/ShardLimitValidator.java b/server/src/main/java/org/elasticsearch/indices/ShardLimitValidator.java index 06bb1439be4e6..f58ee757cc511 100644 --- a/server/src/main/java/org/elasticsearch/indices/ShardLimitValidator.java +++ b/server/src/main/java/org/elasticsearch/indices/ShardLimitValidator.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -296,7 +297,8 @@ static String errorMessageFrom(Result result) { + result.maxShardsInCluster + "] maximum " + result.group - + " shards open"; + + " shards open; for more information, see " + + ReferenceDocs.MAX_SHARDS_PER_NODE; } /** diff --git a/server/src/main/java/org/elasticsearch/inference/ServiceSettings.java b/server/src/main/java/org/elasticsearch/inference/ServiceSettings.java index b143f74c848c1..34a58f83963ce 100644 --- a/server/src/main/java/org/elasticsearch/inference/ServiceSettings.java +++ b/server/src/main/java/org/elasticsearch/inference/ServiceSettings.java @@ -41,4 +41,12 @@ default Integer dimensions() { default DenseVectorFieldMapper.ElementType elementType() { return null; } + + /** + * The model to use in the inference endpoint (e.g. text-embedding-ada-002). Sometimes the model is not defined in the service + * settings. This can happen for external providers (e.g. hugging face, azure ai studio) where the provider requires that the model + * be chosen when initializing a deployment within their service. In this situation, return null. + * @return the model used to perform inference or null if the model is not defined + */ + String modelId(); } diff --git a/server/src/main/java/org/elasticsearch/inference/SimilarityMeasure.java b/server/src/main/java/org/elasticsearch/inference/SimilarityMeasure.java index ff9fedee02fac..058631044c8b1 100644 --- a/server/src/main/java/org/elasticsearch/inference/SimilarityMeasure.java +++ b/server/src/main/java/org/elasticsearch/inference/SimilarityMeasure.java @@ -39,8 +39,7 @@ public static SimilarityMeasure fromString(String name) { * @return the similarity that is known to the version passed in */ public static SimilarityMeasure translateSimilarity(SimilarityMeasure similarityMeasure, TransportVersion version) { - if (version.before(TransportVersions.ML_INFERENCE_L2_NORM_SIMILARITY_ADDED) - && BEFORE_L2_NORM_ENUMS.contains(similarityMeasure) == false) { + if (version.before(TransportVersions.V_8_14_0) && BEFORE_L2_NORM_ENUMS.contains(similarityMeasure) == false) { return null; } diff --git a/server/src/main/java/org/elasticsearch/ingest/EnterpriseGeoIpTask.java b/server/src/main/java/org/elasticsearch/ingest/EnterpriseGeoIpTask.java new file mode 100644 index 0000000000000..a204060ff0c7e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/ingest/EnterpriseGeoIpTask.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.persistent.PersistentTaskParams; +import org.elasticsearch.xcontent.ObjectParser; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; + +/** + * As a relatively minor hack, this class holds the string constant that defines both the id + * and the name of the task for the new ip geolocation database downloader feature. It also provides the + * PersistentTaskParams that are necessary to start the task and to run it. + *

+ * Defining this in Elasticsearch itself gives us a reasonably tidy version of things where we don't + * end up with strange inter-module dependencies. It's not ideal, but it works fine. + */ +public final class EnterpriseGeoIpTask { + + private EnterpriseGeoIpTask() { + // utility class + } + + public static final String ENTERPRISE_GEOIP_DOWNLOADER = "enterprise-geoip-downloader"; + public static final NodeFeature GEOIP_DOWNLOADER_DATABASE_CONFIGURATION = new NodeFeature("geoip.downloader.database.configuration"); + + public static class EnterpriseGeoIpTaskParams implements PersistentTaskParams { + + public static final ObjectParser PARSER = new ObjectParser<>( + ENTERPRISE_GEOIP_DOWNLOADER, + true, + EnterpriseGeoIpTaskParams::new + ); + + public EnterpriseGeoIpTaskParams() {} + + public EnterpriseGeoIpTaskParams(StreamInput in) {} + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.endObject(); + return builder; + } + + @Override + public String getWriteableName() { + return ENTERPRISE_GEOIP_DOWNLOADER; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER; + } + + @Override + public void writeTo(StreamOutput out) {} + + public static EnterpriseGeoIpTaskParams fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public int hashCode() { + return 0; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof EnterpriseGeoIpTaskParams; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java b/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java new file mode 100644 index 0000000000000..0d989ad9f7ab2 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest; + +import org.elasticsearch.features.FeatureSpecification; +import org.elasticsearch.features.NodeFeature; + +import java.util.Set; + +import static org.elasticsearch.ingest.EnterpriseGeoIpTask.GEOIP_DOWNLOADER_DATABASE_CONFIGURATION; + +public class IngestGeoIpFeatures implements FeatureSpecification { + public Set getFeatures() { + return Set.of(GEOIP_DOWNLOADER_DATABASE_CONFIGURATION); + } +} diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 0a60886797813..357fea343ea55 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -745,7 +745,7 @@ protected void doRun() { } final int slot = i; final Releasable ref = refs.acquire(); - final DocumentSizeObserver documentSizeObserver = documentParsingProvider.newDocumentSizeObserver(); + final DocumentSizeObserver documentSizeObserver = documentParsingProvider.newDocumentSizeObserver(indexRequest); final IngestDocument ingestDocument = newIngestDocument(indexRequest, documentSizeObserver); final org.elasticsearch.script.Metadata originalDocumentMetadata = ingestDocument.getMetadata().clone(); // the document listener gives us three-way logic: a document can fail processing (1), or it can diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 11eb8760b2dbb..56f2c6c8d8c57 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.version.CompatibilityVersions; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.StopWatch; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.component.LifecycleComponent; @@ -396,7 +397,12 @@ public void onClusterServiceClose() { @Override public void onTimeout(TimeValue timeout) { - logger.warn("timed out while waiting for initial discovery state - timeout: {}", initialStateTimeout); + logger.warn( + "timed out after [{}={}] while waiting for initial discovery state; for troubleshooting guidance see [{}]", + INITIAL_STATE_TIMEOUT_SETTING.getKey(), + initialStateTimeout, + ReferenceDocs.DISCOVERY_TROUBLESHOOTING + ); latch.countDown(); } }, state -> state.nodes().getMasterNodeId() != null, initialStateTimeout); @@ -404,6 +410,7 @@ public void onTimeout(TimeValue timeout) { try { latch.await(); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new ElasticsearchTimeoutException("Interrupted while waiting for initial discovery state"); } } diff --git a/server/src/main/java/org/elasticsearch/node/NodeClosedException.java b/server/src/main/java/org/elasticsearch/node/NodeClosedException.java index 4a99c9be7b78a..d2e0f71426df0 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeClosedException.java +++ b/server/src/main/java/org/elasticsearch/node/NodeClosedException.java @@ -28,4 +28,9 @@ public NodeClosedException(DiscoveryNode node) { public NodeClosedException(StreamInput in) throws IOException { super(in); } + + @Override + public Throwable fillInStackTrace() { + return this; // this exception doesn't imply a bug, no need for a stack trace + } } diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index aa0f9b8552d22..83e23d9a60b58 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -40,6 +40,7 @@ import org.elasticsearch.cluster.coordination.Coordinator; import org.elasticsearch.cluster.coordination.MasterHistoryService; import org.elasticsearch.cluster.coordination.StableMasterHealthIndicatorService; +import org.elasticsearch.cluster.features.NodeFeaturesFixupListener; import org.elasticsearch.cluster.metadata.DataStreamFactoryRetention; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver; import org.elasticsearch.cluster.metadata.IndexMetadataVerifier; @@ -752,6 +753,7 @@ private void construct( clusterService.addListener( new TransportVersionsFixupListener(clusterService, client.admin().cluster(), featureService, threadPool) ); + clusterService.addListener(new NodeFeaturesFixupListener(clusterService, client.admin().cluster(), threadPool)); } SourceFieldMetrics sourceFieldMetrics = new SourceFieldMetrics( diff --git a/server/src/main/java/org/elasticsearch/plugins/internal/DocumentParsingProvider.java b/server/src/main/java/org/elasticsearch/plugins/internal/DocumentParsingProvider.java index 6fe1e48b25272..d29b893447be0 100644 --- a/server/src/main/java/org/elasticsearch/plugins/internal/DocumentParsingProvider.java +++ b/server/src/main/java/org/elasticsearch/plugins/internal/DocumentParsingProvider.java @@ -8,6 +8,7 @@ package org.elasticsearch.plugins.internal; +import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.index.mapper.MapperService; /** @@ -17,20 +18,6 @@ public interface DocumentParsingProvider { DocumentParsingProvider EMPTY_INSTANCE = new DocumentParsingProvider() { }; - /** - * @return a new 'empty' observer to use when observing parsing - */ - default DocumentSizeObserver newDocumentSizeObserver() { - return DocumentSizeObserver.EMPTY_INSTANCE; - } - - /** - * @return an observer with a previously observed value (fixed to this value, not continuing) - */ - default DocumentSizeObserver newFixedSizeDocumentObserver(long normalisedBytesParsed) { - return DocumentSizeObserver.EMPTY_INSTANCE; - } - /** * @return an instance of a reporter to use when parsing has been completed and indexing successful */ @@ -49,4 +36,10 @@ default DocumentSizeAccumulator createDocumentSizeAccumulator() { return DocumentSizeAccumulator.EMPTY_INSTANCE; } + /** + * @return an observer + */ + default DocumentSizeObserver newDocumentSizeObserver(DocWriteRequest request) { + return DocumentSizeObserver.EMPTY_INSTANCE; + } } diff --git a/server/src/main/java/org/elasticsearch/plugins/internal/DocumentSizeObserver.java b/server/src/main/java/org/elasticsearch/plugins/internal/DocumentSizeObserver.java index 149b89844a74c..386a90b65b60f 100644 --- a/server/src/main/java/org/elasticsearch/plugins/internal/DocumentSizeObserver.java +++ b/server/src/main/java/org/elasticsearch/plugins/internal/DocumentSizeObserver.java @@ -28,6 +28,7 @@ public XContentParser wrapParser(XContentParser xContentParser) { public long normalisedBytesParsed() { return 0; } + }; /** @@ -40,7 +41,17 @@ public long normalisedBytesParsed() { /** * Returns the state gathered during parsing + * * @return a number representing a state parsed */ long normalisedBytesParsed(); + + /** + * Indicates if an observer was used on an update request with script + * + * @return true if update was done by script, false otherwise + */ + default boolean isUpdateByScript() { + return false; + } } diff --git a/server/src/main/java/org/elasticsearch/repositories/FinalizeSnapshotContext.java b/server/src/main/java/org/elasticsearch/repositories/FinalizeSnapshotContext.java index b459e1cfc7338..0e38c5722c116 100644 --- a/server/src/main/java/org/elasticsearch/repositories/FinalizeSnapshotContext.java +++ b/server/src/main/java/org/elasticsearch/repositories/FinalizeSnapshotContext.java @@ -99,8 +99,13 @@ public Map> obsoleteShardGenerations() { return obsoleteGenerations.get(); } + /** + * Returns a new {@link ClusterState}, based on the given {@code state} with the create-snapshot entry removed. + */ public ClusterState updatedClusterState(ClusterState state) { final ClusterState updatedState = SnapshotsService.stateWithoutSnapshot(state, snapshotInfo.snapshot(), updatedShardGenerations); + // Now that the updated cluster state may have changed in-progress shard snapshots' shard generations to the latest shard + // generation, let's mark any now unreferenced shard generations as obsolete and ready to be deleted. obsoleteGenerations.set( SnapshotsInProgress.get(updatedState).obsoleteGenerations(snapshotInfo.repository(), SnapshotsInProgress.get(state)) ); diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index a90b0a217285c..06a53053bca88 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -133,8 +133,8 @@ public void onFailure(Exception e) { IndexMetadata getSnapshotIndexMetaData(RepositoryData repositoryData, SnapshotId snapshotId, IndexId index) throws IOException; /** - * Returns a {@link RepositoryData} to describe the data in the repository, including the snapshots and the indices across all snapshots - * found in the repository. Completes the listener with a {@link RepositoryException} if there was an error in reading the data. + * Fetches the {@link RepositoryData} and passes it into the listener. May completes the listener with a {@link RepositoryException} if + * there is an error in reading the repository data. * * @param responseExecutor Executor to use to complete the listener if not using the calling thread. Using {@link * org.elasticsearch.common.util.concurrent.EsExecutors#DIRECT_EXECUTOR_SERVICE} means to complete the listener diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java index 17ac4ef38f1b6..c6494eca9823b 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java @@ -47,8 +47,7 @@ import java.util.stream.Collectors; /** - * A class that represents the data in a repository, as captured in the - * repository's index blob. + * Represents the data in a repository: the snapshots and the indices across all snapshots found in the repository. */ public final class RepositoryData { diff --git a/server/src/main/java/org/elasticsearch/repositories/ShardGeneration.java b/server/src/main/java/org/elasticsearch/repositories/ShardGeneration.java index 5bdd68b14762e..275bbdb3da45d 100644 --- a/server/src/main/java/org/elasticsearch/repositories/ShardGeneration.java +++ b/server/src/main/java/org/elasticsearch/repositories/ShardGeneration.java @@ -76,9 +76,9 @@ public void writeTo(StreamOutput out) throws IOException { } /** - * Convert to a {@link String} for use in naming the {@code index-$SHARD_GEN} blob containing a {@link BlobStoreIndexShardSnapshots}. + * For use in naming the {@code index-$SHARD_GEN} blob containing a {@link BlobStoreIndexShardSnapshots}. */ - public String toBlobNamePart() { + public String getGenerationUUID() { return rawGeneration; } diff --git a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryCoordinationAction.java b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryCoordinationAction.java index b892ff93c7a9c..8d15510c308e2 100644 --- a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryCoordinationAction.java +++ b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryCoordinationAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.repositories.VerifyNodeRepositoryAction.Request; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportException; @@ -68,7 +69,7 @@ public LocalAction( ClusterService clusterService, NodeClient client ) { - super(NAME, actionFilters, transportService.getTaskManager()); + super(NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.transportService = transportService; this.clusterService = clusterService; this.client = client; diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 5b7a11969973d..96fcf0512cbff 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -501,7 +501,9 @@ protected void doClose() { @Override public void awaitIdle() { assert lifecycle.closed(); - PlainActionFuture.get(closedAndIdleListeners::addListener); + final var future = new PlainActionFuture(); + closedAndIdleListeners.addListener(future); + future.actionGet(); // wait for as long as it takes } @SuppressForbidden(reason = "legacy usage of unbatched task") // TODO add support for batching here @@ -596,7 +598,7 @@ public void cloneShardSnapshot( INDEX_SHARD_SNAPSHOTS_FORMAT.write( existingSnapshots.withClone(source.getName(), target.getName()), shardContainer, - newGen.toBlobNamePart(), + newGen.getGenerationUUID(), compress ); return new ShardSnapshotResult( @@ -1305,7 +1307,7 @@ private void deleteFromShardSnapshotMeta(BlobStoreIndexShardSnapshots updatedSna INDEX_SHARD_SNAPSHOTS_FORMAT.write( updatedSnapshots, shardContainer, - writtenGeneration.toBlobNamePart(), + writtenGeneration.getGenerationUUID(), compress ); } else { @@ -1328,7 +1330,7 @@ private void deleteFromShardSnapshotMeta(BlobStoreIndexShardSnapshots updatedSna "Failed to finalize snapshot deletion " + snapshotIds + " with shard index [" - + INDEX_SHARD_SNAPSHOTS_FORMAT.blobName(writtenGeneration.toBlobNamePart()) + + INDEX_SHARD_SNAPSHOTS_FORMAT.blobName(writtenGeneration.getGenerationUUID()) + "]", e ); @@ -1874,7 +1876,7 @@ private void cleanupOldMetadata( (indexId, gens) -> gens.forEach( (shardId, oldGen) -> toDelete.add( shardPath(indexId, shardId).buildAsString().substring(prefixPathLen) + INDEX_FILE_PREFIX + oldGen - .toBlobNamePart() + .getGenerationUUID() ) ) ); @@ -1935,7 +1937,7 @@ public void getSnapshotInfo( } /** - * Tries to poll a {@link SnapshotId} to load {@link SnapshotInfo} for from the given {@code queue}. + * Tries to poll a {@link SnapshotId} to load {@link SnapshotInfo} from the given {@code queue}. */ private void getOneSnapshotInfo(BlockingQueue queue, GetSnapshotInfoContext context) { final SnapshotId snapshotId = queue.poll(); @@ -3287,7 +3289,7 @@ private void doSnapshotShard(SnapshotShardContext context) { INDEX_SHARD_SNAPSHOTS_FORMAT.write( updatedBlobStoreIndexShardSnapshots, shardContainer, - indexGeneration.toBlobNamePart(), + indexGeneration.getGenerationUUID(), compress, serializationParams ); @@ -3298,7 +3300,7 @@ private void doSnapshotShard(SnapshotShardContext context) { "Failed to write shard level snapshot metadata for [" + snapshotId + "] to [" - + INDEX_SHARD_SNAPSHOTS_FORMAT.blobName(indexGeneration.toBlobNamePart()) + + INDEX_SHARD_SNAPSHOTS_FORMAT.blobName(indexGeneration.getGenerationUUID()) + "]", e ); @@ -3308,7 +3310,7 @@ private void doSnapshotShard(SnapshotShardContext context) { // When not using shard generations we can only write the index-${N} blob after all other work for this shard has // completed. // Also, in case of numeric shard generations the data node has to take care of deleting old shard generations. - final long newGen = Long.parseLong(fileListGeneration.toBlobNamePart()) + 1; + final long newGen = Long.parseLong(fileListGeneration.getGenerationUUID()) + 1; indexGeneration = new ShardGeneration(newGen); // Delete all previous index-N blobs final List blobsToDelete = blobs.stream().filter(blob -> blob.startsWith(SNAPSHOT_INDEX_PREFIX)).toList(); @@ -3336,7 +3338,7 @@ private void doSnapshotShard(SnapshotShardContext context) { "Failed to finalize snapshot creation [" + snapshotId + "] with shard index [" - + INDEX_SHARD_SNAPSHOTS_FORMAT.blobName(indexGeneration.toBlobNamePart()) + + INDEX_SHARD_SNAPSHOTS_FORMAT.blobName(indexGeneration.getGenerationUUID()) + "]", e ); @@ -3824,7 +3826,7 @@ private Tuple buildBlobStoreIndex return new Tuple<>(BlobStoreIndexShardSnapshots.EMPTY, ShardGenerations.NEW_SHARD_GEN); } return new Tuple<>( - INDEX_SHARD_SNAPSHOTS_FORMAT.read(metadata.name(), shardContainer, generation.toBlobNamePart(), namedXContentRegistry), + INDEX_SHARD_SNAPSHOTS_FORMAT.read(metadata.name(), shardContainer, generation.getGenerationUUID(), namedXContentRegistry), generation ); } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/GetSnapshotInfoContext.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/GetSnapshotInfoContext.java index 96782bca31a15..3338a3c2e2a76 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/GetSnapshotInfoContext.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/GetSnapshotInfoContext.java @@ -22,7 +22,7 @@ import java.util.function.BooleanSupplier; /** - * Describes the context of fetching one or more {@link SnapshotInfo} via {@link Repository#getSnapshotInfo}. + * A context through which a consumer can act on one or more {@link SnapshotInfo} via {@link Repository#getSnapshotInfo}. */ final class GetSnapshotInfoContext implements ActionListener { diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/package-info.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/package-info.java index 48d8a0730f48c..5bc09e4653d16 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/package-info.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/package-info.java @@ -59,8 +59,8 @@ * | | |- snap-20131011.dat - SMILE serialized {@link org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot} for * | | | snapshot "20131011" * | | |- index-123 - SMILE serialized {@link org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshots} for - * | | | the shard (files with numeric suffixes were created by older versions, newer ES versions use a uuid - * | | | suffix instead) + * | | | the shard. The suffix is the {@link org.elasticsearch.repositories.ShardGeneration } (files with + * | | | numeric suffixes were created by older versions, newer ES versions use a uuid suffix instead) * | | * | |- 1/ - data for shard "1" of index "foo" * | | |- __1 @@ -158,20 +158,23 @@ * *

    *
  1. Create the {@link org.apache.lucene.index.IndexCommit} for the shard to snapshot.
    2. - *
  2. Get the {@link org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshots} blob - * with name {@code index-${uuid}} with the {@code uuid} generation returned by - * {@link org.elasticsearch.repositories.ShardGenerations#getShardGen} to get the information of what segment files are - * already available in the blobstore.
    3. - *
  3. By comparing the files in the {@code IndexCommit} and the available file list from the previous step, determine the segment files - * that need to be written to the blob store. For each segment that needs to be added to the blob store, generate a unique name by combining - * the segment data blob prefix {@code __} and a UUID and write the segment to the blobstore.
    4. - *
  4. After completing all segment writes, a blob containing a + *
  5. Get the current {@link org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshots} blob file with name + * {@code index-${uuid}} by loading the index shard's generation {@code uuid} from {@link org.elasticsearch.repositories.ShardGenerations} + * (via {@link org.elasticsearch.repositories.ShardGenerations#getShardGen}). This blob file will list what segment files are already + * available in the blobstore.
    6. + *
  6. By comparing the files in the {@code IndexCommit} and the available file list from the previous step's blob file, determine the new + * segment files that need to be written to the blob store. For each segment that needs to be added to the blob store, generate a unique + * name by combining the segment data blob prefix {@code __} and a new UUID and write the segment to the blobstore.
    7. + *
  7. After completing all segment writes, a new blob file containing the new shard snapshot's * {@link org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot} with name {@code snap-${snapshot-uuid}.dat} is written to * the shard's path and contains a list of all the files referenced by the snapshot as well as some metadata about the snapshot. See the * documentation of {@code BlobStoreIndexShardSnapshot} for details on its contents.
    8. *
  8. Once all the segments and the {@code BlobStoreIndexShardSnapshot} blob have been written, an updated * {@code BlobStoreIndexShardSnapshots} blob is written to the shard's path with name {@code index-${newUUID}}.
    9. *
+ * At this point, all of the necessary shard data and shard metadata for the new shard snapshot have been written to the repository, but the + * metadata outside of the shard directory has not been updated to point to the new shard snapshot as the latest. The next finalization step + * will handle updates external to the index shard directory, and add references in the root directory. * *

Finalizing the Snapshot

* @@ -180,10 +183,10 @@ * following actions in order:

*
    *
  1. Write a blob containing the cluster metadata to the root of the blob store repository at {@code /meta-${snapshot-uuid}.dat}
    2. - *
  2. Write the metadata for each index to a blob in that index's directory at + *
  3. Write the metadata for the index to a blob in that index's directory at * {@code /indices/${index-snapshot-uuid}/meta-${snapshot-uuid}.dat}
    4. - *
  4. Write the {@link org.elasticsearch.snapshots.SnapshotInfo} blob for the given snapshot to the key {@code /snap-${snapshot-uuid}.dat} - * directly under the repository root.
    5. + *
  5. Write the {@link org.elasticsearch.snapshots.SnapshotInfo} blob for the given snapshot in a new blob file + * {@code /snap-${snapshot-uuid}.dat} directly under the repository root.
    6. *
  6. Write an updated {@code RepositoryData} blob containing the new snapshot.
    7. *
* diff --git a/server/src/main/java/org/elasticsearch/reservedstate/TransformState.java b/server/src/main/java/org/elasticsearch/reservedstate/TransformState.java index 05c3d2be5d174..a958b0ea1e29d 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/TransformState.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/TransformState.java @@ -8,24 +8,13 @@ package org.elasticsearch.reservedstate; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterState; import java.util.Set; -import java.util.function.Consumer; /** * A {@link ClusterState} wrapper used by the ReservedClusterStateService to pass the * current state as well as previous keys set by an {@link ReservedClusterStateHandler} to each transform * step of the cluster state update. - * - * Each {@link ReservedClusterStateHandler} can also provide a non cluster state transform consumer that should run after - * the cluster state is fully validated. This allows for handlers to perform extra steps, like clearing caches or saving - * other state outside the cluster state. The consumer, if provided, must return a {@link NonStateTransformResult} with - * the keys that will be saved as reserved in the cluster state. */ -public record TransformState(ClusterState state, Set keys, Consumer> nonStateTransform) { - public TransformState(ClusterState state, Set keys) { - this(state, keys, null); - } -} +public record TransformState(ClusterState state, Set keys) {} diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java index a281db9f02383..20a115a484ab5 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java @@ -13,7 +13,6 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.support.RefCountingListener; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata; import org.elasticsearch.cluster.metadata.ReservedStateMetadata; @@ -22,7 +21,6 @@ import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.common.Priority; import org.elasticsearch.core.Tuple; -import org.elasticsearch.reservedstate.NonStateTransformResult; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.reservedstate.TransformState; import org.elasticsearch.xcontent.ConstructingObjectParser; @@ -30,8 +28,6 @@ import org.elasticsearch.xcontent.XContentParser; import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; @@ -156,7 +152,6 @@ public void initEmpty(String namespace, ActionListener lis new ReservedStateUpdateTask( namespace, emptyState, - List.of(), Map.of(), List.of(), // error state should not be possible since there is no metadata being parsed or processed @@ -210,65 +205,44 @@ public void process(String namespace, ReservedStateChunk reservedStateChunk, Con return; } - // We trial run all handler validations to ensure that we can process all of the cluster state error free. During - // the trial run we collect 'consumers' (functions) for any non cluster state transforms that need to run. - var trialRunResult = trialRun(namespace, state, reservedStateChunk, orderedHandlers); + // We trial run all handler validations to ensure that we can process all of the cluster state error free. + var trialRunErrors = trialRun(namespace, state, reservedStateChunk, orderedHandlers); // this is not using the modified trial state above, but that doesn't matter, we're just setting errors here - var error = checkAndReportError(namespace, trialRunResult.errors, reservedStateVersion); + var error = checkAndReportError(namespace, trialRunErrors, reservedStateVersion); if (error != null) { errorListener.accept(error); return; } - - // Since we have validated that the cluster state update can be correctly performed in the trial run, we now - // execute the non cluster state transforms. These are assumed to be async and we continue with the cluster state update - // after all have completed. This part of reserved cluster state update is non-atomic, some or all of the non-state - // transformations can succeed, and we can fail to eventually write the reserved cluster state. - executeNonStateTransformationSteps(trialRunResult.nonStateTransforms, new ActionListener<>() { - @Override - public void onResponse(Collection nonStateTransformResults) { - // Once all of the non-state transformation results complete, we can proceed to - // do the final save of the cluster state. The non-state transformation reserved keys are applied - // to the reserved state after all other key handlers. - updateTaskQueue.submitTask( - "reserved cluster state [" + namespace + "]", - new ReservedStateUpdateTask( - namespace, - reservedStateChunk, - nonStateTransformResults, - handlers, - orderedHandlers, - ReservedClusterStateService.this::updateErrorState, - new ActionListener<>() { - @Override - public void onResponse(ActionResponse.Empty empty) { - logger.info("Successfully applied new reserved cluster state for namespace [{}]", namespace); - errorListener.accept(null); - } - - @Override - public void onFailure(Exception e) { - // Don't spam the logs on repeated errors - if (isNewError(existingMetadata, reservedStateVersion.version())) { - logger.debug("Failed to apply reserved cluster state", e); - errorListener.accept(e); - } else { - errorListener.accept(null); - } - } + updateTaskQueue.submitTask( + "reserved cluster state [" + namespace + "]", + new ReservedStateUpdateTask( + namespace, + reservedStateChunk, + handlers, + orderedHandlers, + ReservedClusterStateService.this::updateErrorState, + new ActionListener<>() { + @Override + public void onResponse(ActionResponse.Empty empty) { + logger.info("Successfully applied new reserved cluster state for namespace [{}]", namespace); + errorListener.accept(null); + } + + @Override + public void onFailure(Exception e) { + // Don't spam the logs on repeated errors + if (isNewError(existingMetadata, reservedStateVersion.version())) { + logger.debug("Failed to apply reserved cluster state", e); + errorListener.accept(e); + } else { + errorListener.accept(null); } - ), - null - ); - } - - @Override - public void onFailure(Exception e) { - // If we encounter an error while runnin the non-state transforms, we avoid saving any cluster state. - errorListener.accept(checkAndReportError(namespace, List.of(stackTrace(e)), reservedStateVersion)); - } - }); + } + } + ), + null + ); } // package private for testing @@ -324,14 +298,13 @@ public void onFailure(Exception e) { /** * Goes through all of the handlers, runs the validation and the transform part of the cluster state. *

- * While running the handlers we also collect any non cluster state transformation consumer actions that - * need to be performed asynchronously before we attempt to save the cluster state. The trial run does not - * result in an update of the cluster state, it's only purpose is to verify if we can correctly perform a - * cluster state update with the given reserved state chunk. + * The trial run does not result in an update of the cluster state, it's only purpose is to verify + * if we can correctly perform a cluster state update with the given reserved state chunk. * * Package private for testing + * @return Any errors that occured */ - TrialRunResult trialRun( + List trialRun( String namespace, ClusterState currentState, ReservedStateChunk stateChunk, @@ -341,7 +314,6 @@ TrialRunResult trialRun( Map reservedState = stateChunk.state(); List errors = new ArrayList<>(); - List>> nonStateTransforms = new ArrayList<>(); ClusterState state = currentState; @@ -351,39 +323,12 @@ TrialRunResult trialRun( Set existingKeys = keysForHandler(existingMetadata, handlerName); TransformState transformState = handler.transform(reservedState.get(handlerName), new TransformState(state, existingKeys)); state = transformState.state(); - if (transformState.nonStateTransform() != null) { - nonStateTransforms.add(transformState.nonStateTransform()); - } } catch (Exception e) { errors.add(format("Error processing %s state change: %s", handler.name(), stackTrace(e))); } } - return new TrialRunResult(nonStateTransforms, errors); - } - - /** - * Runs the non cluster state transformations asynchronously, collecting the {@link NonStateTransformResult} objects. - *

- * Once all non cluster state transformations have completed, we submit the cluster state update task, which - * updates all of the handler state, including the keys produced by the non cluster state transforms. The new reserved - * state version isn't written to the cluster state until the cluster state task runs. - * - * Package private for testing - */ - static void executeNonStateTransformationSteps( - List>> nonStateTransforms, - ActionListener> listener - ) { - final List result = Collections.synchronizedList(new ArrayList<>(nonStateTransforms.size())); - try (var listeners = new RefCountingListener(listener.map(ignored -> result))) { - for (var transform : nonStateTransforms) { - // non cluster state transforms don't modify the cluster state, they however are given a chance to return a more - // up-to-date version of the modified keys we should save in the reserved state. These calls are - // async and report back when they are done through the postTasksListener. - transform.accept(listeners.acquire(result::add)); - } - } + return errors; } /** @@ -449,9 +394,4 @@ private void addStateHandler(String key, Set keys, LinkedHashSet public void installStateHandler(ReservedClusterStateHandler handler) { this.handlers.put(handler.name(), handler); } - - /** - * Helper record class to combine the result of a trial run, non cluster state actions and any errors - */ - record TrialRunResult(List>> nonStateTransforms, List errors) {} } diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTask.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTask.java index 1ac42a91736c3..93d3619889a48 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTask.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTask.java @@ -20,7 +20,6 @@ import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata; import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.gateway.GatewayService; -import org.elasticsearch.reservedstate.NonStateTransformResult; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.reservedstate.TransformState; @@ -51,12 +50,10 @@ public class ReservedStateUpdateTask implements ClusterStateTaskListener { private final Collection orderedHandlers; private final Consumer errorReporter; private final ActionListener listener; - private final Collection nonStateTransformResults; public ReservedStateUpdateTask( String namespace, ReservedStateChunk stateChunk, - Collection nonStateTransformResults, Map> handlers, Collection orderedHandlers, Consumer errorReporter, @@ -64,7 +61,6 @@ public ReservedStateUpdateTask( ) { this.namespace = namespace; this.stateChunk = stateChunk; - this.nonStateTransformResults = nonStateTransformResults; this.handlers = handlers; this.orderedHandlers = orderedHandlers; this.errorReporter = errorReporter; @@ -115,12 +111,6 @@ protected ClusterState execute(final ClusterState currentState) { checkAndThrowOnError(errors, reservedStateVersion); - // Once we have set all of the handler state from the cluster state update tasks, we add the reserved keys - // from the non cluster state transforms. - for (var transform : nonStateTransformResults) { - reservedMetadataBuilder.putHandler(new ReservedStateHandlerMetadata(transform.handlerName(), transform.updatedKeys())); - } - // Remove the last error if we had previously encountered any in prior processing of reserved state reservedMetadataBuilder.errorMetadata(null); diff --git a/server/src/main/java/org/elasticsearch/rest/RestUtils.java b/server/src/main/java/org/elasticsearch/rest/RestUtils.java index e4dd38b9bc688..0e7200fa83b1c 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestUtils.java +++ b/server/src/main/java/org/elasticsearch/rest/RestUtils.java @@ -146,17 +146,17 @@ private static String decodeComponent(final String s, final Charset charset, boo } private static boolean decodingNeeded(String s, int size, boolean plusAsSpace) { - boolean decodingNeeded = false; - for (int i = 0; i < size; i++) { + if (Strings.isEmpty(s)) { + return false; + } + int num = Math.min(s.length(), size); + for (int i = 0; i < num; i++) { final char c = s.charAt(i); - if (c == '%') { - i++; // We can skip at least one char, e.g. `%%'. - decodingNeeded = true; - } else if (plusAsSpace && c == '+') { - decodingNeeded = true; + if (c == '%' || (plusAsSpace && c == '+')) { + return true; } } - return decodingNeeded; + return false; } @SuppressWarnings("fallthrough") diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java index 2cb9acb50dc47..cf883046d1e14 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java @@ -10,7 +10,7 @@ import org.elasticsearch.action.NodeStatsLevel; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag; import org.elasticsearch.client.internal.node.NodeClient; @@ -54,17 +54,6 @@ public List routes() { ); } - static final Map> METRICS; - - static { - Map> map = new HashMap<>(); - for (NodesStatsRequestParameters.Metric metric : NodesStatsRequestParameters.Metric.values()) { - map.put(metric.metricName(), request -> request.addMetric(metric.metricName())); - } - map.put("indices", request -> request.indices(true)); - METRICS = Collections.unmodifiableMap(map); - } - static final Map> FLAGS; static { @@ -88,14 +77,14 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC } String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")); - Set metrics = Strings.tokenizeByCommaToSet(request.param("metric", "_all")); + Set metricNames = Strings.tokenizeByCommaToSet(request.param("metric", "_all")); NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(nodesIds); nodesStatsRequest.timeout(getTimeout(request)); // level parameter validation nodesStatsRequest.setIncludeShardsStats(NodeStatsLevel.of(request, NodeStatsLevel.NODE) != NodeStatsLevel.NODE); - if (metrics.size() == 1 && metrics.contains("_all")) { + if (metricNames.size() == 1 && metricNames.contains("_all")) { if (request.hasParam("index_metric")) { throw new IllegalArgumentException( String.format( @@ -108,7 +97,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC } nodesStatsRequest.all(); nodesStatsRequest.indices(CommonStatsFlags.ALL); - } else if (metrics.contains("_all")) { + } else if (metricNames.contains("_all")) { throw new IllegalArgumentException( String.format( Locale.ROOT, @@ -122,21 +111,22 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC // use a sorted set so the unrecognized parameters appear in a reliable sorted order final Set invalidMetrics = new TreeSet<>(); - for (final String metric : metrics) { - final Consumer handler = METRICS.get(metric); - if (handler != null) { - handler.accept(nodesStatsRequest); + for (final String metricName : metricNames) { + if (Metric.isValid(metricName)) { + nodesStatsRequest.addMetric(Metric.get(metricName)); } else { - invalidMetrics.add(metric); + if (metricName.equals("indices")) continue; // indices metric has different implications, see below + invalidMetrics.add(metricName); } } if (invalidMetrics.isEmpty() == false) { - throw new IllegalArgumentException(unrecognized(request, invalidMetrics, METRICS.keySet(), "metric")); + throw new IllegalArgumentException(unrecognized(request, invalidMetrics, Metric.ALL_NAMES, "metric")); } // check for index specific metrics - if (metrics.contains("indices")) { + if (metricNames.contains("indices")) { + nodesStatsRequest.indices(true); Set indexMetrics = Strings.tokenizeByCommaToSet(request.param("index_metric", "_all")); if (indexMetrics.size() == 1 && indexMetrics.contains("_all")) { nodesStatsRequest.indices(CommonStatsFlags.ALL); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestResetFeatureStateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestResetFeatureStateAction.java index 21a8349770a45..84003f972dabf 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestResetFeatureStateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestResetFeatureStateAction.java @@ -45,17 +45,17 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { final var req = new ResetFeatureStateRequest(RestUtils.getMasterNodeTimeout(request)); - return restChannel -> client.execute(ResetFeatureStateAction.INSTANCE, req, new RestToXContentListener<>(restChannel, r -> { - long failures = r.getFeatureStateResetStatuses() - .stream() - .filter(status -> status.getStatus() == ResetFeatureStateResponse.ResetFeatureStateStatus.Status.FAILURE) - .count(); - if (failures == 0) { - return RestStatus.OK; - } else if (failures == r.getFeatureStateResetStatuses().size()) { - return RestStatus.INTERNAL_SERVER_ERROR; - } - return RestStatus.MULTI_STATUS; - })); + return restChannel -> client.execute( + ResetFeatureStateAction.INSTANCE, + req, + new RestToXContentListener<>( + restChannel, + r -> r.getFeatureStateResetStatuses() + .stream() + .anyMatch(status -> status.getStatus() == ResetFeatureStateResponse.ResetFeatureStateStatus.Status.FAILURE) + ? RestStatus.INTERNAL_SERVER_ERROR + : RestStatus.OK + ) + ); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java index 700baac09865e..218348325e0a4 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java @@ -18,7 +18,7 @@ public class CreateIndexCapabilities { /** * Support for using the 'logs' index mode. */ - private static final String LOGS_INDEX_MODE_CAPABILITY = "logs_index_mode"; + private static final String LOGSDB_INDEX_MODE_CAPABILITY = "logsdb_index_mode"; - public static Set CAPABILITIES = Set.of(LOGS_INDEX_MODE_CAPABILITY); + public static Set CAPABILITIES = Set.of(LOGSDB_INDEX_MODE_CAPABILITY); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java index 806e3939b6d1e..d185f3f921d38 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java @@ -70,8 +70,8 @@ public void processResponse(final ClusterStateResponse state) { NodesStatsRequest statsRequest = new NodesStatsRequest(nodes); statsRequest.setIncludeShardsStats(false); statsRequest.clear() - .addMetric(NodesStatsRequestParameters.Metric.FS.metricName()) - .addMetric(NodesStatsRequestParameters.Metric.ALLOCATIONS.metricName()) + .addMetric(NodesStatsRequestParameters.Metric.FS) + .addMetric(NodesStatsRequestParameters.Metric.ALLOCATIONS) .indices(new CommonStatsFlags(CommonStatsFlags.Flag.Store)); client.admin().cluster().nodesStats(statsRequest, new RestResponseListener<>(channel) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java index 2c1f57f291969..d2162544abb31 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java @@ -103,11 +103,11 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli nodesStatsRequest.clear() .indices(true) .addMetrics( - NodesStatsRequestParameters.Metric.JVM.metricName(), - NodesStatsRequestParameters.Metric.OS.metricName(), - NodesStatsRequestParameters.Metric.FS.metricName(), - NodesStatsRequestParameters.Metric.PROCESS.metricName(), - NodesStatsRequestParameters.Metric.SCRIPT.metricName() + NodesStatsRequestParameters.Metric.JVM, + NodesStatsRequestParameters.Metric.OS, + NodesStatsRequestParameters.Metric.FS, + NodesStatsRequestParameters.Metric.PROCESS, + NodesStatsRequestParameters.Metric.SCRIPT ); nodesStatsRequest.indices().includeUnloadedSegments(request.paramAsBoolean("include_unloaded_segments", false)); diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java index 260ce4a3aeb3d..95fc945226f6f 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java @@ -86,7 +86,7 @@ public void processResponse(final ClusterStateResponse clusterStateResponse) { public void processResponse(final NodesInfoResponse nodesInfoResponse) { NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(); nodesStatsRequest.setIncludeShardsStats(false); - nodesStatsRequest.clear().addMetric(NodesStatsRequestParameters.Metric.THREAD_POOL.metricName()); + nodesStatsRequest.clear().addMetric(NodesStatsRequestParameters.Metric.THREAD_POOL); client.admin().cluster().nodesStats(nodesStatsRequest, new RestResponseListener(channel) { @Override public RestResponse buildResponse(NodesStatsResponse nodesStatsResponse) throws Exception { diff --git a/server/src/main/java/org/elasticsearch/rest/action/info/RestClusterInfoAction.java b/server/src/main/java/org/elasticsearch/rest/action/info/RestClusterInfoAction.java index 0a38d59d29729..abe82aa4c91ff 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/info/RestClusterInfoAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/info/RestClusterInfoAction.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; @@ -40,6 +41,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import static java.util.stream.Collectors.toUnmodifiableSet; import static org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric.HTTP; import static org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric.INGEST; import static org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric.SCRIPT; @@ -49,29 +51,30 @@ @ServerlessScope(Scope.PUBLIC) public class RestClusterInfoAction extends BaseRestHandler { - static final Map> RESPONSE_MAPPER = Map.of( - HTTP.metricName(), + static final Map> RESPONSE_MAPPER = Map.of( + HTTP, nodesStatsResponse -> nodesStatsResponse.getNodes().stream().map(NodeStats::getHttp).reduce(HttpStats.IDENTITY, HttpStats::merge), // - INGEST.metricName(), + INGEST, nodesStatsResponse -> nodesStatsResponse.getNodes() .stream() .map(NodeStats::getIngestStats) .reduce(IngestStats.IDENTITY, IngestStats::merge), // - THREAD_POOL.metricName(), + THREAD_POOL, nodesStatsResponse -> nodesStatsResponse.getNodes() .stream() .map(NodeStats::getThreadPool) .reduce(ThreadPoolStats.IDENTITY, ThreadPoolStats::merge), // - SCRIPT.metricName(), + SCRIPT, nodesStatsResponse -> nodesStatsResponse.getNodes() .stream() .map(NodeStats::getScriptStats) .reduce(ScriptStats.IDENTITY, ScriptStats::merge) ); - static final Set AVAILABLE_TARGETS = RESPONSE_MAPPER.keySet(); + static final Set AVAILABLE_TARGETS = RESPONSE_MAPPER.keySet(); + static final Set AVAILABLE_TARGET_NAMES = AVAILABLE_TARGETS.stream().map(Metric::metricName).collect(toUnmodifiableSet()); @Override public String getName() { @@ -93,7 +96,7 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client targets.clear(); AVAILABLE_TARGETS.forEach(m -> { nodesStatsRequest.addMetric(m); - targets.add(m); + targets.add(m.metricName()); }); } else if (targets.contains("_all")) { @@ -103,14 +106,14 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client } else { var invalidTargetParams = targets.stream() - .filter(Predicate.not(AVAILABLE_TARGETS::contains)) + .filter(Predicate.not(AVAILABLE_TARGET_NAMES::contains)) .collect(Collectors.toCollection(TreeSet::new)); if (invalidTargetParams.isEmpty() == false) { - throw new IllegalArgumentException(unrecognized(request, invalidTargetParams, AVAILABLE_TARGETS, "target")); + throw new IllegalArgumentException(unrecognized(request, invalidTargetParams, AVAILABLE_TARGET_NAMES, "target")); } - targets.forEach(nodesStatsRequest::addMetric); + targets.forEach(metricName -> nodesStatsRequest.addMetric(Metric.get(metricName))); } return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).admin() @@ -118,7 +121,11 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client .nodesStats(nodesStatsRequest, new RestResponseListener<>(channel) { @Override public RestResponse buildResponse(NodesStatsResponse response) throws Exception { - var chunkedResponses = targets.stream().map(RESPONSE_MAPPER::get).map(mapper -> mapper.apply(response)).iterator(); + var chunkedResponses = targets.stream() + .map(Metric::get) + .map(RESPONSE_MAPPER::get) + .map(mapper -> mapper.apply(response)) + .iterator(); return RestResponse.chunked( RestStatus.OK, diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java index 2e2be59689b65..7431e510f4e20 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java @@ -14,10 +14,13 @@ import java.util.Map; +/** + * Container class for aggregated metrics about search responses. + */ public class SearchResponseMetrics { public enum ResponseCountTotalStatus { - SUCCESS("succes"), + SUCCESS("success"), PARTIAL_FAILURE("partial_failure"), FAILURE("failure"); @@ -32,7 +35,7 @@ public String getDisplayName() { } } - public static final String RESPONSE_COUNT_TOTAL_STATUS_ATTRIBUTE_NAME = "status"; + public static final String RESPONSE_COUNT_TOTAL_STATUS_ATTRIBUTE_NAME = "response_status"; public static final String TOOK_DURATION_TOTAL_HISTOGRAM_NAME = "es.search_response.took_durations.histogram"; public static final String RESPONSE_COUNT_TOTAL_COUNTER_NAME = "es.search_response.response_count.total"; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/MultiBucketConsumerService.java b/server/src/main/java/org/elasticsearch/search/aggregations/MultiBucketConsumerService.java index a6f634ec371b1..2519e4e263d00 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/MultiBucketConsumerService.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/MultiBucketConsumerService.java @@ -65,6 +65,11 @@ public TooManyBucketsException(StreamInput in) throws IOException { maxBuckets = in.readInt(); } + @Override + public Throwable fillInStackTrace() { + return this; // this exception doesn't imply a bug, no need for a stack trace + } + @Override protected void writeTo(StreamOutput out, Writer nestedExceptionsWriter) throws IOException { super.writeTo(out, nestedExceptionsWriter); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java index 4cfa7f449cf57..c0c7f68802762 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java @@ -238,7 +238,7 @@ public InternalDateHistogram(StreamInput in) throws IOException { } buckets = in.readCollectionAsList(stream -> Bucket.readFrom(stream, keyed, format)); // we changed the order format in 8.13 for partial reduce, therefore we need to order them to perform merge sort - if (in.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.HISTOGRAM_AGGS_KEY_SORTED)) { + if (in.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.V_8_14_0)) { // list is mutable by #readCollectionAsList contract buckets.sort(Comparator.comparingLong(b -> b.key)); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java index 2404de76fdd35..b09c84a80ac2c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java @@ -221,7 +221,7 @@ public InternalHistogram(StreamInput in) throws IOException { keyed = in.readBoolean(); buckets = in.readCollectionAsList(stream -> Bucket.readFrom(stream, keyed, format)); // we changed the order format in 8.13 for partial reduce, therefore we need to order them to perform merge sort - if (in.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.HISTOGRAM_AGGS_KEY_SORTED)) { + if (in.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.V_8_14_0)) { // list is mutable by #readCollectionAsList contract buckets.sort(Comparator.comparingDouble(b -> b.key)); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogram.java index 675b5d218c882..01a178e83db77 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogram.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogram.java @@ -246,7 +246,7 @@ public InternalVariableWidthHistogram(StreamInput in) throws IOException { buckets = in.readCollectionAsList(stream -> Bucket.readFrom(stream, format)); targetNumBuckets = in.readVInt(); // we changed the order format in 8.13 for partial reduce, therefore we need to order them to perform merge sort - if (in.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.HISTOGRAM_AGGS_KEY_SORTED)) { + if (in.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.V_8_14_0)) { // list is mutable by #readCollectionAsList contract buckets.sort(Comparator.comparingDouble(b -> b.centroid)); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java index 68a1a22369d2a..9fb94f77be020 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java @@ -55,7 +55,7 @@ public InternalRandomSampler(StreamInput in) throws IOException { super(in); this.seed = in.readInt(); this.probability = in.readDouble(); - if (in.getTransportVersion().onOrAfter(TransportVersions.RANDOM_AGG_SHARD_SEED)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { this.shardSeed = in.readOptionalInt(); } else { this.shardSeed = null; @@ -67,7 +67,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { super.doWriteTo(out); out.writeInt(seed); out.writeDouble(probability); - if (out.getTransportVersion().onOrAfter(TransportVersions.RANDOM_AGG_SHARD_SEED)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalInt(shardSeed); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregationBuilder.java index 9bd9ab45b633a..14aef1187b99b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregationBuilder.java @@ -79,7 +79,7 @@ public RandomSamplerAggregationBuilder(StreamInput in) throws IOException { super(in); this.p = in.readDouble(); this.seed = in.readInt(); - if (in.getTransportVersion().onOrAfter(TransportVersions.RANDOM_AGG_SHARD_SEED)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { this.shardSeed = in.readOptionalInt(); } } @@ -99,7 +99,7 @@ protected RandomSamplerAggregationBuilder( protected void doWriteTo(StreamOutput out) throws IOException { out.writeDouble(p); out.writeInt(seed); - if (out.getTransportVersion().onOrAfter(TransportVersions.RANDOM_AGG_SHARD_SEED)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalInt(shardSeed); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/MapStringTermsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/MapStringTermsAggregator.java index 9cea884667325..936fcf2edc225 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/MapStringTermsAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/MapStringTermsAggregator.java @@ -9,7 +9,10 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.PriorityQueue; @@ -419,25 +422,66 @@ void collectZeroDocEntriesIfNeeded(long owningBucketOrd, boolean excludeDeletedD } // we need to fill-in the blanks for (LeafReaderContext ctx : searcher().getTopReaderContext().leaves()) { - SortedBinaryDocValues values = valuesSource.bytesValues(ctx); - // brute force - for (int docId = 0; docId < ctx.reader().maxDoc(); ++docId) { - if (excludeDeletedDocs && ctx.reader().getLiveDocs() != null && ctx.reader().getLiveDocs().get(docId) == false) { - continue; + final Bits liveDocs = excludeDeletedDocs ? ctx.reader().getLiveDocs() : null; + if (liveDocs == null && valuesSource.hasOrdinals()) { + final SortedSetDocValues values = ((ValuesSource.Bytes.WithOrdinals) valuesSource).ordinalsValues(ctx); + collectZeroDocEntries(values, owningBucketOrd); + } else { + final SortedBinaryDocValues values = valuesSource.bytesValues(ctx); + final BinaryDocValues singleton = FieldData.unwrapSingleton(values); + if (singleton != null) { + collectZeroDocEntries(singleton, liveDocs, ctx.reader().maxDoc(), owningBucketOrd); + } else { + collectZeroDocEntries(values, liveDocs, ctx.reader().maxDoc(), owningBucketOrd); } - if (values.advanceExact(docId)) { - int valueCount = values.docValueCount(); - for (int i = 0; i < valueCount; ++i) { - BytesRef term = values.nextValue(); - if (includeExclude == null || includeExclude.accept(term)) { - bucketOrds.add(owningBucketOrd, term); - } + } + } + } + + private void collectZeroDocEntries(SortedSetDocValues values, long owningBucketOrd) throws IOException { + final TermsEnum termsEnum = values.termsEnum(); + BytesRef term; + while ((term = termsEnum.next()) != null) { + if (includeExclude == null || includeExclude.accept(term)) { + bucketOrds.add(owningBucketOrd, term); + } + } + } + + private void collectZeroDocEntries(SortedBinaryDocValues values, Bits liveDocs, int maxDoc, long owningBucketOrd) + throws IOException { + // brute force + for (int docId = 0; docId < maxDoc; ++docId) { + if (liveDocs != null && liveDocs.get(docId) == false) { + continue; + } + if (values.advanceExact(docId)) { + final int valueCount = values.docValueCount(); + for (int i = 0; i < valueCount; ++i) { + final BytesRef term = values.nextValue(); + if (includeExclude == null || includeExclude.accept(term)) { + bucketOrds.add(owningBucketOrd, term); } } } } } + private void collectZeroDocEntries(BinaryDocValues values, Bits liveDocs, int maxDoc, long owningBucketOrd) throws IOException { + // brute force + for (int docId = 0; docId < maxDoc; ++docId) { + if (liveDocs != null && liveDocs.get(docId) == false) { + continue; + } + if (values.advanceExact(docId)) { + final BytesRef term = values.binaryValue(); + if (includeExclude == null || includeExclude.accept(term)) { + bucketOrds.add(owningBucketOrd, term); + } + } + } + } + @Override Supplier emptyBucketBuilder(long owningBucketOrd) { return () -> new StringTerms.Bucket(new BytesRef(), 0, null, showTermDocCountError, 0, format); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java index bf923339c73f5..1c7ca7441ad80 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java @@ -37,8 +37,6 @@ import java.util.Objects; import java.util.function.ToLongFunction; -import static org.elasticsearch.TransportVersions.AGGS_EXCLUDED_DELETED_DOCS; - public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder { public static final int KEY_ORDER_CONCURRENCY_THRESHOLD = 50; @@ -199,7 +197,7 @@ public TermsAggregationBuilder(StreamInput in) throws IOException { includeExclude = in.readOptionalWriteable(IncludeExclude::new); order = InternalOrder.Streams.readOrder(in); showTermDocCountError = in.readBoolean(); - if (in.getTransportVersion().onOrAfter(AGGS_EXCLUDED_DELETED_DOCS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { excludeDeletedDocs = in.readBoolean(); } } @@ -217,7 +215,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(includeExclude); order.writeTo(out); out.writeBoolean(showTermDocCountError); - if (out.getTransportVersion().onOrAfter(AGGS_EXCLUDED_DELETED_DOCS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeBoolean(excludeDeletedDocs); } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java index fe430aeec4723..7620da2c227cb 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java @@ -141,7 +141,7 @@ protected AbstractHighlighterBuilder(StreamInput in) throws IOException { postTags(in.readOptionalStringArray()); fragmentSize(in.readOptionalVInt()); numOfFragments(in.readOptionalVInt()); - if (in.getTransportVersion().onOrAfter(TransportVersions.HIGHLIGHTERS_TAGS_ON_FIELD_LEVEL)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { encoder(in.readOptionalString()); } highlighterType(in.readOptionalString()); @@ -180,7 +180,7 @@ public final void writeTo(StreamOutput out) throws IOException { out.writeOptionalStringArray(postTags); out.writeOptionalVInt(fragmentSize); out.writeOptionalVInt(numOfFragments); - if (out.getTransportVersion().onOrAfter(TransportVersions.HIGHLIGHTERS_TAGS_ON_FIELD_LEVEL)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalString(encoder); } out.writeOptionalString(highlighterType); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java index 05767d8fc7dbf..7059d54f3fa8a 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java @@ -121,7 +121,7 @@ public HighlightBuilder(HighlightBuilder template, QueryBuilder highlightQuery, */ public HighlightBuilder(StreamInput in) throws IOException { super(in); - if (in.getTransportVersion().before(TransportVersions.HIGHLIGHTERS_TAGS_ON_FIELD_LEVEL)) { + if (in.getTransportVersion().before(TransportVersions.V_8_14_0)) { encoder(in.readOptionalString()); } useExplicitFieldOrder(in.readBoolean()); @@ -131,7 +131,7 @@ public HighlightBuilder(StreamInput in) throws IOException { @Override protected void doWriteTo(StreamOutput out) throws IOException { - if (out.getTransportVersion().before(TransportVersions.HIGHLIGHTERS_TAGS_ON_FIELD_LEVEL)) { + if (out.getTransportVersion().before(TransportVersions.V_8_14_0)) { out.writeOptionalString(encoder); } out.writeBoolean(useExplicitFieldOrder); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java index e7fa0e67cb453..3d180dd094b18 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java @@ -218,6 +218,9 @@ private static int findGoodEndForNoHighlightExcerpt(int noMatchSize, Analyzer an // Can't split on term boundaries without offsets return -1; } + if (contents.length() <= noMatchSize) { + return contents.length(); + } int end = -1; tokenStream.reset(); while (tokenStream.incrementToken()) { diff --git a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index b0a3a558e2956..91aa33b24d883 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -50,6 +50,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.Locale; import java.util.Objects; import java.util.function.Function; @@ -543,10 +544,11 @@ public static boolean hasPrimaryFieldSort(SearchSourceBuilder source) { * is an instance of this class, null otherwise. */ public static FieldSortBuilder getPrimaryFieldSortOrNull(SearchSourceBuilder source) { - if (source == null || source.sorts() == null || source.sorts().isEmpty()) { + final List> sorts; + if (source == null || (sorts = source.sorts()) == null || sorts.isEmpty()) { return null; } - return source.sorts().get(0) instanceof FieldSortBuilder ? (FieldSortBuilder) source.sorts().get(0) : null; + return sorts.get(0) instanceof FieldSortBuilder fieldSortBuilder ? fieldSortBuilder : null; } /** diff --git a/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java b/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java index 7c29f52f33847..c512b6695befb 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java +++ b/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java @@ -55,16 +55,27 @@ public T getMax() { return maxValue; } + @SuppressWarnings({ "unchecked", "rawtypes" }) + private static final Comparator ASC_COMPARATOR = (left, right) -> { + if (left == null) { + return right == null ? 0 : -1; // nulls last + } + return right == null ? 1 : left.getMin().compareTo(right.getMin()); + }; + + @SuppressWarnings({ "unchecked", "rawtypes" }) + private static final Comparator DESC_COMPARATOR = (left, right) -> { + if (left == null) { + return right == null ? 0 : 1; // nulls first + } + return right == null ? -1 : right.getMax().compareTo(left.getMax()); + }; + /** * Return a {@link Comparator} for {@link MinAndMax} values according to the provided {@link SortOrder}. */ + @SuppressWarnings({ "unchecked", "rawtypes" }) public static > Comparator> getComparator(SortOrder order) { - Comparator> cmp = order == SortOrder.ASC - ? Comparator.comparing(MinAndMax::getMin) - : Comparator.comparing(MinAndMax::getMax); - if (order == SortOrder.DESC) { - cmp = cmp.reversed(); - } - return Comparator.nullsLast(cmp); + return (Comparator) (order == SortOrder.ASC ? ASC_COMPARATOR : DESC_COMPARATOR); } } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilder.java index 1f05b215699b1..4ac8d14c0b79d 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilder.java @@ -56,7 +56,7 @@ public ExactKnnQueryBuilder(VectorData query, String field) { public ExactKnnQueryBuilder(StreamInput in) throws IOException { super(in); - if (in.getTransportVersion().onOrAfter(TransportVersions.KNN_EXPLICIT_BYTE_QUERY_VECTOR_PARSING)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { this.query = in.readOptionalWriteable(VectorData::new); } else { this.query = VectorData.fromFloats(in.readFloatArray()); @@ -79,7 +79,7 @@ public String getWriteableName() { @Override protected void doWriteTo(StreamOutput out) throws IOException { - if (out.getTransportVersion().onOrAfter(TransportVersions.KNN_EXPLICIT_BYTE_QUERY_VECTOR_PARSING)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalWriteable(query); } else { out.writeFloatArray(query.asFloatVector()); diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java index 65f8c60297ad8..8c0cf44b1f181 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java @@ -38,16 +38,6 @@ public class KnnScoreDocQueryBuilder extends AbstractQueryBuilder shard : runningSnapshot.shardsByRepoShardId() + for (Map.Entry shard : runningSnapshot + .shardSnapshotStatusByRepoShardId() .entrySet()) { final RepositoryShardId sid = shard.getKey(); addStateInformation(generations, busyIds, shard.getValue(), sid.shardId(), sid.indexName()); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 1a022d08d3a24..286b08a0d3f3c 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -177,14 +177,15 @@ public SnapshotInfo( public static SnapshotInfo inProgress(SnapshotsInProgress.Entry entry) { int successfulShards = 0; List shardFailures = new ArrayList<>(); - for (Map.Entry c : entry.shardsByRepoShardId().entrySet()) { + for (Map.Entry c : entry.shardSnapshotStatusByRepoShardId() + .entrySet()) { if (c.getValue().state() == SnapshotsInProgress.ShardState.SUCCESS) { successfulShards++; } else if (c.getValue().state().failed() && c.getValue().state().completed()) { shardFailures.add(new SnapshotShardFailure(c.getValue().nodeId(), entry.shardId(c.getKey()), c.getValue().reason())); } } - int totalShards = entry.shardsByRepoShardId().size(); + int totalShards = entry.shardSnapshotStatusByRepoShardId().size(); return new SnapshotInfo( entry.snapshot(), List.copyOf(entry.indices().keySet()), diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 1529ef556037a..ef8840c90be0a 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -20,7 +20,6 @@ import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.SnapshotsInProgress.ShardSnapshotStatus; import org.elasticsearch.cluster.SnapshotsInProgress.ShardState; -import org.elasticsearch.cluster.SnapshotsInProgress.State; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -67,6 +66,7 @@ * This service runs on data nodes and controls currently running shard snapshots on these nodes. It is responsible for * starting and stopping shard level snapshots. * See package level documentation of {@link org.elasticsearch.snapshots} for details. + * See {@link SnapshotsService} for the master node snapshotting steps. */ public final class SnapshotShardsService extends AbstractLifecycleComponent implements ClusterStateListener, IndexEventListener { private static final Logger logger = LogManager.getLogger(SnapshotShardsService.class); @@ -205,6 +205,9 @@ public Map currentSnapshotShards(Snapsho } } + /** + * Cancels any snapshots that have been removed from the given list of SnapshotsInProgress. + */ private void cancelRemoved(SnapshotsInProgress snapshotsInProgress) { // First, remove snapshots that are no longer there Iterator>> it = shardSnapshots.entrySet().iterator(); @@ -250,7 +253,7 @@ private void handleUpdatedSnapshotsInProgressEntry(String localNodeId, boolean r // Abort all running shards for this snapshot final Snapshot snapshot = entry.snapshot(); Map snapshotShards = shardSnapshots.getOrDefault(snapshot, emptyMap()); - for (Map.Entry shard : entry.shardsByRepoShardId().entrySet()) { + for (Map.Entry shard : entry.shardSnapshotStatusByRepoShardId().entrySet()) { final ShardId sid = entry.shardId(shard.getKey()); final IndexShardSnapshotStatus snapshotStatus = snapshotShards.get(sid); if (snapshotStatus == null) { @@ -561,7 +564,7 @@ public static String getShardStateId(IndexShard indexShard, IndexCommit snapshot */ private void syncShardStatsOnNewMaster(List entries) { for (SnapshotsInProgress.Entry snapshot : entries) { - if (snapshot.state() == State.STARTED || snapshot.state() == State.ABORTED) { + if (snapshot.state() == SnapshotsInProgress.State.STARTED || snapshot.state() == SnapshotsInProgress.State.ABORTED) { final Map localShards; synchronized (shardSnapshots) { final var currentLocalShards = shardSnapshots.get(snapshot.snapshot()); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 9178050ff2a0b..75b5a4e6a2ea6 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -38,7 +38,6 @@ import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.SnapshotsInProgress.ShardSnapshotStatus; import org.elasticsearch.cluster.SnapshotsInProgress.ShardState; -import org.elasticsearch.cluster.SnapshotsInProgress.State; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException; import org.elasticsearch.cluster.metadata.DataStream; @@ -134,6 +133,7 @@ * Service responsible for creating snapshots. This service runs all the steps executed on the master node during snapshot creation and * deletion. * See package level documentation of {@link org.elasticsearch.snapshots} for details. + * See {@link SnapshotShardsService} for the data node snapshotting steps. */ public final class SnapshotsService extends AbstractLifecycleComponent implements ClusterStateApplier { @@ -179,7 +179,7 @@ public final class SnapshotsService extends AbstractLifecycleComponent implement // Set of snapshots that are currently being ended by this node private final Set endingSnapshots = Collections.synchronizedSet(new HashSet<>()); - // Set of currently initializing clone operations + /** Set of currently initializing clone operations */ private final Set initializingClones = Collections.synchronizedSet(new HashSet<>()); private final UpdateSnapshotStatusAction updateSnapshotStatusHandler; @@ -288,6 +288,9 @@ public void createSnapshot(final CreateSnapshotRequest request, final ActionList submitCreateSnapshotRequest(request, listener, repository, new Snapshot(repositoryName, snapshotId), repository.getMetadata()); } + /** + * Updates the cluster state with the new {@link CreateSnapshotRequest}, which triggers async snapshot creation. + */ private void submitCreateSnapshotRequest( CreateSnapshotRequest request, ActionListener listener, @@ -408,6 +411,9 @@ public void clusterStateProcessed(ClusterState oldState, final ClusterState newS }, "clone_snapshot [" + request.source() + "][" + snapshotName + ']', listener::onFailure); } + /** + * Checks the cluster state for any in-progress repository cleanup tasks ({@link RepositoryCleanupInProgress}). + */ private static void ensureNoCleanupInProgress( final ClusterState currentState, final String repositoryName, @@ -570,7 +576,8 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) if (updatedEntry != null) { final Snapshot target = updatedEntry.snapshot(); final SnapshotId sourceSnapshot = updatedEntry.source(); - for (Map.Entry indexClone : updatedEntry.shardsByRepoShardId().entrySet()) { + for (Map.Entry indexClone : updatedEntry.shardSnapshotStatusByRepoShardId() + .entrySet()) { final ShardSnapshotStatus shardStatusBefore = indexClone.getValue(); if (shardStatusBefore.state() != ShardState.INIT) { continue; @@ -579,7 +586,7 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) runReadyClone(target, sourceSnapshot, shardStatusBefore, repoShardId, repository); } } else { - // Extremely unlikely corner case of master failing over between between starting the clone and + // Extremely unlikely corner case of master failing over between starting the clone and // starting shard clones. logger.warn("Did not find expected entry [{}] in the cluster state", cloneEntry); } @@ -739,9 +746,9 @@ private static void validate(final String repositoryName, final String snapshotN private static ShardGenerations buildGenerations(SnapshotsInProgress.Entry snapshot, Metadata metadata) { ShardGenerations.Builder builder = ShardGenerations.builder(); if (snapshot.isClone()) { - snapshot.shardsByRepoShardId().forEach((key, value) -> builder.put(key.index(), key.shardId(), value)); + snapshot.shardSnapshotStatusByRepoShardId().forEach((key, value) -> builder.put(key.index(), key.shardId(), value)); } else { - snapshot.shardsByRepoShardId().forEach((key, value) -> { + snapshot.shardSnapshotStatusByRepoShardId().forEach((key, value) -> { final Index index = snapshot.indexByName(key.indexName()); if (metadata.index(index) == null) { assert snapshot.partial() : "Index [" + index + "] was deleted during a snapshot but snapshot was not partial."; @@ -936,7 +943,7 @@ private static boolean assertNoDanglingSnapshots(ClusterState state) { .collect(Collectors.toSet()); for (List repoEntry : snapshotsInProgress.entriesByRepo()) { final SnapshotsInProgress.Entry entry = repoEntry.get(0); - for (ShardSnapshotStatus value : entry.shardsByRepoShardId().values()) { + for (ShardSnapshotStatus value : entry.shardSnapshotStatusByRepoShardId().values()) { if (value.equals(ShardSnapshotStatus.UNASSIGNED_QUEUED)) { assert reposWithRunningDelete.contains(entry.repository()) : "Found shard snapshot waiting to be assigned in [" + entry + "] but it is not blocked by any running delete"; @@ -981,19 +988,18 @@ private void processExternalChanges(boolean changedNodes, boolean changedShards) @Override public ClusterState execute(ClusterState currentState) { RoutingTable routingTable = currentState.routingTable(); - final SnapshotsInProgress snapshots = SnapshotsInProgress.get(currentState); - final SnapshotDeletionsInProgress deletes = SnapshotDeletionsInProgress.get(currentState); + final SnapshotsInProgress snapshotsInProgress = SnapshotsInProgress.get(currentState); + final SnapshotDeletionsInProgress deletesInProgress = SnapshotDeletionsInProgress.get(currentState); DiscoveryNodes nodes = currentState.nodes(); - final EnumSet statesToUpdate; - // If we are reacting to a change in the cluster node configuration we have to update the shard states of both started - // and - // aborted snapshots to potentially fail shards running on the removed nodes + final EnumSet statesToUpdate; if (changedNodes) { - statesToUpdate = EnumSet.of(State.STARTED, State.ABORTED); + // If we are reacting to a change in the cluster node configuration we have to update the shard states of both started + // and aborted snapshots to potentially fail shards running on the removed nodes + statesToUpdate = EnumSet.of(SnapshotsInProgress.State.STARTED, SnapshotsInProgress.State.ABORTED); } else { // We are reacting to shards that started only so which only affects the individual shard states of started // snapshots - statesToUpdate = EnumSet.of(State.STARTED); + statesToUpdate = EnumSet.of(SnapshotsInProgress.State.STARTED); } // We keep a cache of shards that failed in this map. If we fail a shardId for a given repository because of @@ -1003,9 +1009,9 @@ public ClusterState execute(ClusterState currentState) { // TODO: the code in this state update duplicates large chunks of the logic in #SHARD_STATE_EXECUTOR. // We should refactor it to ideally also go through #SHARD_STATE_EXECUTOR by hand-crafting shard state updates // that encapsulate nodes leaving or indices having been deleted and passing them to the executor instead. - SnapshotsInProgress updatedSnapshots = snapshots; + SnapshotsInProgress updatedSnapshots = snapshotsInProgress; - for (final List snapshotsInRepo : snapshots.entriesByRepo()) { + for (final List snapshotsInRepo : snapshotsInProgress.entriesByRepo()) { boolean changed = false; final List updatedEntriesForRepo = new ArrayList<>(); final Map knownFailures = new HashMap<>(); @@ -1013,17 +1019,17 @@ public ClusterState execute(ClusterState currentState) { for (SnapshotsInProgress.Entry snapshotEntry : snapshotsInRepo) { if (statesToUpdate.contains(snapshotEntry.state())) { if (snapshotEntry.isClone()) { - if (snapshotEntry.shardsByRepoShardId().isEmpty()) { + if (snapshotEntry.shardSnapshotStatusByRepoShardId().isEmpty()) { // Currently initializing clone if (initializingClones.contains(snapshotEntry.snapshot())) { updatedEntriesForRepo.add(snapshotEntry); } else { - logger.debug("removing not yet start clone operation [{}]", snapshotEntry); + logger.debug("removing not yet started clone operation [{}]", snapshotEntry); changed = true; } } else { // see if any clones may have had a shard become available for execution because of failures - if (deletes.hasExecutingDeletion(repositoryName)) { + if (deletesInProgress.hasExecutingDeletion(repositoryName)) { // Currently executing a delete for this repo, no need to try and update any clone operations. // The logic for finishing the delete will update running clones with the latest changes. updatedEntriesForRepo.add(snapshotEntry); @@ -1033,7 +1039,7 @@ public ClusterState execute(ClusterState currentState) { InFlightShardSnapshotStates inFlightShardSnapshotStates = null; for (Map.Entry failureEntry : knownFailures.entrySet()) { final RepositoryShardId repositoryShardId = failureEntry.getKey(); - final ShardSnapshotStatus existingStatus = snapshotEntry.shardsByRepoShardId() + final ShardSnapshotStatus existingStatus = snapshotEntry.shardSnapshotStatusByRepoShardId() .get(repositoryShardId); if (ShardSnapshotStatus.UNASSIGNED_QUEUED.equals(existingStatus)) { if (inFlightShardSnapshotStates == null) { @@ -1047,7 +1053,7 @@ public ClusterState execute(ClusterState currentState) { continue; } if (clones == null) { - clones = ImmutableOpenMap.builder(snapshotEntry.shardsByRepoShardId()); + clones = ImmutableOpenMap.builder(snapshotEntry.shardSnapshotStatusByRepoShardId()); } // We can use the generation from the shard failure to start the clone operation here // because #processWaitingShardsAndRemovedNodes adds generations to failure statuses that @@ -1075,7 +1081,7 @@ public ClusterState execute(ClusterState currentState) { snapshotEntry, routingTable, nodes, - snapshots::isNodeIdForRemoval, + snapshotsInProgress::isNodeIdForRemoval, knownFailures ); if (shards != null) { @@ -1098,7 +1104,7 @@ public ClusterState execute(ClusterState currentState) { } else { // Now we're down to completed or un-modified snapshots - if (snapshotEntry.state().completed() || completed(snapshotEntry.shardsByRepoShardId().values())) { + if (snapshotEntry.state().completed() || completed(snapshotEntry.shardSnapshotStatusByRepoShardId().values())) { finishedSnapshots.add(snapshotEntry); } updatedEntriesForRepo.add(snapshotEntry); @@ -1109,7 +1115,7 @@ public ClusterState execute(ClusterState currentState) { } } final ClusterState res = readyDeletions( - updatedSnapshots != snapshots + updatedSnapshots != snapshotsInProgress ? ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, updatedSnapshots).build() : currentState ).v1(); @@ -1176,7 +1182,8 @@ private static ImmutableOpenMap processWaitingShar assert snapshotEntry.isClone() == false : "clones take a different path"; boolean snapshotChanged = false; ImmutableOpenMap.Builder shards = ImmutableOpenMap.builder(); - for (Map.Entry shardSnapshotEntry : snapshotEntry.shardsByRepoShardId().entrySet()) { + for (Map.Entry shardSnapshotEntry : snapshotEntry.shardSnapshotStatusByRepoShardId() + .entrySet()) { ShardSnapshotStatus shardStatus = shardSnapshotEntry.getValue(); ShardId shardId = snapshotEntry.shardId(shardSnapshotEntry.getKey()); if (shardStatus.equals(ShardSnapshotStatus.UNASSIGNED_QUEUED)) { @@ -1239,7 +1246,7 @@ private static ImmutableOpenMap processWaitingShar } // Shard that we were waiting for went into unassigned state or disappeared (index or shard is gone) - giving up snapshotChanged = true; - logger.warn("failing snapshot of shard [{}] on unassigned shard [{}]", shardId, shardStatus.nodeId()); + logger.warn("failing snapshot of shard [{}] on node [{}] because shard is unassigned", shardId, shardStatus.nodeId()); final ShardSnapshotStatus failedState = new ShardSnapshotStatus( shardStatus.nodeId(), ShardState.FAILED, @@ -1278,8 +1285,9 @@ private static ImmutableOpenMap processWaitingShar private static boolean waitingShardsStartedOrUnassigned(SnapshotsInProgress snapshotsInProgress, ClusterChangedEvent event) { for (List entries : snapshotsInProgress.entriesByRepo()) { for (SnapshotsInProgress.Entry entry : entries) { - if (entry.state() == State.STARTED && entry.isClone() == false) { - for (Map.Entry shardStatus : entry.shardsByRepoShardId().entrySet()) { + if (entry.state() == SnapshotsInProgress.State.STARTED && entry.isClone() == false) { + for (Map.Entry shardStatus : entry.shardSnapshotStatusByRepoShardId() + .entrySet()) { final ShardState state = shardStatus.getValue().state(); if (state != ShardState.WAITING && state != ShardState.QUEUED && state != ShardState.PAUSED_FOR_NODE_REMOVAL) { continue; @@ -1317,7 +1325,7 @@ private static boolean removedNodesCleanupNeeded(SnapshotsInProgress snapshotsIn // nothing to do for already completed snapshots or clones that run on master anyways return false; } - for (ShardSnapshotStatus shardSnapshotStatus : snapshot.shardsByRepoShardId().values()) { + for (ShardSnapshotStatus shardSnapshotStatus : snapshot.shardSnapshotStatusByRepoShardId().values()) { if (shardSnapshotStatus.state().completed() == false && removedNodeIds.contains(shardSnapshotStatus.nodeId())) { // Snapshot had an incomplete shard running on a removed node so we need to adjust that shard's snapshot status return true; @@ -1335,7 +1343,7 @@ private static boolean removedNodesCleanupNeeded(SnapshotsInProgress snapshotsIn private void endSnapshot(SnapshotsInProgress.Entry entry, Metadata metadata, @Nullable RepositoryData repositoryData) { final Snapshot snapshot = entry.snapshot(); final boolean newFinalization = endingSnapshots.add(snapshot); - if (entry.isClone() && entry.state() == State.FAILED) { + if (entry.isClone() && entry.state() == SnapshotsInProgress.State.FAILED) { logger.debug("Removing failed snapshot clone [{}] from cluster state", entry); if (newFinalization) { removeFailedSnapshotFromClusterState( @@ -1415,7 +1423,7 @@ private void finalizeSnapshotEntry(Snapshot snapshot, Metadata metadata, Reposit final List finalIndices = shardGenerations.indices().stream().map(IndexId::getName).toList(); final Set indexNames = new HashSet<>(finalIndices); ArrayList shardFailures = new ArrayList<>(); - for (Map.Entry shardStatus : entry.shardsByRepoShardId().entrySet()) { + for (Map.Entry shardStatus : entry.shardSnapshotStatusByRepoShardId().entrySet()) { RepositoryShardId shardId = shardStatus.getKey(); if (indexNames.contains(shardId.indexName()) == false) { assert entry.partial() : "only ignoring shard failures for concurrently deleted indices for partial snapshots"; @@ -1467,7 +1475,7 @@ private void finalizeSnapshotEntry(Snapshot snapshot, Metadata metadata, Reposit final Map indexSnapshotDetails = Maps.newMapWithExpectedSize( finalIndices.size() ); - for (Map.Entry shardEntry : entry.shardsByRepoShardId().entrySet()) { + for (Map.Entry shardEntry : entry.shardSnapshotStatusByRepoShardId().entrySet()) { indexSnapshotDetails.compute(shardEntry.getKey().indexName(), (indexName, current) -> { if (current == SnapshotInfo.IndexSnapshotDetails.SKIPPED) { // already found an unsuccessful shard in this index, skip this shard @@ -1506,7 +1514,7 @@ private void finalizeSnapshotEntry(Snapshot snapshot, Metadata metadata, Reposit entry.partial() ? onlySuccessfulFeatureStates(entry, finalIndices) : entry.featureStates(), failure, threadPool.absoluteTimeInMillis(), - entry.partial() ? shardGenerations.totalShards() : entry.shardsByRepoShardId().size(), + entry.partial() ? shardGenerations.totalShards() : entry.shardSnapshotStatusByRepoShardId().size(), shardFailures, entry.includeGlobalState(), entry.userMetadata(), @@ -1579,7 +1587,7 @@ private static List onlySuccessfulFeatureStates(SnapshotsIn // Figure out which indices have unsuccessful shards Set indicesWithUnsuccessfulShards = new HashSet<>(); - entry.shardsByRepoShardId().forEach((key, value) -> { + entry.shardSnapshotStatusByRepoShardId().forEach((key, value) -> { final ShardState shardState = value.state(); if (shardState.failed() || shardState.completed() == false) { indicesWithUnsuccessfulShards.add(key.indexName()); @@ -1749,16 +1757,21 @@ private static Tuple> read * Computes the cluster state resulting from removing a given snapshot create operation from the given state. This method will update * the shard generations of snapshots that the given snapshot depended on so that finalizing them will not cause rolling back to an * outdated shard generation. + *

+ * For example, shard snapshot X can be taken, but not finalized yet. Shard snapshot Y can then depend upon shard snapshot X. Then shard + * snapshot Y may finalize before shard snapshot X, but including X. However, X does not include Y. Therefore we update X to use Y's + * shard generation file (list of snapshots and dependencies) to avoid overwriting with X's file that is missing Y. * * @param state current cluster state * @param snapshot snapshot for which to remove the snapshot operation * @return updated cluster state */ public static ClusterState stateWithoutSnapshot(ClusterState state, Snapshot snapshot, ShardGenerations shardGenerations) { - final SnapshotsInProgress snapshots = SnapshotsInProgress.get(state); + final SnapshotsInProgress inProgressSnapshots = SnapshotsInProgress.get(state); ClusterState result = state; int indexOfEntry = -1; - final List entryList = snapshots.forRepo(snapshot.getRepository()); + // Find the in-progress snapshot entry that matches {@code snapshot}. + final List entryList = inProgressSnapshots.forRepo(snapshot.getRepository()); for (int i = 0; i < entryList.size(); i++) { SnapshotsInProgress.Entry entry = entryList.get(i); if (entry.snapshot().equals(snapshot)) { @@ -1767,14 +1780,15 @@ public static ClusterState stateWithoutSnapshot(ClusterState state, Snapshot sna } } if (indexOfEntry >= 0) { - final List entries = new ArrayList<>(entryList.size() - 1); + final List updatedEntries = new ArrayList<>(entryList.size() - 1); final SnapshotsInProgress.Entry removedEntry = entryList.get(indexOfEntry); for (int i = 0; i < indexOfEntry; i++) { final SnapshotsInProgress.Entry previousEntry = entryList.get(i); if (removedEntry.isClone()) { if (previousEntry.isClone()) { ImmutableOpenMap.Builder updatedShardAssignments = null; - for (Map.Entry finishedShardEntry : removedEntry.shardsByRepoShardId() + for (Map.Entry finishedShardEntry : removedEntry + .shardSnapshotStatusByRepoShardId() .entrySet()) { final ShardSnapshotStatus shardState = finishedShardEntry.getValue(); if (shardState.state() == ShardState.SUCCESS) { @@ -1782,19 +1796,20 @@ public static ClusterState stateWithoutSnapshot(ClusterState state, Snapshot sna updatedShardAssignments, shardState, finishedShardEntry.getKey(), - previousEntry.shardsByRepoShardId() + previousEntry.shardSnapshotStatusByRepoShardId() ); } } - addCloneEntry(entries, previousEntry, updatedShardAssignments); + addCloneEntry(updatedEntries, previousEntry, updatedShardAssignments); } else { ImmutableOpenMap.Builder updatedShardAssignments = null; - for (Map.Entry finishedShardEntry : removedEntry.shardsByRepoShardId() + for (Map.Entry finishedShardEntry : removedEntry + .shardSnapshotStatusByRepoShardId() .entrySet()) { final ShardSnapshotStatus shardState = finishedShardEntry.getValue(); final RepositoryShardId repositoryShardId = finishedShardEntry.getKey(); if (shardState.state() != ShardState.SUCCESS - || previousEntry.shardsByRepoShardId().containsKey(repositoryShardId) == false) { + || previousEntry.shardSnapshotStatusByRepoShardId().containsKey(repositoryShardId) == false) { continue; } updatedShardAssignments = maybeAddUpdatedAssignment( @@ -1805,17 +1820,18 @@ public static ClusterState stateWithoutSnapshot(ClusterState state, Snapshot sna ); } - addSnapshotEntry(entries, previousEntry, updatedShardAssignments); + addSnapshotEntry(updatedEntries, previousEntry, updatedShardAssignments); } } else { if (previousEntry.isClone()) { ImmutableOpenMap.Builder updatedShardAssignments = null; - for (Map.Entry finishedShardEntry : removedEntry.shardsByRepoShardId() + for (Map.Entry finishedShardEntry : removedEntry + .shardSnapshotStatusByRepoShardId() .entrySet()) { final ShardSnapshotStatus shardState = finishedShardEntry.getValue(); final RepositoryShardId repositoryShardId = finishedShardEntry.getKey(); if (shardState.state() != ShardState.SUCCESS - || previousEntry.shardsByRepoShardId().containsKey(repositoryShardId) == false + || previousEntry.shardSnapshotStatusByRepoShardId().containsKey(repositoryShardId) == false || shardGenerations.hasShardGen(finishedShardEntry.getKey()) == false) { continue; } @@ -1823,17 +1839,18 @@ public static ClusterState stateWithoutSnapshot(ClusterState state, Snapshot sna updatedShardAssignments, shardState, repositoryShardId, - previousEntry.shardsByRepoShardId() + previousEntry.shardSnapshotStatusByRepoShardId() ); } - addCloneEntry(entries, previousEntry, updatedShardAssignments); + addCloneEntry(updatedEntries, previousEntry, updatedShardAssignments); } else { ImmutableOpenMap.Builder updatedShardAssignments = null; - for (Map.Entry finishedShardEntry : removedEntry.shardsByRepoShardId() + for (Map.Entry finishedShardEntry : removedEntry + .shardSnapshotStatusByRepoShardId() .entrySet()) { final ShardSnapshotStatus shardState = finishedShardEntry.getValue(); if (shardState.state() == ShardState.SUCCESS - && previousEntry.shardsByRepoShardId().containsKey(finishedShardEntry.getKey()) + && previousEntry.shardSnapshotStatusByRepoShardId().containsKey(finishedShardEntry.getKey()) && shardGenerations.hasShardGen(finishedShardEntry.getKey())) { updatedShardAssignments = maybeAddUpdatedAssignment( updatedShardAssignments, @@ -1843,15 +1860,18 @@ public static ClusterState stateWithoutSnapshot(ClusterState state, Snapshot sna ); } } - addSnapshotEntry(entries, previousEntry, updatedShardAssignments); + addSnapshotEntry(updatedEntries, previousEntry, updatedShardAssignments); } } } for (int i = indexOfEntry + 1; i < entryList.size(); i++) { - entries.add(entryList.get(i)); + updatedEntries.add(entryList.get(i)); } result = ClusterState.builder(state) - .putCustom(SnapshotsInProgress.TYPE, snapshots.withUpdatedEntriesForRepo(snapshot.getRepository(), entries)) + .putCustom( + SnapshotsInProgress.TYPE, + inProgressSnapshots.withUpdatedEntriesForRepo(snapshot.getRepository(), updatedEntries) + ) .build(); } return readyDeletions(result).v1(); @@ -1880,7 +1900,7 @@ private static void addCloneEntry( entries.add(entryToUpdate); } else { final ImmutableOpenMap.Builder updatedStatus = ImmutableOpenMap.builder( - entryToUpdate.shardsByRepoShardId() + entryToUpdate.shardSnapshotStatusByRepoShardId() ); updatedStatus.putAllFromMap(updatedShardAssignments.build()); entries.add(entryToUpdate.withClones(updatedStatus.build())); @@ -2123,7 +2143,7 @@ public ClusterState execute(ClusterState currentState) { final SnapshotsInProgress updatedSnapshots = snapshotsInProgress.withUpdatedEntriesForRepo( repositoryName, snapshotsInProgress.forRepo(repositoryName).stream().map(existing -> { - if (existing.state() == State.STARTED + if (existing.state() == SnapshotsInProgress.State.STARTED && snapshotIdsRequiringCleanup.contains(existing.snapshot().getSnapshotId())) { // snapshot is started - mark every non completed shard as aborted final SnapshotsInProgress.Entry abortedEntry = existing.abort(); @@ -2257,7 +2277,7 @@ private static boolean isWritingToRepository(SnapshotsInProgress.Entry entry) { // Entry is writing to the repo because it's finalizing on master return true; } - for (ShardSnapshotStatus value : entry.shardsByRepoShardId().values()) { + for (ShardSnapshotStatus value : entry.shardSnapshotStatusByRepoShardId().values()) { if (value.isActive()) { // Entry is writing to the repo because it's writing to a shard on a data node or waiting to do so for a concrete shard return true; @@ -2746,7 +2766,8 @@ private SnapshotsInProgress updatedSnapshotsInProgress(ClusterState currentState if (entry.isClone()) { // Collect waiting shards from that entry that we can assign now that we are done with the deletion final List canBeUpdated = new ArrayList<>(); - for (Map.Entry value : entry.shardsByRepoShardId().entrySet()) { + for (Map.Entry value : entry.shardSnapshotStatusByRepoShardId() + .entrySet()) { if (value.getValue().equals(ShardSnapshotStatus.UNASSIGNED_QUEUED) && reassignedShardIds.contains(value.getKey()) == false) { canBeUpdated.add(value.getKey()); @@ -2762,7 +2783,7 @@ private SnapshotsInProgress updatedSnapshotsInProgress(ClusterState currentState inFlightShardStates = InFlightShardSnapshotStates.forEntries(snapshotsInProgress.forRepo(repoName)); } final ImmutableOpenMap.Builder updatedAssignmentsBuilder = - ImmutableOpenMap.builder(entry.shardsByRepoShardId()); + ImmutableOpenMap.builder(entry.shardSnapshotStatusByRepoShardId()); for (RepositoryShardId shardId : canBeUpdated) { if (inFlightShardStates.isActive(shardId.indexName(), shardId.shardId()) == false) { markShardReassigned(shardId, reassignedShardIds); @@ -2785,7 +2806,8 @@ private SnapshotsInProgress updatedSnapshotsInProgress(ClusterState currentState } else { // Collect waiting shards that in entry that we can assign now that we are done with the deletion final List canBeUpdated = new ArrayList<>(); - for (Map.Entry value : entry.shardsByRepoShardId().entrySet()) { + for (Map.Entry value : entry.shardSnapshotStatusByRepoShardId() + .entrySet()) { final RepositoryShardId repositoryShardId = value.getKey(); if (value.getValue().equals(ShardSnapshotStatus.UNASSIGNED_QUEUED) && reassignedShardIds.contains(repositoryShardId) == false) { @@ -3272,7 +3294,12 @@ SnapshotsInProgress.Entry computeUpdatedEntry() { if (entry.snapshot().getSnapshotId().equals(update.snapshot.getSnapshotId())) { // update a currently running shard level operation if (update.isClone()) { - executeShardSnapshotUpdate(entry.shardsByRepoShardId(), this::clonesBuilder, update, update.repoShardId); + executeShardSnapshotUpdate( + entry.shardSnapshotStatusByRepoShardId(), + this::clonesBuilder, + update, + update.repoShardId + ); } else { executeShardSnapshotUpdate(entry.shards(), this::shardsBuilder, update, update.shardId); } @@ -3398,7 +3425,7 @@ private void tryStartNextTaskAfterCloneUpdated(RepositoryShardId repoShardId, Sh // start a shard snapshot or clone operation on the current entry if (entry.isClone() == false) { tryStartSnapshotAfterCloneFinish(repoShardId, updatedState.generation()); - } else if (isQueued(entry.shardsByRepoShardId().get(repoShardId))) { + } else if (isQueued(entry.shardSnapshotStatusByRepoShardId().get(repoShardId))) { final String localNodeId = initialState.nodes().getLocalNodeId(); assert updatedState.nodeId().equals(localNodeId) : "Clone updated with node id [" + updatedState.nodeId() + "] but local node id is [" + localNodeId + "]"; @@ -3412,7 +3439,7 @@ private void tryStartNextTaskAfterSnapshotUpdated(ShardId shardId, ShardSnapshot final IndexId indexId = entry.indices().get(shardId.getIndexName()); if (indexId != null) { final RepositoryShardId repoShardId = new RepositoryShardId(indexId, shardId.id()); - if (isQueued(entry.shardsByRepoShardId().get(repoShardId))) { + if (isQueued(entry.shardSnapshotStatusByRepoShardId().get(repoShardId))) { if (entry.isClone()) { // shard snapshot was completed, we check if we can start a clone operation for the same repo shard startShardOperation( @@ -3431,7 +3458,7 @@ private void tryStartNextTaskAfterSnapshotUpdated(ShardId shardId, ShardSnapshot private void tryStartSnapshotAfterCloneFinish(RepositoryShardId repoShardId, ShardGeneration generation) { assert entry.source() == null; // current entry is a snapshot operation so we must translate the repository shard id to a routing shard id - if (isQueued(entry.shardsByRepoShardId().get(repoShardId))) { + if (isQueued(entry.shardSnapshotStatusByRepoShardId().get(repoShardId))) { startShardSnapshot(repoShardId, generation); } } @@ -3467,7 +3494,7 @@ private void startShardSnapshot(RepositoryShardId repoShardId, ShardGeneration g private ImmutableOpenMap.Builder clonesBuilder() { assert shardsBuilder == null; if (clonesBuilder == null) { - clonesBuilder = ImmutableOpenMap.builder(entry.shardsByRepoShardId()); + clonesBuilder = ImmutableOpenMap.builder(entry.shardSnapshotStatusByRepoShardId()); } return clonesBuilder; } @@ -3620,9 +3647,9 @@ private void startExecutableClones(SnapshotsInProgress snapshotsInProgress, @Nul private void startExecutableClones(List entries) { for (SnapshotsInProgress.Entry entry : entries) { - if (entry.isClone() && entry.state() == State.STARTED) { + if (entry.isClone() && entry.state() == SnapshotsInProgress.State.STARTED) { // this is a clone, see if new work is ready - for (Map.Entry clone : entry.shardsByRepoShardId().entrySet()) { + for (Map.Entry clone : entry.shardSnapshotStatusByRepoShardId().entrySet()) { if (clone.getValue().state() == ShardState.INIT) { runReadyClone( entry.snapshot(), diff --git a/server/src/main/java/org/elasticsearch/snapshots/package-info.java b/server/src/main/java/org/elasticsearch/snapshots/package-info.java index 4c175bc88faf9..a6dc8021fcba8 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/package-info.java +++ b/server/src/main/java/org/elasticsearch/snapshots/package-info.java @@ -98,13 +98,13 @@ *

    *
  1. First, {@link org.elasticsearch.snapshots.SnapshotsService#cloneSnapshot} is invoked which will place a placeholder entry into * {@code SnapshotsInProgress} that does not yet contain any shard clone assignments. Note that unlike in the case of snapshot - * creation, the shard level clone tasks in {@link org.elasticsearch.cluster.SnapshotsInProgress.Entry#shardsByRepoShardId()} are not - * created in the initial cluster state update as is done for shard snapshot assignments in - * {@link org.elasticsearch.cluster.SnapshotsInProgress.Entry#shards}. This is due to the fact that shard snapshot assignments are - * computed purely from information in the current cluster state while shard clone assignments require information to be read from the - * repository, which is too slow of a process to be done inside a cluster state update. Loading this information ahead of creating a - * task in the cluster state, runs the risk of race conditions where the source snapshot is being deleted before the clone task is - * enqueued in the cluster state.
    2. + * creation, the shard level clone tasks in + * {@link org.elasticsearch.cluster.SnapshotsInProgress.Entry#shardSnapshotStatusByRepoShardId()} are not created in the initial cluster + * state update as is done for shard snapshot assignments in {@link org.elasticsearch.cluster.SnapshotsInProgress.Entry#shards}. This is + * due to the fact that shard snapshot assignments are computed purely from information in the current cluster state while shard clone + * assignments require information to be read from the repository, which is too slow of a process to be done inside a cluster state + * update. Loading this information ahead of creating a task in the cluster state, runs the risk of race conditions where the source + * snapshot is being deleted before the clone task is enqueued in the cluster state. *
  2. Once a placeholder task for the clone operation is put into the cluster state, we must determine the number of shards in each * index that is to be cloned as well as ensure the health of the index snapshots in the source snapshot. In order to determine the * shard count for each index that is to be cloned, we load the index metadata for each such index using the repository's diff --git a/server/src/main/java/org/elasticsearch/tasks/CancellableTask.java b/server/src/main/java/org/elasticsearch/tasks/CancellableTask.java index 8a0aa2033a30e..8a385617bee89 100644 --- a/server/src/main/java/org/elasticsearch/tasks/CancellableTask.java +++ b/server/src/main/java/org/elasticsearch/tasks/CancellableTask.java @@ -109,6 +109,11 @@ public final boolean notifyIfCancelled(ActionListener listener) { return true; } + @Override + public String toString() { + return "CancellableTask{" + super.toString() + ", reason='" + reason + '\'' + ", isCancelled=" + isCancelled + '}'; + } + private TaskCancelledException getTaskCancelledException() { assert Thread.holdsLock(this); assert isCancelled; diff --git a/server/src/main/java/org/elasticsearch/tasks/Task.java b/server/src/main/java/org/elasticsearch/tasks/Task.java index 83ee08574df4e..46eb59c3a8cd8 100644 --- a/server/src/main/java/org/elasticsearch/tasks/Task.java +++ b/server/src/main/java/org/elasticsearch/tasks/Task.java @@ -225,6 +225,8 @@ public String toString() { + parentTask + ", startTime=" + startTime + + ", headers=" + + headers + ", startTimeNanos=" + startTimeNanos + '}'; diff --git a/server/src/main/java/org/elasticsearch/transport/NodeNotConnectedException.java b/server/src/main/java/org/elasticsearch/transport/NodeNotConnectedException.java index 6e1f29353f78a..b54d6bdfe3d75 100644 --- a/server/src/main/java/org/elasticsearch/transport/NodeNotConnectedException.java +++ b/server/src/main/java/org/elasticsearch/transport/NodeNotConnectedException.java @@ -27,4 +27,9 @@ public NodeNotConnectedException(DiscoveryNode node, String msg) { public NodeNotConnectedException(StreamInput in) throws IOException { super(in); } + + @Override + public Throwable fillInStackTrace() { + return this; // this exception doesn't imply a bug, no need for a stack trace + } } diff --git a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification index a9d9c6a5a1938..1bdc17b6cf6f8 100644 --- a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification +++ b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification @@ -6,6 +6,7 @@ # Side Public License, v 1. # +org.elasticsearch.action.bulk.BulkFeatures org.elasticsearch.features.FeatureInfrastructureFeatures org.elasticsearch.health.HealthFeatures org.elasticsearch.cluster.service.TransportFeatures @@ -14,6 +15,7 @@ org.elasticsearch.rest.RestFeatures org.elasticsearch.indices.IndicesFeatures org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures org.elasticsearch.index.mapper.MapperFeatures +org.elasticsearch.ingest.IngestGeoIpFeatures org.elasticsearch.search.SearchFeatures org.elasticsearch.search.retriever.RetrieversFeatures org.elasticsearch.script.ScriptFeatures diff --git a/server/src/main/resources/org/elasticsearch/TransportVersions.csv b/server/src/main/resources/org/elasticsearch/TransportVersions.csv index 5f1972e30198a..7d2697539fa13 100644 --- a/server/src/main/resources/org/elasticsearch/TransportVersions.csv +++ b/server/src/main/resources/org/elasticsearch/TransportVersions.csv @@ -124,3 +124,4 @@ 8.14.0,8636001 8.14.1,8636001 8.14.2,8636001 +8.14.3,8636001 diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy index 681a52eb84b8a..1635be24d2e84 100644 --- a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -67,22 +67,10 @@ grant codeBase "${codebase.elasticsearch-cli}" { permission java.util.PropertyPermission "*", "read,write"; }; -grant codeBase "${codebase.jna}" { - // for registering native methods - permission java.lang.RuntimePermission "accessDeclaredMembers"; - permission java.lang.reflect.ReflectPermission "newProxyInPackage.org.elasticsearch.preallocate"; -}; - grant codeBase "${codebase.log4j-api}" { permission java.lang.RuntimePermission "getClassLoader"; }; -grant codeBase "${codebase.elasticsearch-preallocate}" { - // for registering native methods - permission java.lang.RuntimePermission "accessDeclaredMembers"; - permission java.lang.reflect.ReflectPermission "newProxyInPackage.org.elasticsearch.preallocate"; -}; - grant codeBase "${codebase.elasticsearch-simdvec}" { // for access MemorySegmentIndexInput internals permission java.lang.RuntimePermission "accessDeclaredMembers"; diff --git a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json index febcaec1ba057..0d11629803ced 100644 --- a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json +++ b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json @@ -37,5 +37,7 @@ "ALLOCATION_EXPLAIN_API": "cluster-allocation-explain.html", "NETWORK_BINDING_AND_PUBLISHING": "modules-network.html#modules-network-binding-publishing", "SNAPSHOT_REPOSITORY_ANALYSIS": "repo-analysis-api.html", - "S3_COMPATIBLE_REPOSITORIES": "repository-s3.html#repository-s3-compatible-services" + "S3_COMPATIBLE_REPOSITORIES": "repository-s3.html#repository-s3-compatible-services", + "LUCENE_MAX_DOCS_LIMIT": "size-your-shards.html#troubleshooting-max-docs-limit", + "MAX_SHARDS_PER_NODE": "size-your-shards.html#troubleshooting-max-shards-open" } diff --git a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv index d1116ddf99ee7..f177ab1468cb2 100644 --- a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv +++ b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv @@ -124,3 +124,4 @@ 8.14.0,8505000 8.14.1,8505000 8.14.2,8505000 +8.14.3,8505000 diff --git a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java index 7afa7adedc7bf..c015dc6177cad 100644 --- a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java +++ b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.indices.TestIndexNameExpressionResolver; @@ -88,7 +89,7 @@ public ActionRequestValidationException validate() { } class FakeTransportAction extends TransportAction { protected FakeTransportAction(String actionName, ActionFilters actionFilters, TaskManager taskManager) { - super(actionName, actionFilters, taskManager); + super(actionName, actionFilters, taskManager, EsExecutors.DIRECT_EXECUTOR_SERVICE); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainActionTests.java index 15a1bdde7706a..651599d5ee042 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainActionTests.java @@ -67,6 +67,7 @@ public void testCanNotTripCircuitBreaker() { ); } + @Override @After public void tearDown() throws Exception { super.tearDown(); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java index 6885e6851c77d..686a7eda4fedc 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.admin.cluster.allocation; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; @@ -48,7 +49,6 @@ import java.util.List; import java.util.Map; import java.util.Queue; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; @@ -131,11 +131,7 @@ public DesiredBalance compute( SNAPSHOT_INFO_SERVICE_WITH_NO_SHARD_SIZES ); - PlainActionFuture.get( - f -> allocationService.reroute(clusterState, "inital-allocate", f), - 10, - TimeUnit.SECONDS - ); + safeAwait((ActionListener listener) -> allocationService.reroute(clusterState, "inital-allocate", listener)); var balanceBeforeReset = allocator.getDesiredBalance(); assertThat(balanceBeforeReset.lastConvergedIndex(), greaterThan(DesiredBalance.INITIAL.lastConvergedIndex())); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java new file mode 100644 index 0000000000000..d99538ac5e6e7 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.allocation; + +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.routing.allocation.AllocationStatsService; +import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsTests; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.transport.CapturingTransport; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.junit.After; +import org.junit.Before; + +import java.util.EnumSet; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.not; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class TransportGetAllocationStatsActionTests extends ESTestCase { + + private ThreadPool threadPool; + private ClusterService clusterService; + private TransportService transportService; + private AllocationStatsService allocationStatsService; + private FeatureService featureService; + + private TransportGetAllocationStatsAction action; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + threadPool = new TestThreadPool(TransportClusterAllocationExplainActionTests.class.getName()); + clusterService = ClusterServiceUtils.createClusterService(threadPool); + transportService = new CapturingTransport().createTransportService( + clusterService.getSettings(), + threadPool, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, + address -> clusterService.localNode(), + clusterService.getClusterSettings(), + Set.of() + ); + allocationStatsService = mock(AllocationStatsService.class); + featureService = mock(FeatureService.class); + action = new TransportGetAllocationStatsAction( + transportService, + clusterService, + threadPool, + new ActionFilters(Set.of()), + null, + allocationStatsService, + featureService + ); + } + + @Override + @After + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdown(); + clusterService.close(); + transportService.close(); + } + + public void testReturnsOnlyRequestedStats() throws Exception { + + var metrics = EnumSet.copyOf(randomSubsetOf(Metric.values().length, Metric.values())); + + var request = new TransportGetAllocationStatsAction.Request( + TimeValue.ONE_MINUTE, + new TaskId(randomIdentifier(), randomNonNegativeLong()), + metrics + ); + + when(allocationStatsService.stats()).thenReturn(Map.of(randomIdentifier(), NodeAllocationStatsTests.randomNodeAllocationStats())); + when(featureService.clusterHasFeature(any(ClusterState.class), eq(AllocationStatsFeatures.INCLUDE_DISK_THRESHOLD_SETTINGS))) + .thenReturn(true); + + var future = new PlainActionFuture(); + action.masterOperation(mock(Task.class), request, ClusterState.EMPTY_STATE, future); + var response = future.get(); + + if (metrics.contains(Metric.ALLOCATIONS)) { + assertThat(response.getNodeAllocationStats(), not(anEmptyMap())); + verify(allocationStatsService).stats(); + } else { + assertThat(response.getNodeAllocationStats(), anEmptyMap()); + verify(allocationStatsService, never()).stats(); + } + + if (metrics.contains(Metric.FS)) { + assertNotNull(response.getDiskThresholdSettings()); + } else { + assertNull(response.getDiskThresholdSettings()); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java index 414dc45ee458f..c1ecadc5207e8 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java @@ -9,7 +9,7 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.cluster.ClusterInfo; import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterInfoTests; @@ -53,7 +53,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.elasticsearch.cluster.ClusterModule.BALANCED_ALLOCATOR; @@ -69,8 +68,8 @@ public class TransportGetDesiredBalanceActionTests extends ESAllocationTestCase private final DesiredBalanceShardsAllocator desiredBalanceShardsAllocator = mock(DesiredBalanceShardsAllocator.class); private final ClusterInfoService clusterInfoService = mock(ClusterInfoService.class); - private ThreadPool threadPool = mock(ThreadPool.class); - private TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + private final ThreadPool threadPool = mock(ThreadPool.class); + private final TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); private TransportGetDesiredBalanceAction transportGetDesiredBalanceAction; @Before @@ -87,24 +86,25 @@ public void initialize() { ); } - private static DesiredBalanceResponse execute(TransportGetDesiredBalanceAction action, ClusterState clusterState) throws Exception { - return PlainActionFuture.get( - future -> action.masterOperation( + private static SubscribableListener execute( + TransportGetDesiredBalanceAction action, + ClusterState clusterState + ) { + return SubscribableListener.newForked( + listener -> action.masterOperation( new Task(1, "test", TransportGetDesiredBalanceAction.TYPE.name(), "", TaskId.EMPTY_TASK_ID, Map.of()), new DesiredBalanceRequest(TEST_REQUEST_TIMEOUT), clusterState, - future - ), - 10, - TimeUnit.SECONDS + listener + ) ); } - private DesiredBalanceResponse executeAction(ClusterState clusterState) throws Exception { + private SubscribableListener executeAction(ClusterState clusterState) { return execute(transportGetDesiredBalanceAction, clusterState); } - public void testReturnsErrorIfAllocatorIsNotDesiredBalanced() throws Exception { + public void testReturnsErrorIfAllocatorIsNotDesiredBalanced() { var clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadataWithConfiguredAllocator(BALANCED_ALLOCATOR)).build(); final var action = new TransportGetDesiredBalanceAction( transportService, @@ -117,7 +117,7 @@ public void testReturnsErrorIfAllocatorIsNotDesiredBalanced() throws Exception { mock(WriteLoadForecaster.class) ); - final var exception = expectThrows(ResourceNotFoundException.class, () -> execute(action, clusterState)); + final var exception = asInstanceOf(ResourceNotFoundException.class, safeAwaitFailure(execute(action, clusterState))); assertEquals("Desired balance allocator is not in use, no desired balance found", exception.getMessage()); assertThat(exception.status(), equalTo(RestStatus.NOT_FOUND)); } @@ -129,7 +129,7 @@ public void testReturnsErrorIfDesiredBalanceIsNotAvailable() throws Exception { assertEquals( "Desired balance is not computed yet", - expectThrows(ResourceNotFoundException.class, () -> executeAction(clusterState)).getMessage() + asInstanceOf(ResourceNotFoundException.class, safeAwaitFailure(executeAction(clusterState))).getMessage() ); } @@ -230,7 +230,7 @@ public void testGetDesiredBalance() throws Exception { .routingTable(routingTable) .build(); - final var desiredBalanceResponse = executeAction(clusterState); + final var desiredBalanceResponse = safeAwait(executeAction(clusterState)); assertThat(desiredBalanceResponse.getStats(), equalTo(desiredBalanceStats)); assertThat(desiredBalanceResponse.getClusterBalanceStats(), notNullValue()); assertThat(desiredBalanceResponse.getClusterInfo(), equalTo(clusterInfo)); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java index 82f22a2572c1d..09b0a78849cd9 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -561,7 +561,12 @@ private static int expectedChunks(@Nullable NodeIndicesStats nodeIndicesStats, N private static CommonStats createIndexLevelCommonStats() { CommonStats stats = new CommonStats(new CommonStatsFlags().clear().set(CommonStatsFlags.Flag.Mappings, true)); - stats.nodeMappings = new NodeMappingStats(randomNonNegativeLong(), randomNonNegativeLong()); + stats.nodeMappings = new NodeMappingStats( + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong() + ); return stats; } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParametersTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParametersTests.java new file mode 100644 index 0000000000000..86548d1b33e85 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParametersTests.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.node.stats; + +import org.elasticsearch.TransportVersions; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric; +import org.elasticsearch.common.io.stream.ByteArrayStreamInput; +import org.elasticsearch.common.io.stream.BytesRefStreamOutput; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.EnumSerializationTestUtils; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.List; + +public class NodesStatsRequestParametersTests extends ESTestCase { + + public void testReadWriteMetricSet() { + for (var version : List.of(TransportVersions.VERSIONED_MASTER_NODE_REQUESTS, TransportVersions.NODES_STATS_ENUM_SET)) { + var randSet = randomSubsetOf(Metric.ALL); + var metricsOut = randSet.isEmpty() ? EnumSet.noneOf(Metric.class) : EnumSet.copyOf(randSet); + try { + var out = new BytesRefStreamOutput(); + out.setTransportVersion(version); + Metric.writeSetTo(out, metricsOut); + var in = new ByteArrayStreamInput(out.get().bytes); + in.setTransportVersion(version); + var metricsIn = Metric.readSetFrom(in); + assertEquals(metricsOut, metricsIn); + } catch (IOException e) { + var errMsg = "metrics=" + metricsOut.toString(); + throw new AssertionError(errMsg, e); + } + } + } + + // future-proof of accidental enum ordering change or extension + public void testEnsureMetricOrdinalsOrder() { + EnumSerializationTestUtils.assertEnumSerialization( + Metric.class, + Metric.OS, + Metric.PROCESS, + Metric.JVM, + Metric.THREAD_POOL, + Metric.FS, + Metric.TRANSPORT, + Metric.HTTP, + Metric.BREAKER, + Metric.SCRIPT, + Metric.DISCOVERY, + Metric.INGEST, + Metric.ADAPTIVE_SELECTION, + Metric.SCRIPT_CACHE, + Metric.INDEXING_PRESSURE, + Metric.REPOSITORIES, + Metric.ALLOCATIONS + ); + } + +} diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java index 000f99b270df2..c33419170fbcc 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java @@ -8,15 +8,15 @@ package org.elasticsearch.action.admin.cluster.node.stats; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; -import java.util.HashSet; +import java.util.List; import java.util.Map; -import java.util.Set; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; @@ -34,7 +34,7 @@ public class NodesStatsRequestTests extends ESTestCase { public void testAddMetrics() throws Exception { NodesStatsRequest request = new NodesStatsRequest(randomAlphaOfLength(8)); request.indices(randomFrom(CommonStatsFlags.ALL)); - String[] metrics = randomSubsetOf(NodesStatsRequestParameters.Metric.allMetrics()).toArray(String[]::new); + List metrics = randomSubsetOf(Metric.ALL); request.addMetrics(metrics); NodesStatsRequest deserializedRequest = roundTripRequest(request); assertRequestsEqual(request, deserializedRequest); @@ -45,7 +45,7 @@ public void testAddMetrics() throws Exception { */ public void testAddSingleMetric() throws Exception { NodesStatsRequest request = new NodesStatsRequest(); - request.addMetric(randomFrom(NodesStatsRequestParameters.Metric.allMetrics())); + request.addMetric(randomFrom(Metric.ALL)); NodesStatsRequest deserializedRequest = roundTripRequest(request); assertRequestsEqual(request, deserializedRequest); } @@ -56,7 +56,7 @@ public void testAddSingleMetric() throws Exception { public void testRemoveSingleMetric() throws Exception { NodesStatsRequest request = new NodesStatsRequest(); request.all(); - String metric = randomFrom(NodesStatsRequestParameters.Metric.allMetrics()); + Metric metric = randomFrom(Metric.ALL); request.removeMetric(metric); NodesStatsRequest deserializedRequest = roundTripRequest(request); assertThat(request.requestedMetrics(), equalTo(deserializedRequest.requestedMetrics())); @@ -83,7 +83,7 @@ public void testNodesInfoRequestAll() throws Exception { request.all(); assertThat(request.indices().getFlags(), equalTo(CommonStatsFlags.ALL.getFlags())); - assertThat(request.requestedMetrics(), equalTo(NodesStatsRequestParameters.Metric.allMetrics())); + assertThat(request.requestedMetrics(), equalTo(Metric.ALL)); } /** @@ -97,32 +97,6 @@ public void testNodesInfoRequestClear() throws Exception { assertThat(request.requestedMetrics(), empty()); } - /** - * Test that (for now) we can only add metrics from a set of known metrics. - */ - public void testUnknownMetricsRejected() { - String unknownMetric1 = "unknown_metric1"; - String unknownMetric2 = "unknown_metric2"; - Set unknownMetrics = new HashSet<>(); - unknownMetrics.add(unknownMetric1); - unknownMetrics.addAll(randomSubsetOf(NodesStatsRequestParameters.Metric.allMetrics())); - - NodesStatsRequest request = new NodesStatsRequest(); - - IllegalStateException exception = expectThrows(IllegalStateException.class, () -> request.addMetric(unknownMetric1)); - assertThat(exception.getMessage(), equalTo("Used an illegal metric: " + unknownMetric1)); - - exception = expectThrows(IllegalStateException.class, () -> request.removeMetric(unknownMetric1)); - assertThat(exception.getMessage(), equalTo("Used an illegal metric: " + unknownMetric1)); - - exception = expectThrows(IllegalStateException.class, () -> request.addMetrics(unknownMetrics.toArray(String[]::new))); - assertThat(exception.getMessage(), equalTo("Used illegal metric: [" + unknownMetric1 + "]")); - - unknownMetrics.add(unknownMetric2); - exception = expectThrows(IllegalStateException.class, () -> request.addMetrics(unknownMetrics.toArray(String[]::new))); - assertThat(exception.getMessage(), equalTo("Used illegal metrics: [" + unknownMetric1 + ", " + unknownMetric2 + "]")); - } - /** * Serialize and deserialize a request. * @param request A request to serialize. @@ -145,7 +119,7 @@ private static void assertRequestsEqual(NodesStatsRequest request1, NodesStatsRe public void testGetDescription() { final var request = new NodesStatsRequest("nodeid1", "nodeid2"); request.clear(); - request.addMetrics(NodesStatsRequestParameters.Metric.OS.metricName(), NodesStatsRequestParameters.Metric.TRANSPORT.metricName()); + request.addMetrics(Metric.OS, Metric.TRANSPORT); request.indices(new CommonStatsFlags(CommonStatsFlags.Flag.Store, CommonStatsFlags.Flag.Flush)); final var description = request.getDescription(); @@ -154,9 +128,9 @@ public void testGetDescription() { allOf( containsString("nodeid1"), containsString("nodeid2"), - containsString(NodesStatsRequestParameters.Metric.OS.metricName()), - containsString(NodesStatsRequestParameters.Metric.TRANSPORT.metricName()), - not(containsString(NodesStatsRequestParameters.Metric.SCRIPT.metricName())), + containsString(Metric.OS.metricName()), + containsString(Metric.TRANSPORT.metricName()), + not(containsString(Metric.SCRIPT.metricName())), containsString(CommonStatsFlags.Flag.Store.toString()), containsString(CommonStatsFlags.Flag.Flush.toString()), not(containsString(CommonStatsFlags.Flag.FieldData.toString())) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java index ca885b081452b..66e84f9fe6c17 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java @@ -741,10 +741,8 @@ protected void taskOperation( } reachabilityChecker.checkReachable(); - PlainActionFuture.get( - fut -> testNodes[0].transportService.getTaskManager().cancelTaskAndDescendants(task, "test", false, fut), - 10, - TimeUnit.SECONDS + safeAwait( + (ActionListener l) -> testNodes[0].transportService.getTaskManager().cancelTaskAndDescendants(task, "test", false, l) ); reachabilityChecker.ensureUnreachable(); @@ -817,10 +815,8 @@ protected void taskOperation( reachabilityChecker.checkReachable(); } - PlainActionFuture.get( - fut -> testNodes[0].transportService.getTaskManager().cancelTaskAndDescendants(task, "test", false, fut), - 10, - TimeUnit.SECONDS + safeAwait( + (ActionListener l) -> testNodes[0].transportService.getTaskManager().cancelTaskAndDescendants(task, "test", false, l) ); reachabilityChecker.ensureUnreachable(); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterActionTests.java index d76bfc03e1d7f..a7541447a7491 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterActionTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.action.support.ActionFilter; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.node.VersionInformation; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamOutput; @@ -88,13 +87,9 @@ public void writeTo(StreamOutput out) throws IOException { null ); - IllegalArgumentException ex = expectThrows( + final var ex = asInstanceOf( IllegalArgumentException.class, - () -> PlainActionFuture.get( - future -> action.doExecute(null, request, future), - 10, - TimeUnit.SECONDS - ) + safeAwaitFailure(ResolveClusterActionResponse.class, listener -> action.doExecute(null, request, listener)) ); assertThat(ex.getMessage(), containsString("not compatible with version")); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryActionTests.java index 26061de632f55..8057b02b1b4cc 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryActionTests.java @@ -9,11 +9,10 @@ package org.elasticsearch.action.admin.indices.validate.query; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.test.ESSingleNodeTestCase; -import java.util.concurrent.TimeUnit; +import static org.hamcrest.Matchers.instanceOf; public class TransportValidateQueryActionTests extends ESSingleNodeTestCase { @@ -24,15 +23,14 @@ public class TransportValidateQueryActionTests extends ESSingleNodeTestCase { * them garbled together, or trying to write one after the channel had closed, etc. */ public void testListenerOnlyInvokedOnceWhenIndexDoesNotExist() { - expectThrows( - IndexNotFoundException.class, - () -> PlainActionFuture.get( - future -> client().admin() + assertThat( + safeAwaitFailure( + ValidateQueryResponse.class, + listener -> client().admin() .indices() - .validateQuery(new ValidateQueryRequest("non-existent-index"), ActionListener.assertOnce(future)), - 10, - TimeUnit.SECONDS - ) + .validateQuery(new ValidateQueryRequest("non-existent-index"), ActionListener.assertOnce(listener)) + ), + instanceOf(IndexNotFoundException.class) ); } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverterTests.java b/server/src/test/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverterTests.java index 0595f35a6a05b..85cedff9145be 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverterTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverterTests.java @@ -83,7 +83,7 @@ public void testFailureStoreDocumentConversion() throws Exception { assertThat(ObjectPath.eval("document.id", convertedRequest.sourceAsMap()), is(equalTo("1"))); assertThat(ObjectPath.eval("document.routing", convertedRequest.sourceAsMap()), is(equalTo("fake_routing"))); - assertThat(ObjectPath.eval("document.index", convertedRequest.sourceAsMap()), is(equalTo("original_index"))); + assertThat(ObjectPath.eval("document.index", convertedRequest.sourceAsMap()), is(equalTo(targetIndexName))); assertThat(ObjectPath.eval("document.source.key", convertedRequest.sourceAsMap()), is(equalTo("value"))); assertThat(ObjectPath.eval("error.type", convertedRequest.sourceAsMap()), is(equalTo("exception"))); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java index 3b18c541bd80c..1f54d8dd1edd5 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java @@ -51,6 +51,7 @@ import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool.Names; +import org.mockito.ArgumentCaptor; import java.io.IOException; import java.util.Collections; @@ -588,6 +589,7 @@ public void testUpdateRequestWithFailure() throws Exception { assertThat(failure.getStatus(), equalTo(RestStatus.INTERNAL_SERVER_ERROR)); } + @SuppressWarnings("unchecked") public void testUpdateRequestWithConflictFailure() throws Exception { IndexSettings indexSettings = new IndexSettings(indexMetadata(), Settings.EMPTY); int retries = randomInt(4); @@ -651,11 +653,14 @@ public void testUpdateRequestWithConflictFailure() throws Exception { assertThat(failure.getCause(), equalTo(err)); assertThat(failure.getStatus(), equalTo(RestStatus.CONFLICT)); - // we have set noParsedBytesToReport on the IndexRequest, like it happens with updates by script. - verify(documentParsingProvider, times(0)).newDocumentSizeObserver(); - verify(documentParsingProvider, times(0)).newFixedSizeDocumentObserver(any(Integer.class)); + // we have set 0 value on normalisedBytesParsed on the IndexRequest, like it happens with updates by script. + ArgumentCaptor argument = ArgumentCaptor.forClass(IndexRequest.class); + verify(documentParsingProvider, times(retries + 1)).newDocumentSizeObserver(argument.capture()); + IndexRequest value = argument.getValue(); + assertThat(value.getNormalisedBytesParsed(), equalTo(0L)); } + @SuppressWarnings("unchecked") public void testUpdateRequestWithSuccess() throws Exception { IndexSettings indexSettings = new IndexSettings(indexMetadata(), Settings.EMPTY); DocWriteRequest writeRequest = new UpdateRequest("index", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value"); @@ -715,8 +720,11 @@ public void testUpdateRequestWithSuccess() throws Exception { DocWriteResponse response = primaryResponse.getResponse(); assertThat(response.status(), equalTo(created ? RestStatus.CREATED : RestStatus.OK)); assertThat(response.getSeqNo(), equalTo(13L)); - verify(documentParsingProvider, times(0)).newDocumentSizeObserver(); - verify(documentParsingProvider, times(1)).newFixedSizeDocumentObserver(eq(100L)); + + ArgumentCaptor argument = ArgumentCaptor.forClass(IndexRequest.class); + verify(documentParsingProvider, times(1)).newDocumentSizeObserver(argument.capture()); + IndexRequest value = argument.getValue(); + assertThat(value.getNormalisedBytesParsed(), equalTo(100L)); } public void testUpdateWithDelete() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java index 9e80f73d4df4a..8d4017a756e48 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java @@ -8,22 +8,30 @@ package org.elasticsearch.action.bulk; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.ingest.SimulateIndexResponse; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.indices.EmptySystemIndices; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.index.IndexVersionUtils; @@ -35,9 +43,12 @@ import org.elasticsearch.xcontent.json.JsonXContent; import org.junit.After; import org.junit.Before; +import org.mockito.stubbing.Answer; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -47,7 +58,9 @@ import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TransportSimulateBulkActionTests extends ESTestCase { @@ -57,6 +70,7 @@ public class TransportSimulateBulkActionTests extends ESTestCase { private TransportService transportService; private ClusterService clusterService; private TestThreadPool threadPool; + private IndicesService indicesService; private TestTransportSimulateBulkAction bulkAction; @@ -70,7 +84,8 @@ class TestTransportSimulateBulkAction extends TransportSimulateBulkAction { null, new ActionFilters(Set.of()), new IndexingPressure(Settings.EMPTY), - EmptySystemIndices.INSTANCE + EmptySystemIndices.INSTANCE, + indicesService ); } } @@ -98,6 +113,7 @@ public void setUp() throws Exception { ); transportService.start(); transportService.acceptIncomingRequests(); + indicesService = mock(IndicesService.class); bulkAction = new TestTransportSimulateBulkAction(); } @@ -180,6 +196,157 @@ public void onFailure(Exception e) { assertThat(onResponseCalled.get(), equalTo(true)); } + public void testIndexDataWithValidation() throws IOException { + /* + * This test makes sure that we validate mappings if we're indexing into an index that exists. It simulates 3 cases: + * (1) An indexing request to a nonexistent index (the index is not in the cluster state) + * (2) An indexing request to an index with non-strict mappings, or an index request that is valid with respect to the mappings + * (the index is in the cluster state, but our mock indicesService.withTempIndexService() does not throw an exception) + * (3) An indexing request that is invalid with respect to the mappings (the index is in the cluster state, and our mock + * indicesService.withTempIndexService() throws an exception) + */ + Task task = mock(Task.class); // unused + BulkRequest bulkRequest = new SimulateBulkRequest((Map>) null); + int bulkItemCount = randomIntBetween(0, 200); + Map indicesMap = new HashMap<>(); + Metadata.Builder metadataBuilder = new Metadata.Builder(); + Set indicesWithInvalidMappings = new HashSet<>(); + for (int i = 0; i < bulkItemCount; i++) { + Map source = Map.of(randomAlphaOfLength(10), randomAlphaOfLength(5)); + IndexRequest indexRequest = new IndexRequest(randomAlphaOfLength(10)).id(randomAlphaOfLength(10)).source(source); + indexRequest.setListExecutedPipelines(true); + for (int j = 0; j < randomIntBetween(0, 10); j++) { + indexRequest.addPipeline(randomAlphaOfLength(12)); + } + bulkRequest.add(indexRequest); + // Now we randomly decide what we're going to simulate with requests to this index: + String indexName = indexRequest.index(); + switch (between(0, 2)) { + case 0 -> { + // Index does not exist, so we don't put it in the indicesMap + } + case 1 -> { + // Indices that have non-strict mappings, or we're sending valid requests for their mappings + indicesMap.put(indexName, newIndexMetadata(indexName)); + } + case 2 -> { + // Indices that we'll pretend to have sent invalid requests to + indicesWithInvalidMappings.add(indexName); + indicesMap.put(indexName, newIndexMetadata(indexName)); + } + default -> throw new AssertionError("Illegal branch"); + } + } + metadataBuilder.indices(indicesMap); + ClusterServiceUtils.setState(clusterService, new ClusterState.Builder(clusterService.state()).metadata(metadataBuilder)); + AtomicBoolean onResponseCalled = new AtomicBoolean(false); + ActionListener listener = new ActionListener<>() { + @Override + public void onResponse(BulkResponse response) { + onResponseCalled.set(true); + BulkItemResponse[] responseItems = response.getItems(); + assertThat(responseItems.length, equalTo(bulkItemCount)); + assertThat(responseItems.length, equalTo(bulkRequest.requests().size())); + for (int i = 0; i < responseItems.length; i++) { + BulkItemResponse responseItem = responseItems[i]; + IndexRequest indexRequest = (IndexRequest) bulkRequest.requests().get(i); + assertNull(responseItem.getFailure()); + assertThat(responseItem.getResponse(), instanceOf(SimulateIndexResponse.class)); + SimulateIndexResponse simulateIndexResponse = responseItem.getResponse(); + assertThat(simulateIndexResponse.getIndex(), equalTo(indexRequest.index())); + /* + * SimulateIndexResponse doesn't have an equals() method, and most of its state is private. So we check that + * its toXContent method produces the expected output. + */ + String output = Strings.toString(simulateIndexResponse); + try { + String indexName = indexRequest.index(); + if (indicesWithInvalidMappings.contains(indexName)) { + assertEquals( + XContentHelper.stripWhitespace( + Strings.format( + """ + { + "_id": "%s", + "_index": "%s", + "_version": -3, + "_source": %s, + "executed_pipelines": [%s], + "error":{"type":"exception","reason":"invalid mapping"} + }""", + indexRequest.id(), + indexName, + convertMapToJsonString(indexRequest.sourceAsMap()), + indexRequest.getExecutedPipelines() + .stream() + .map(pipeline -> "\"" + pipeline + "\"") + .collect(Collectors.joining(",")) + ) + ), + output + ); + } else { + /* + * Anything else (a non-existent index, a request to an index without strict mappings, or a valid request) + * results in no error being reported. + */ + assertEquals( + XContentHelper.stripWhitespace( + Strings.format( + """ + { + "_id": "%s", + "_index": "%s", + "_version": -3, + "_source": %s, + "executed_pipelines": [%s] + }""", + indexRequest.id(), + indexName, + convertMapToJsonString(indexRequest.sourceAsMap()), + indexRequest.getExecutedPipelines() + .stream() + .map(pipeline -> "\"" + pipeline + "\"") + .collect(Collectors.joining(",")) + ) + ), + output + ); + } + } catch (IOException e) { + fail(e); + } + } + } + + @Override + public void onFailure(Exception e) { + fail(e, "Unexpected error"); + } + }; + when(indicesService.withTempIndexService(any(), any())).thenAnswer((Answer) invocation -> { + IndexMetadata imd = invocation.getArgument(0); + if (indicesWithInvalidMappings.contains(imd.getIndex().getName())) { + throw new ElasticsearchException("invalid mapping"); + } else { + // we don't actually care what is returned, as long as no exception is thrown the request is considered valid: + return null; + } + }); + bulkAction.doInternalExecute(task, bulkRequest, r -> fail("executor is unused"), listener, randomLongBetween(0, Long.MAX_VALUE)); + assertThat(onResponseCalled.get(), equalTo(true)); + } + + private IndexMetadata newIndexMetadata(String indexName) { + Settings dummyIndexSettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) + .build(); + return new IndexMetadata.Builder(indexName).settings(dummyIndexSettings).build(); + } + private String convertMapToJsonString(Map map) throws IOException { try (XContentBuilder builder = JsonXContent.contentBuilder().map(map)) { return BytesReference.bytes(builder).utf8ToString(); diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesActionTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesActionTests.java index 6b0fea6271f5e..6346950f0af2a 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesActionTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.action.support.ActionFilter; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.node.VersionInformation; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamOutput; @@ -45,7 +44,7 @@ public void tearDown() throws Exception { ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS); } - public void testCCSCompatibilityCheck() throws Exception { + public void testCCSCompatibilityCheck() { Settings settings = Settings.builder() .put("node.name", TransportFieldCapabilitiesActionTests.class.getSimpleName()) .put(SearchService.CCS_VERSION_CHECK_SETTING.getKey(), "true") @@ -87,13 +86,9 @@ protected void doWriteTo(StreamOutput out) throws IOException { null ); - IllegalArgumentException ex = expectThrows( + IllegalArgumentException ex = asInstanceOf( IllegalArgumentException.class, - () -> PlainActionFuture.get( - future -> action.doExecute(null, fieldCapsRequest, future), - 10, - TimeUnit.SECONDS - ) + safeAwaitFailure(FieldCapabilitiesResponse.class, l -> action.doExecute(null, fieldCapsRequest, l)) ); assertThat( diff --git a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java index 607d83d4aab31..97ff852096952 100644 --- a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java @@ -82,7 +82,9 @@ private AbstractSearchAsyncAction createAction( null, request, listener, - new GroupShardsIterator<>(Collections.singletonList(new SearchShardIterator(null, null, Collections.emptyList(), null))), + new GroupShardsIterator<>( + Collections.singletonList(new SearchShardIterator(null, new ShardId("index", "_na", 0), Collections.emptyList(), null)) + ), timeProvider, ClusterState.EMPTY_STATE, null, diff --git a/server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java index 8889df78eb1d6..a59bf8bfaf9f5 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java @@ -120,7 +120,7 @@ public void sendFreeContext( if (freed) { numFreed.incrementAndGet(); } - Thread t = new Thread(() -> listener.onResponse(new SearchFreeContextResponse(freed))); + Thread t = new Thread(() -> listener.onResponse(SearchFreeContextResponse.of(freed))); t.start(); } @@ -200,7 +200,7 @@ public void sendFreeContext( if (freed) { numFreed.incrementAndGet(); } - listener.onResponse(new SearchFreeContextResponse(freed)); + listener.onResponse(SearchFreeContextResponse.of(freed)); } }); t.start(); diff --git a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java index e9ff8336ef4c9..47dbe8f126556 100644 --- a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java @@ -13,6 +13,7 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHits; import org.apache.lucene.tests.store.MockDirectoryWrapper; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.NoopCircuitBreaker; import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore; @@ -35,6 +36,7 @@ import org.elasticsearch.search.rank.TestRankBuilder; import org.elasticsearch.search.vectors.KnnScoreDocQueryBuilder; import org.elasticsearch.search.vectors.KnnSearchBuilder; +import org.elasticsearch.search.vectors.VectorData; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.InternalAggregationTestCase; import org.elasticsearch.transport.Transport; @@ -77,7 +79,7 @@ public void sendExecuteQuery( Transport.Connection connection, QuerySearchRequest request, SearchTask task, - SearchActionListener listener + ActionListener listener ) { if (request.contextId().getId() == 1) { QuerySearchResult queryResult = new QuerySearchResult( @@ -179,7 +181,7 @@ public void sendExecuteQuery( Transport.Connection connection, QuerySearchRequest request, SearchTask task, - SearchActionListener listener + ActionListener listener ) { if (request.contextId().getId() == 1) { QuerySearchResult queryResult = new QuerySearchResult( @@ -266,7 +268,7 @@ public void sendExecuteQuery( Transport.Connection connection, QuerySearchRequest request, SearchTask task, - SearchActionListener listener + ActionListener listener ) { if (request.contextId().getId() == 1) { QuerySearchResult queryResult = new QuerySearchResult( @@ -351,12 +353,12 @@ public void testRewriteShardSearchRequestWithRank() { KnnScoreDocQueryBuilder ksdqb0 = new KnnScoreDocQueryBuilder( new ScoreDoc[] { new ScoreDoc(1, 3.0f, 1), new ScoreDoc(4, 1.5f, 1) }, "vector", - new float[] { 0.0f } + VectorData.fromFloats(new float[] { 0.0f }) ); KnnScoreDocQueryBuilder ksdqb1 = new KnnScoreDocQueryBuilder( new ScoreDoc[] { new ScoreDoc(1, 2.0f, 1) }, "vector2", - new float[] { 0.0f } + VectorData.fromFloats(new float[] { 0.0f }) ); assertEquals( List.of(bm25, ksdqb0, ksdqb1), diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java index 60e334704f1fa..54c98a8b72d7e 100644 --- a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java @@ -11,6 +11,7 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHits; import org.apache.lucene.tests.store.MockDirectoryWrapper; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.NoopCircuitBreaker; @@ -201,7 +202,7 @@ public void sendExecuteFetch( Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task, - SearchActionListener listener + ActionListener listener ) { FetchSearchResult fetchResult = new FetchSearchResult(); try { @@ -317,7 +318,7 @@ public void sendExecuteFetch( Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task, - SearchActionListener listener + ActionListener listener ) { if (request.contextId().getId() == 321) { FetchSearchResult fetchResult = new FetchSearchResult(); @@ -426,7 +427,7 @@ public void sendExecuteFetch( Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task, - SearchActionListener listener + ActionListener listener ) { new Thread(() -> { FetchSearchResult fetchResult = new FetchSearchResult(); @@ -562,7 +563,7 @@ public void sendExecuteFetch( Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task, - SearchActionListener listener + ActionListener listener ) { FetchSearchResult fetchResult = new FetchSearchResult(); try { @@ -667,7 +668,7 @@ public void sendExecuteFetch( Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task, - SearchActionListener listener + ActionListener listener ) { FetchSearchResult fetchResult = new FetchSearchResult(); try { diff --git a/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java index af0ce461e9486..11903f00d35f6 100644 --- a/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java @@ -106,7 +106,7 @@ public void sendExecuteRankFeature( Transport.Connection connection, final RankFeatureShardRequest request, SearchTask task, - final SearchActionListener listener + final ActionListener listener ) { // make sure to match the context id generated above, otherwise we throw if (request.contextId().getId() == 123 && Arrays.equals(request.getDocIds(), new int[] { 1, 2 })) { @@ -212,7 +212,7 @@ public void sendExecuteRankFeature( Transport.Connection connection, final RankFeatureShardRequest request, SearchTask task, - final SearchActionListener listener + final ActionListener listener ) { // make sure to match the context id generated above, otherwise we throw // first shard @@ -327,7 +327,7 @@ public void sendExecuteRankFeature( Transport.Connection connection, final RankFeatureShardRequest request, SearchTask task, - final SearchActionListener listener + final ActionListener listener ) { // make sure to match the context id generated above, otherwise we throw if (request.contextId().getId() == 123 && Arrays.equals(request.getDocIds(), new int[] { 1, 2 })) { @@ -419,7 +419,7 @@ public void sendExecuteRankFeature( Transport.Connection connection, final RankFeatureShardRequest request, SearchTask task, - final SearchActionListener listener + final ActionListener listener ) { // make sure to match the context id generated above, otherwise we throw // first shard @@ -511,7 +511,7 @@ public void sendExecuteRankFeature( Transport.Connection connection, final RankFeatureShardRequest request, SearchTask task, - final SearchActionListener listener + final ActionListener listener ) { // make sure to match the context id generated above, otherwise we throw if (request.contextId().getId() == 123 && Arrays.equals(request.getDocIds(), new int[] { 1, 2 })) { @@ -637,7 +637,7 @@ public void sendExecuteRankFeature( Transport.Connection connection, final RankFeatureShardRequest request, SearchTask task, - final SearchActionListener listener + final ActionListener listener ) { RankFeatureResult rankFeatureResult = new RankFeatureResult(); @@ -779,7 +779,7 @@ public void sendExecuteRankFeature( Transport.Connection connection, final RankFeatureShardRequest request, SearchTask task, - final SearchActionListener listener + final ActionListener listener ) { RankFeatureResult rankFeatureResult = new RankFeatureResult(); // make sure to match the context id generated above, otherwise we throw diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java index 118a7055cd782..ed02328d388b6 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java @@ -748,43 +748,34 @@ public void testConsumerConcurrently() throws Exception { ) ) { AtomicInteger max = new AtomicInteger(); - Thread[] threads = new Thread[expectedNumResults]; CountDownLatch latch = new CountDownLatch(expectedNumResults); - for (int i = 0; i < expectedNumResults; i++) { - int id = i; - threads[i] = new Thread(() -> { - int number = randomIntBetween(1, 1000); - max.updateAndGet(prev -> Math.max(prev, number)); - QuerySearchResult result = new QuerySearchResult( - new ShardSearchContextId("", id), - new SearchShardTarget("node", new ShardId("a", "b", id), null), - null + runInParallel(expectedNumResults, id -> { + int number = randomIntBetween(1, 1000); + max.updateAndGet(prev -> Math.max(prev, number)); + QuerySearchResult result = new QuerySearchResult( + new ShardSearchContextId("", id), + new SearchShardTarget("node", new ShardId("a", "b", id), null), + null + ); + try { + result.topDocs( + new TopDocsAndMaxScore( + new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] { new ScoreDoc(0, number) }), + number + ), + new DocValueFormat[0] ); - try { - result.topDocs( - new TopDocsAndMaxScore( - new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] { new ScoreDoc(0, number) }), - number - ), - new DocValueFormat[0] - ); - InternalAggregations aggs = InternalAggregations.from( - Collections.singletonList(new Max("test", (double) number, DocValueFormat.RAW, Collections.emptyMap())) - ); - result.aggregations(aggs); - result.setShardIndex(id); - result.size(1); - consumer.consumeResult(result, latch::countDown); - } finally { - result.decRef(); - } - - }); - threads[i].start(); - } - for (int i = 0; i < expectedNumResults; i++) { - threads[i].join(); - } + InternalAggregations aggs = InternalAggregations.from( + Collections.singletonList(new Max("test", (double) number, DocValueFormat.RAW, Collections.emptyMap())) + ); + result.aggregations(aggs); + result.setShardIndex(id); + result.size(1); + consumer.consumeResult(result, latch::countDown); + } finally { + result.decRef(); + } + }); latch.await(); SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce(); @@ -1264,42 +1255,34 @@ public void onFinalReduce(List shards, TotalHits totalHits, Interna ) ) { AtomicInteger max = new AtomicInteger(); - Thread[] threads = new Thread[expectedNumResults]; CountDownLatch latch = new CountDownLatch(expectedNumResults); - for (int i = 0; i < expectedNumResults; i++) { - int id = i; - threads[i] = new Thread(() -> { - int number = randomIntBetween(1, 1000); - max.updateAndGet(prev -> Math.max(prev, number)); - QuerySearchResult result = new QuerySearchResult( - new ShardSearchContextId("", id), - new SearchShardTarget("node", new ShardId("a", "b", id), null), - null + runInParallel(expectedNumResults, id -> { + int number = randomIntBetween(1, 1000); + max.updateAndGet(prev -> Math.max(prev, number)); + QuerySearchResult result = new QuerySearchResult( + new ShardSearchContextId("", id), + new SearchShardTarget("node", new ShardId("a", "b", id), null), + null + ); + try { + result.topDocs( + new TopDocsAndMaxScore( + new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] { new ScoreDoc(0, number) }), + number + ), + new DocValueFormat[0] ); - try { - result.topDocs( - new TopDocsAndMaxScore( - new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] { new ScoreDoc(0, number) }), - number - ), - new DocValueFormat[0] - ); - InternalAggregations aggs = InternalAggregations.from( - Collections.singletonList(new Max("test", (double) number, DocValueFormat.RAW, Collections.emptyMap())) - ); - result.aggregations(aggs); - result.setShardIndex(id); - result.size(1); - consumer.consumeResult(result, latch::countDown); - } finally { - result.decRef(); - } - }); - threads[i].start(); - } - for (int i = 0; i < expectedNumResults; i++) { - threads[i].join(); - } + InternalAggregations aggs = InternalAggregations.from( + Collections.singletonList(new Max("test", (double) number, DocValueFormat.RAW, Collections.emptyMap())) + ); + result.aggregations(aggs); + result.setShardIndex(id); + result.size(1); + consumer.consumeResult(result, latch::countDown); + } finally { + result.decRef(); + } + }); latch.await(); SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce(); assertAggReduction(request); @@ -1354,39 +1337,31 @@ private void testReduceCase(int numShards, int bufferSize, boolean shouldFail) t ) ) { CountDownLatch latch = new CountDownLatch(numShards); - Thread[] threads = new Thread[numShards]; - for (int i = 0; i < numShards; i++) { - final int index = i; - threads[index] = new Thread(() -> { - QuerySearchResult result = new QuerySearchResult( - new ShardSearchContextId(UUIDs.randomBase64UUID(), index), - new SearchShardTarget("node", new ShardId("a", "b", index), null), - null + runInParallel(numShards, index -> { + QuerySearchResult result = new QuerySearchResult( + new ShardSearchContextId(UUIDs.randomBase64UUID(), index), + new SearchShardTarget("node", new ShardId("a", "b", index), null), + null + ); + try { + result.topDocs( + new TopDocsAndMaxScore( + new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), Lucene.EMPTY_SCORE_DOCS), + Float.NaN + ), + new DocValueFormat[0] ); - try { - result.topDocs( - new TopDocsAndMaxScore( - new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), Lucene.EMPTY_SCORE_DOCS), - Float.NaN - ), - new DocValueFormat[0] - ); - InternalAggregations aggs = InternalAggregations.from( - Collections.singletonList(new Max("test", 0d, DocValueFormat.RAW, Collections.emptyMap())) - ); - result.aggregations(aggs); - result.setShardIndex(index); - result.size(1); - consumer.consumeResult(result, latch::countDown); - } finally { - result.decRef(); - } - }); - threads[index].start(); - } - for (int i = 0; i < numShards; i++) { - threads[i].join(); - } + InternalAggregations aggs = InternalAggregations.from( + Collections.singletonList(new Max("test", 0d, DocValueFormat.RAW, Collections.emptyMap())) + ); + result.aggregations(aggs); + result.setShardIndex(index); + result.size(1); + consumer.consumeResult(result, latch::countDown); + } finally { + result.decRef(); + } + }); latch.await(); if (shouldFail) { if (shouldFailPartial == false) { diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java index aa55c7176f22a..06967a26cd514 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java @@ -103,7 +103,7 @@ public void sendExecuteQuery( Transport.Connection connection, ShardSearchRequest request, SearchTask task, - SearchActionListener listener + ActionListener listener ) { int shardId = request.shardId().id(); if (request.canReturnNullResponseIfMatchNoDocs()) { @@ -447,7 +447,7 @@ public void sendExecuteQuery( Transport.Connection connection, ShardSearchRequest request, SearchTask task, - SearchActionListener listener + ActionListener listener ) { int shardId = request.shardId().id(); QuerySearchResult queryResult = new QuerySearchResult( @@ -598,7 +598,7 @@ public void sendExecuteQuery( Transport.Connection connection, ShardSearchRequest request, SearchTask task, - SearchActionListener listener + ActionListener listener ) { int shardId = request.shardId().id(); QuerySearchResult queryResult = new QuerySearchResult( diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java index 41e7a5c8ad1e1..e3973e7fc2575 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java @@ -62,7 +62,7 @@ public void testSendRequestsToNodes() throws InterruptedException { protected void executeInitialPhase( Transport.Connection connection, InternalScrollSearchRequest internalRequest, - SearchActionListener searchActionListener + ActionListener searchActionListener ) { new Thread(() -> { SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult = new SearchAsyncActionTests.TestSearchPhaseResult( @@ -159,7 +159,7 @@ public void onFailure(Exception e) { protected void executeInitialPhase( Transport.Connection connection, InternalScrollSearchRequest internalRequest, - SearchActionListener searchActionListener + ActionListener searchActionListener ) { new Thread(() -> { SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult = new SearchAsyncActionTests.TestSearchPhaseResult( @@ -232,7 +232,7 @@ public void testNodeNotAvailable() throws InterruptedException { protected void executeInitialPhase( Transport.Connection connection, InternalScrollSearchRequest internalRequest, - SearchActionListener searchActionListener + ActionListener searchActionListener ) { try { assertNotEquals("node2 is not available", "node2", connection.getNode().getId()); @@ -317,7 +317,7 @@ public void testShardFailures() throws InterruptedException { protected void executeInitialPhase( Transport.Connection connection, InternalScrollSearchRequest internalRequest, - SearchActionListener searchActionListener + ActionListener searchActionListener ) { new Thread(() -> { if (internalRequest.contextId().getId() == 17) { @@ -420,7 +420,7 @@ public void onFailure(Exception e) { protected void executeInitialPhase( Transport.Connection connection, InternalScrollSearchRequest internalRequest, - SearchActionListener searchActionListener + ActionListener searchActionListener ) { new Thread(() -> searchActionListener.onFailure(new IllegalArgumentException("BOOM on shard"))).start(); } diff --git a/server/src/test/java/org/elasticsearch/action/support/ThreadedActionListenerTests.java b/server/src/test/java/org/elasticsearch/action/support/ThreadedActionListenerTests.java index b5f07b7d8d087..36113d9aa931d 100644 --- a/server/src/test/java/org/elasticsearch/action/support/ThreadedActionListenerTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/ThreadedActionListenerTests.java @@ -131,16 +131,16 @@ public void testToString() { assertEquals( "ThreadedActionListener[DeterministicTaskQueue/forkingExecutor/NoopActionListener]/onResponse", - PlainActionFuture.get(future -> new ThreadedActionListener(deterministicTaskQueue.getThreadPool(s -> { - future.onResponse(s.toString()); + safeAwait(listener -> new ThreadedActionListener(deterministicTaskQueue.getThreadPool(s -> { + listener.onResponse(s.toString()); return s; }).generic(), randomBoolean(), ActionListener.noop()).onResponse(null)) ); assertEquals( "ThreadedActionListener[DeterministicTaskQueue/forkingExecutor/NoopActionListener]/onFailure", - PlainActionFuture.get(future -> new ThreadedActionListener(deterministicTaskQueue.getThreadPool(s -> { - future.onResponse(s.toString()); + safeAwait(listener -> new ThreadedActionListener(deterministicTaskQueue.getThreadPool(s -> { + listener.onResponse(s.toString()); return s; }).generic(), randomBoolean(), ActionListener.noop()).onFailure(new ElasticsearchException("test"))) ); diff --git a/server/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainRefCountingTests.java b/server/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainRefCountingTests.java index a199003fc59c4..f4fa42bd1204f 100644 --- a/server/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainRefCountingTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainRefCountingTests.java @@ -152,7 +152,7 @@ public static class TestAction extends TransportAction { @Inject public TestAction(TransportService transportService, ActionFilters actionFilters) { - super(TYPE.name(), actionFilters, transportService.getTaskManager()); + super(TYPE.name(), actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); threadPool = transportService.getThreadPool(); } diff --git a/server/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java b/server/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java index 82c204b1d0b88..f793255f3b98d 100644 --- a/server/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.node.Node; import org.elasticsearch.tasks.Task; @@ -79,7 +80,8 @@ public void testActionFiltersRequest() throws InterruptedException { TransportAction transportAction = new TransportAction( actionName, actionFilters, - new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()) + new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()), + EsExecutors.DIRECT_EXECUTOR_SERVICE ) { @Override protected void doExecute(Task task, TestRequest request, ActionListener listener) { @@ -165,7 +167,8 @@ public void exe TransportAction transportAction = new TransportAction( actionName, actionFilters, - new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()) + new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()), + EsExecutors.DIRECT_EXECUTOR_SERVICE ) { @Override protected void doExecute(Task task, TestRequest request, ActionListener listener) { diff --git a/server/src/test/java/org/elasticsearch/action/support/TransportActionTests.java b/server/src/test/java/org/elasticsearch/action/support/TransportActionTests.java new file mode 100644 index 0000000000000..97fa537874397 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/support/TransportActionTests.java @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.support; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.node.Node; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.telemetry.metric.MeterRegistry; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.util.Collections; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.not; + +public class TransportActionTests extends ESTestCase { + + private ThreadPool threadPool; + + @Before + public void init() throws Exception { + threadPool = new ThreadPool( + Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "TransportActionTests").build(), + MeterRegistry.NOOP + ); + } + + @After + public void shutdown() throws Exception { + terminate(threadPool); + } + + public void testDirectExecuteRunsOnCallingThread() throws ExecutionException, InterruptedException { + + String actionName = randomAlphaOfLength(randomInt(30)); + var testExecutor = new Executor() { + @Override + public void execute(Runnable command) { + fail("executeDirect should not run a TransportAction on a different executor"); + } + }; + + var transportAction = getTestTransportAction(actionName, testExecutor); + + PlainActionFuture future = new PlainActionFuture<>(); + + transportAction.executeDirect(null, new TestRequest(), future); + + var response = future.get(); + assertThat(response, notNullValue()); + assertThat(response.executingThreadName, equalTo(Thread.currentThread().getName())); + assertThat(response.executingThreadId, equalTo(Thread.currentThread().getId())); + } + + public void testExecuteRunsOnExecutor() throws ExecutionException, InterruptedException { + + String actionName = randomAlphaOfLength(randomInt(30)); + + boolean[] executedOnExecutor = new boolean[1]; + var testExecutor = new Executor() { + @Override + public void execute(Runnable command) { + command.run(); + executedOnExecutor[0] = true; + } + }; + + var transportAction = getTestTransportAction(actionName, testExecutor); + + PlainActionFuture future = new PlainActionFuture<>(); + + ActionTestUtils.execute(transportAction, null, new TestRequest(), future); + + var response = future.get(); + assertThat(response, notNullValue()); + assertTrue(executedOnExecutor[0]); + } + + public void testExecuteWithGenericExecutorRunsOnDifferentThread() throws ExecutionException, InterruptedException { + + String actionName = randomAlphaOfLength(randomInt(30)); + var transportAction = getTestTransportAction(actionName, threadPool.executor(ThreadPool.Names.GENERIC)); + + PlainActionFuture future = new PlainActionFuture<>(); + + ActionTestUtils.execute(transportAction, null, new TestRequest(), future); + + var response = future.get(); + assertThat(response, notNullValue()); + assertThat(response.executingThreadName, not(equalTo(Thread.currentThread().getName()))); + assertThat(response.executingThreadName, containsString("[generic]")); + assertThat(response.executingThreadId, not(equalTo(Thread.currentThread().getId()))); + } + + public void testExecuteWithDirectExecutorRunsOnCallingThread() throws ExecutionException, InterruptedException { + + String actionName = randomAlphaOfLength(randomInt(30)); + var transportAction = getTestTransportAction(actionName, EsExecutors.DIRECT_EXECUTOR_SERVICE); + + PlainActionFuture future = new PlainActionFuture<>(); + + ActionTestUtils.execute(transportAction, null, new TestRequest(), future); + + var response = future.get(); + assertThat(response, notNullValue()); + assertThat(response, notNullValue()); + assertThat(response.executingThreadName, equalTo(Thread.currentThread().getName())); + assertThat(response.executingThreadId, equalTo(Thread.currentThread().getId())); + } + + private TransportAction getTestTransportAction(String actionName, Executor executor) { + ActionFilters actionFilters = new ActionFilters(Collections.emptySet()); + TransportAction transportAction = new TransportAction<>( + actionName, + actionFilters, + new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()), + executor + ) { + @Override + protected void doExecute(Task task, TestRequest request, ActionListener listener) { + listener.onResponse(new TestResponse(Thread.currentThread().getName(), Thread.currentThread().getId())); + } + }; + return transportAction; + } + + private static class TestRequest extends ActionRequest { + @Override + public ActionRequestValidationException validate() { + return null; + } + } + + private static class TestResponse extends ActionResponse { + + private final String executingThreadName; + private final long executingThreadId; + + TestResponse(String executingThreadName, long executingThreadId) { + this.executingThreadName = executingThreadName; + this.executingThreadId = executingThreadId; + } + + @Override + public void writeTo(StreamOutput out) {} + } +} diff --git a/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java index 9748fb4a0d422..359f13a19fabe 100644 --- a/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java @@ -246,11 +246,7 @@ public String[] concreteIndexNames(ClusterState state, IndicesRequest request) { private static final String TEST_THREAD_POOL_NAME = "test_thread_pool"; private static void awaitForkedTasks() { - PlainActionFuture.get( - listener -> THREAD_POOL.executor(TEST_THREAD_POOL_NAME).execute(ActionRunnable.run(listener, () -> {})), - 10, - TimeUnit.SECONDS - ); + safeAwait(listener -> THREAD_POOL.executor(TEST_THREAD_POOL_NAME).execute(ActionRunnable.run(listener, () -> {}))); } @BeforeClass @@ -347,13 +343,9 @@ public void testGlobalBlock() { assertEquals( "blocked by: [SERVICE_UNAVAILABLE/1/test-block];", - expectThrows( + asInstanceOf( ClusterBlockException.class, - () -> PlainActionFuture.get( - listener -> action.doExecute(null, request, listener), - 10, - TimeUnit.SECONDS - ) + safeAwaitFailure(Response.class, listener -> action.doExecute(null, request, listener)) ).getMessage() ); } @@ -369,13 +361,9 @@ public void testRequestBlock() { setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block)); assertEquals( "index [" + TEST_INDEX + "] blocked by: [SERVICE_UNAVAILABLE/1/test-block];", - expectThrows( + asInstanceOf( ClusterBlockException.class, - () -> PlainActionFuture.get( - listener -> action.doExecute(null, request, listener), - 10, - TimeUnit.SECONDS - ) + safeAwaitFailure(Response.class, listener -> action.doExecute(null, request, listener)) ).getMessage() ); } diff --git a/server/src/test/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableActionTests.java b/server/src/test/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableActionTests.java index 7eccd65dfea8c..e8f75fcc6cdde 100644 --- a/server/src/test/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; @@ -323,13 +324,14 @@ public void testInvalidNodes() throws Exception { // The request fails if we don't mark shards as stale assertThat( - expectThrows(NodeNotConnectedException.class, () -> brodcastUnpromotableRequest(wrongRoutingTable, false)).toString(), + asInstanceOf(NodeNotConnectedException.class, safeAwaitFailure(broadcastUnpromotableRequest(wrongRoutingTable, false))) + .toString(), containsString("discovery node must not be null") ); Mockito.verifyNoInteractions(shardStateAction); // We were able to mark shards as stale, so the request finishes successfully - assertThat(brodcastUnpromotableRequest(wrongRoutingTable, true), equalTo(ActionResponse.Empty.INSTANCE)); + assertThat(safeAwait(broadcastUnpromotableRequest(wrongRoutingTable, true)), equalTo(ActionResponse.Empty.INSTANCE)); for (var shardRouting : wrongRoutingTable.unpromotableShards()) { Mockito.verify(shardStateAction) .remoteShardFailed( @@ -354,40 +356,35 @@ public void testInvalidNodes() throws Exception { .when(shardStateAction) .remoteShardFailed(any(ShardId.class), anyString(), anyLong(), anyBoolean(), anyString(), any(Exception.class), any()); assertThat( - expectThrows(NodeNotConnectedException.class, () -> brodcastUnpromotableRequest(wrongRoutingTable, true)).toString(), + asInstanceOf(NodeNotConnectedException.class, safeAwaitFailure(broadcastUnpromotableRequest(wrongRoutingTable, true))) + .toString(), containsString("discovery node must not be null") ); } - private ActionResponse brodcastUnpromotableRequest(IndexShardRoutingTable wrongRoutingTable, boolean failShardOnError) - throws Exception { - return PlainActionFuture.get( - f -> ActionTestUtils.execute( + private SubscribableListener broadcastUnpromotableRequest( + IndexShardRoutingTable wrongRoutingTable, + boolean failShardOnError + ) { + return SubscribableListener.newForked( + listener -> ActionTestUtils.execute( broadcastUnpromotableAction, null, new TestBroadcastUnpromotableRequest(wrongRoutingTable, failShardOnError), - f - ), - 10, - TimeUnit.SECONDS + listener + ) ); } public void testNullIndexShardRoutingTable() { - IndexShardRoutingTable shardRoutingTable = null; assertThat( - expectThrows( NullPointerException.class, - () -> PlainActionFuture.get( - f -> ActionTestUtils.execute( - broadcastUnpromotableAction, - null, - new TestBroadcastUnpromotableRequest(shardRoutingTable), - f - ), - 10, - TimeUnit.SECONDS + () -> ActionTestUtils.execute( + broadcastUnpromotableAction, + null, + new TestBroadcastUnpromotableRequest((IndexShardRoutingTable) null), + ActionListener.running(ESTestCase::fail) ) ).toString(), containsString("index shard routing table is null") diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java index fcbddb581946b..04ad7d410e9b0 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java @@ -144,7 +144,7 @@ public static > R resolveRequest(TransportReques private static ThreadPool threadPool; - private boolean forceExecute; + private TransportReplicationAction.PrimaryActionExecution primaryActionExecution; private ClusterService clusterService; private TransportService transportService; private CapturingTransport transport; @@ -165,7 +165,7 @@ public static void beforeClass() { @Before public void setUp() throws Exception { super.setUp(); - forceExecute = randomBoolean(); + primaryActionExecution = randomFrom(TransportReplicationAction.PrimaryActionExecution.values()); transport = new CapturingTransport(); clusterService = createClusterService(threadPool); transportService = transport.createTransportService( @@ -951,7 +951,7 @@ public void testSeqNoIsSetOnPrimary() { ActionListener argument = (ActionListener) invocation.getArguments()[0]; argument.onResponse(count::decrementAndGet); return null; - }).when(shard).acquirePrimaryOperationPermit(any(), any(Executor.class), eq(forceExecute)); + }).when(shard).acquirePrimaryOperationPermit(any(), any(Executor.class), eq(shouldForceAcquirePermit(primaryActionExecution))); when(shard.getActiveOperationsCount()).thenAnswer(i -> count.get()); final IndexService indexService = mock(IndexService.class); @@ -979,6 +979,13 @@ public void testSeqNoIsSetOnPrimary() { assertThat(shardRequest.getPrimaryTerm(), equalTo(primaryTerm)); } + private boolean shouldForceAcquirePermit(TransportReplicationAction.PrimaryActionExecution primaryActionExecution) { + return switch (primaryActionExecution) { + case Force -> true; + case RejectOnOverload -> false; + }; + } + public void testCounterOnPrimary() throws Exception { final String index = "test"; final ShardId shardId = new ShardId(index, "_na_", 0); @@ -1511,8 +1518,8 @@ private class TestAction extends TransportReplicationAction isPrimaryMode.get()); doAnswer(invocation -> { long term = (Long) invocation.getArguments()[0]; diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java index d0ae26f97917a..c5642fd9681ac 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java @@ -456,7 +456,9 @@ private abstract class TestAction extends TransportReplicationAction()), Request::new, Request::new, - EsExecutors.DIRECT_EXECUTOR_SERVICE + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SyncGlobalCheckpointAfterOperation.DoNotSync, + PrimaryActionExecution.RejectOnOverload ); this.shardId = Objects.requireNonNull(shardId); this.primary = Objects.requireNonNull(primary); diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java index 340ca87968db0..5cc0e55942818 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.action.support.WriteResponse; import org.elasticsearch.action.support.replication.ReplicationOperation.ReplicaResponse; @@ -253,20 +254,15 @@ public void testReplicaWaitForRefresh() throws Exception { } public void testDocumentFailureInShardOperationOnPrimary() { - assertEquals( - "simulated", - expectThrows( - RuntimeException.class, - () -> PlainActionFuture.get( - (PlainActionFuture> future) -> new TestAction( - true, - randomBoolean() - ).dispatchedShardOperationOnPrimary(new TestRequest(), indexShard, future), - 0, - TimeUnit.SECONDS - ) - ).getMessage() + final var listener = SubscribableListener.newForked( + l -> new TestAction(true, randomBoolean()).dispatchedShardOperationOnPrimary( + new TestRequest(), + indexShard, + ActionTestUtils.assertNoSuccessListener(l::onResponse) + ) ); + assertTrue(listener.isDone()); + assertEquals("simulated", asInstanceOf(RuntimeException.class, safeAwait(listener)).getMessage()); } public void testDocumentFailureInShardOperationOnReplica() throws Exception { @@ -430,7 +426,7 @@ protected TestAction(boolean withDocumentFailureOnPrimary, boolean withDocumentF TestRequest::new, TestRequest::new, (service, ignore) -> EsExecutors.DIRECT_EXECUTOR_SERVICE, - false, + PrimaryActionExecution.RejectOnOverload, new IndexingPressure(Settings.EMPTY), EmptySystemIndices.INSTANCE ); @@ -458,7 +454,7 @@ protected TestAction( TestRequest::new, TestRequest::new, (service, ignore) -> EsExecutors.DIRECT_EXECUTOR_SERVICE, - false, + PrimaryActionExecution.RejectOnOverload, new IndexingPressure(settings), EmptySystemIndices.INSTANCE ); diff --git a/server/src/test/java/org/elasticsearch/client/internal/ParentTaskAssigningClientTests.java b/server/src/test/java/org/elasticsearch/client/internal/ParentTaskAssigningClientTests.java index f0f44407642d8..600c09be2c12f 100644 --- a/server/src/test/java/org/elasticsearch/client/internal/ParentTaskAssigningClientTests.java +++ b/server/src/test/java/org/elasticsearch/client/internal/ParentTaskAssigningClientTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; @@ -95,10 +94,11 @@ public void ); assertEquals( "fake remote-cluster client", - expectThrows( + asInstanceOf( UnsupportedOperationException.class, - () -> PlainActionFuture.get( - fut -> remoteClusterClient.execute(ClusterStateAction.REMOTE_TYPE, new ClusterStateRequest(), fut) + safeAwaitFailure( + ClusterStateResponse.class, + listener -> remoteClusterClient.execute(ClusterStateAction.REMOTE_TYPE, new ClusterStateRequest(), listener) ) ).getMessage() ); diff --git a/server/src/test/java/org/elasticsearch/client/internal/node/NodeClientHeadersTests.java b/server/src/test/java/org/elasticsearch/client/internal/node/NodeClientHeadersTests.java index 9aea310180410..84537359e0399 100644 --- a/server/src/test/java/org/elasticsearch/client/internal/node/NodeClientHeadersTests.java +++ b/server/src/test/java/org/elasticsearch/client/internal/node/NodeClientHeadersTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.client.internal.AbstractClientHeadersTestCase; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.transport.Transport; @@ -54,7 +55,7 @@ private Actions(ActionType[] actions, TaskManager taskManager) { private static class InternalTransportAction extends TransportAction { private InternalTransportAction(String actionName, TaskManager taskManager) { - super(actionName, EMPTY_FILTERS, taskManager); + super(actionName, EMPTY_FILTERS, taskManager, EsExecutors.DIRECT_EXECUTOR_SERVICE); } @Override diff --git a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java index 910c10f6b265a..4f1c5b7fa5dc5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java @@ -298,10 +298,10 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti // a blocked reconnection attempt doesn't also block the node from being deregistered service.disconnectFromNodesExcept(nodes1); - assertThat(PlainActionFuture.get(disconnectFuture1 -> { - assertTrue(disconnectListenerRef.compareAndSet(null, disconnectFuture1)); + assertThat(safeAwait(disconnectListener -> { + assertTrue(disconnectListenerRef.compareAndSet(null, disconnectListener)); connectionBarrier.await(10, TimeUnit.SECONDS); - }, 10, TimeUnit.SECONDS), equalTo(node0)); // node0 connects briefly, must wait here + }), equalTo(node0)); // node0 connects briefly, must wait here assertConnectedExactlyToNodes(nodes1); // a blocked connection attempt to a new node also doesn't prevent an immediate deregistration @@ -312,10 +312,10 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti service.disconnectFromNodesExcept(nodes1); assertConnectedExactlyToNodes(nodes1); - assertThat(PlainActionFuture.get(disconnectFuture2 -> { - assertTrue(disconnectListenerRef.compareAndSet(null, disconnectFuture2)); + assertThat(safeAwait(disconnectListener -> { + assertTrue(disconnectListenerRef.compareAndSet(null, disconnectListener)); connectionBarrier.await(10, TimeUnit.SECONDS); - }, 10, TimeUnit.SECONDS), equalTo(node0)); // node0 connects briefly, must wait here + }), equalTo(node0)); // node0 connects briefly, must wait here assertConnectedExactlyToNodes(nodes1); assertTrue(future5.isDone()); } finally { @@ -596,29 +596,25 @@ public TransportAddress[] addressesFromString(String address) { return new TransportAddress[0]; } - private void runConnectionBlock(CheckedRunnable connectionBlock) { + private void runConnectionBlock(CheckedRunnable connectionBlock) throws Exception { if (connectionBlock == null) { return; } - try { - connectionBlock.run(); - } catch (Exception e) { - throw new AssertionError(e); - } + connectionBlock.run(); } @Override public void openConnection(DiscoveryNode node, ConnectionProfile profile, ActionListener listener) { final CheckedRunnable connectionBlock = nodeConnectionBlocks.get(node); if (profile == null && randomConnectionExceptions && randomBoolean()) { - threadPool.generic().execute(() -> { + threadPool.generic().execute(() -> ActionListener.completeWith(listener, () -> { runConnectionBlock(connectionBlock); - listener.onFailure(new ConnectTransportException(node, "simulated")); - }); + throw new ConnectTransportException(node, "simulated"); + })); } else { - threadPool.generic().execute(() -> { + threadPool.generic().execute(() -> ActionListener.completeWith(listener, () -> { runConnectionBlock(connectionBlock); - listener.onResponse(new Connection() { + return new Connection() { private final SubscribableListener closeListener = new SubscribableListener<>(); private final SubscribableListener removedListener = new SubscribableListener<>(); @@ -682,8 +678,8 @@ public boolean decRef() { public boolean hasReferences() { return refCounted.hasReferences(); } - }); - }); + }; + })); } } @@ -726,18 +722,18 @@ public RequestHandlers getRequestHandlers() { } private static void connectToNodes(NodeConnectionsService service, DiscoveryNodes discoveryNodes) { - PlainActionFuture.get(future -> service.connectToNodes(discoveryNodes, () -> future.onResponse(null)), 10, TimeUnit.SECONDS); + safeAwait(connectListener -> service.connectToNodes(discoveryNodes, () -> connectListener.onResponse(null))); } private static void ensureConnections(NodeConnectionsService service) { - PlainActionFuture.get(future -> service.ensureConnections(() -> future.onResponse(null)), 10, TimeUnit.SECONDS); + safeAwait(ensureListener -> service.ensureConnections(() -> ensureListener.onResponse(null))); } private static void closeConnection(TransportService transportService, DiscoveryNode discoveryNode) { try { final var connection = transportService.getConnection(discoveryNode); connection.close(); - PlainActionFuture.get(connection::addRemovedListener, 10, TimeUnit.SECONDS); + safeAwait(connection::addRemovedListener); } catch (NodeNotConnectedException e) { // ok } diff --git a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java index cada467ea3ad6..ef8742e4a4a35 100644 --- a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java @@ -430,9 +430,9 @@ public void testRemoteShardFailedConcurrently() throws Exception { for (int i = 0; i < failedShards.length; i++) { failedShards[i] = getRandomShardRouting(index); } - Thread[] clientThreads = new Thread[between(1, 6)]; + final int clientThreads = between(1, 6); int iterationsPerThread = scaledRandomIntBetween(50, 500); - Phaser barrier = new Phaser(clientThreads.length + 2); // one for master thread, one for the main thread + Phaser barrier = new Phaser(clientThreads + 1); // +1 for the master thread Thread masterThread = new Thread(() -> { barrier.arriveAndAwaitAdvance(); while (shutdown.get() == false) { @@ -448,39 +448,32 @@ public void testRemoteShardFailedConcurrently() throws Exception { masterThread.start(); AtomicInteger notifiedResponses = new AtomicInteger(); - for (int t = 0; t < clientThreads.length; t++) { - clientThreads[t] = new Thread(() -> { - barrier.arriveAndAwaitAdvance(); - for (int i = 0; i < iterationsPerThread; i++) { - ShardRouting failedShard = randomFrom(failedShards); - shardStateAction.remoteShardFailed( - failedShard.shardId(), - failedShard.allocationId().getId(), - randomLongBetween(1, Long.MAX_VALUE), - randomBoolean(), - "test", - getSimulatedFailure(), - new ActionListener() { - @Override - public void onResponse(Void aVoid) { - notifiedResponses.incrementAndGet(); - } - - @Override - public void onFailure(Exception e) { - notifiedResponses.incrementAndGet(); - } + runInParallel(clientThreads, t -> { + barrier.arriveAndAwaitAdvance(); + for (int i = 0; i < iterationsPerThread; i++) { + ShardRouting failedShard = randomFrom(failedShards); + shardStateAction.remoteShardFailed( + failedShard.shardId(), + failedShard.allocationId().getId(), + randomLongBetween(1, Long.MAX_VALUE), + randomBoolean(), + "test", + getSimulatedFailure(), + new ActionListener<>() { + @Override + public void onResponse(Void aVoid) { + notifiedResponses.incrementAndGet(); } - ); - } - }); - clientThreads[t].start(); - } - barrier.arriveAndAwaitAdvance(); - for (Thread t : clientThreads) { - t.join(); - } - assertBusy(() -> assertThat(notifiedResponses.get(), equalTo(clientThreads.length * iterationsPerThread))); + + @Override + public void onFailure(Exception e) { + notifiedResponses.incrementAndGet(); + } + } + ); + } + }); + assertBusy(() -> assertThat(notifiedResponses.get(), equalTo(clientThreads * iterationsPerThread))); shutdown.set(true); masterThread.join(); } diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsServiceTests.java index 2ad0f18de277f..cbfd43d646d0d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsServiceTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.monitor.StatusInfo; +import org.elasticsearch.test.EnumSerializationTestUtils; import org.elasticsearch.test.EqualsHashCodeTestUtils; import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.threadpool.ThreadPool; @@ -1417,4 +1418,14 @@ private void createAndAddNonMasterNode(Cluster cluster) { ); cluster.clusterNodes.add(nonMasterNode); } + + public void testCoordinationDiagnosticsStatusSerialization() { + EnumSerializationTestUtils.assertEnumSerialization( + CoordinationDiagnosticsStatus.class, + CoordinationDiagnosticsStatus.GREEN, + CoordinationDiagnosticsStatus.UNKNOWN, + CoordinationDiagnosticsStatus.YELLOW, + CoordinationDiagnosticsStatus.RED + ); + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java index e51b817bce594..217c1a51ebffd 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionTestUtils; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.NotMasterException; @@ -814,16 +813,14 @@ public void testPerNodeLogging() { ) ); assertNull( - PlainActionFuture.get( - future -> clusterService.getMasterService() + safeAwait( + (ActionListener listener) -> clusterService.getMasterService() .createTaskQueue("test", Priority.NORMAL, executor) .submitTask( "test", - JoinTask.singleNode(node1, CompatibilityVersionsUtils.staticCurrent(), Set.of(), TEST_REASON, future, 0L), + JoinTask.singleNode(node1, CompatibilityVersionsUtils.staticCurrent(), Set.of(), TEST_REASON, listener, 0L), null - ), - 10, - TimeUnit.SECONDS + ) ) ); mockLog.assertAllExpectationsMatched(); @@ -843,8 +840,8 @@ public void testPerNodeLogging() { ) ); assertNull( - PlainActionFuture.get( - future -> clusterService.getMasterService() + safeAwait( + (ActionListener listener) -> clusterService.getMasterService() .createTaskQueue("test", Priority.NORMAL, executor) .submitTask( "test", @@ -853,13 +850,11 @@ public void testPerNodeLogging() { CompatibilityVersionsUtils.staticCurrent(), Set.of(), testReasonWithLink, - future, + listener, 0L ), null - ), - 10, - TimeUnit.SECONDS + ) ) ); mockLog.assertAllExpectationsMatched(); diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeLeftExecutorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeLeftExecutorTests.java index 41ce520dc9bb6..0292dc42c3a4b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeLeftExecutorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeLeftExecutorTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.cluster.coordination; import org.apache.logging.log4j.Level; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -26,6 +25,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -128,13 +128,11 @@ public void testPerNodeLogging() { "node-left: [" + nodeToRemove.descriptionWithoutAttributes() + "] with reason [test reason]" ) ); - assertNull( - PlainActionFuture.get( - future -> clusterService.getMasterService() - .createTaskQueue("test", Priority.NORMAL, executor) - .submitTask("test", new NodeLeftExecutor.Task(nodeToRemove, "test reason", () -> future.onResponse(null)), null) - ) - ); + final var latch = new CountDownLatch(1); + clusterService.getMasterService() + .createTaskQueue("test", Priority.NORMAL, executor) + .submitTask("test", new NodeLeftExecutor.Task(nodeToRemove, "test reason", latch::countDown), null); + safeAwait(latch); mockLog.assertAllExpectationsMatched(); } finally { TestThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS); diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/stateless/AtomicRegisterPreVoteCollectorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/stateless/AtomicRegisterPreVoteCollectorTests.java index 0659b65be5844..036959068d76e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/stateless/AtomicRegisterPreVoteCollectorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/stateless/AtomicRegisterPreVoteCollectorTests.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.Level; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.core.TimeValue; @@ -66,7 +65,7 @@ protected long absoluteTimeInMillis() { // Either there's no heartbeat or is stale if (randomBoolean()) { - PlainActionFuture.get(f -> heartbeatStore.writeHeartbeat(new Heartbeat(1, fakeClock.get()), f)); + safeAwait((ActionListener l) -> heartbeatStore.writeHeartbeat(new Heartbeat(1, fakeClock.get()), l)); fakeClock.set(maxTimeSinceLastHeartbeat.millis() + randomLongBetween(0, 1000)); } @@ -107,7 +106,7 @@ protected long absoluteTimeInMillis() { } }; - PlainActionFuture.get(f -> heartbeatStore.writeHeartbeat(new Heartbeat(1, fakeClock.get()), f)); + safeAwait((ActionListener l) -> heartbeatStore.writeHeartbeat(new Heartbeat(1, fakeClock.get()), l)); fakeClock.addAndGet(randomLongBetween(0L, maxTimeSinceLastHeartbeat.millis() - 1)); var startElection = new AtomicBoolean(); @@ -141,7 +140,7 @@ protected long absoluteTimeInMillis() { } }; - PlainActionFuture.get(f -> heartbeatStore.writeHeartbeat(new Heartbeat(1, fakeClock.get()), f)); + safeAwait((ActionListener l) -> heartbeatStore.writeHeartbeat(new Heartbeat(1, fakeClock.get()), l)); var startElection = new AtomicBoolean(); var preVoteCollector = new AtomicRegisterPreVoteCollector(heartbeatService, () -> startElection.set(true)); diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/stateless/StoreHeartbeatServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/stateless/StoreHeartbeatServiceTests.java index 9a783d802a68c..ac985e50ca520 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/stateless/StoreHeartbeatServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/stateless/StoreHeartbeatServiceTests.java @@ -66,7 +66,7 @@ public void testHeartBeatStoreScheduling() { final var currentLeader = DiscoveryNodeUtils.create("master"); heartbeatService.start(currentLeader, currentTermProvider.get(), completionListener); - Heartbeat firstHeartbeat = PlainActionFuture.get(heartbeatStore::readLatestHeartbeat); + Heartbeat firstHeartbeat = safeAwait(heartbeatStore::readLatestHeartbeat); assertThat(firstHeartbeat, is(notNullValue())); assertThat(firstHeartbeat.term(), is(equalTo(1L))); assertThat(firstHeartbeat.absoluteTimeInMillis(), is(lessThanOrEqualTo(threadPool.absoluteTimeInMillis()))); @@ -79,7 +79,7 @@ public void testHeartBeatStoreScheduling() { assertThat(completionListener.isDone(), is(false)); - Heartbeat secondHeartbeat = PlainActionFuture.get(heartbeatStore::readLatestHeartbeat); + Heartbeat secondHeartbeat = safeAwait(heartbeatStore::readLatestHeartbeat); assertThat(secondHeartbeat, is(notNullValue())); assertThat(secondHeartbeat.term(), is(equalTo(1L))); assertThat(secondHeartbeat.absoluteTimeInMillis(), is(greaterThanOrEqualTo(firstHeartbeat.absoluteTimeInMillis()))); @@ -95,7 +95,7 @@ public void testHeartBeatStoreScheduling() { // No new tasks are scheduled after stopping the heart beat service assertThat(threadPool.scheduledTasks.poll(), is(nullValue())); - Heartbeat heartbeatAfterStoppingTheService = PlainActionFuture.get(heartbeatStore::readLatestHeartbeat); + Heartbeat heartbeatAfterStoppingTheService = safeAwait(heartbeatStore::readLatestHeartbeat); assertThat(heartbeatAfterStoppingTheService, is(equalTo(secondHeartbeat))); assertThat(completionListener.isDone(), is(false)); @@ -134,7 +134,7 @@ public void writeHeartbeat(Heartbeat newHeartbeat, ActionListener listener heartbeatService.start(currentLeader, currentTermProvider.get(), completionListener); if (failFirstHeartBeat == false) { - Heartbeat firstHeartbeat = PlainActionFuture.get(heartbeatStore::readLatestHeartbeat); + Heartbeat firstHeartbeat = safeAwait(heartbeatStore::readLatestHeartbeat); assertThat(firstHeartbeat, is(notNullValue())); var scheduledTask = threadPool.scheduledTasks.poll(); @@ -179,7 +179,7 @@ public void testServiceStopsAfterTermBump() throws Exception { heartbeatService.start(currentLeader, currentTerm, completionListener); if (termBumpBeforeStart == false) { - Heartbeat firstHeartbeat = PlainActionFuture.get(heartbeatStore::readLatestHeartbeat); + Heartbeat firstHeartbeat = safeAwait(heartbeatStore::readLatestHeartbeat); assertThat(firstHeartbeat, is(notNullValue())); var scheduledTask = threadPool.scheduledTasks.poll(); @@ -229,7 +229,7 @@ protected long absoluteTimeInMillis() { // Empty store { - Heartbeat heartbeat = PlainActionFuture.get(heartbeatStore::readLatestHeartbeat); + Heartbeat heartbeat = safeAwait(heartbeatStore::readLatestHeartbeat); assertThat(heartbeat, is(nullValue())); AtomicBoolean noRecentLeaderFound = new AtomicBoolean(); @@ -239,7 +239,7 @@ protected long absoluteTimeInMillis() { // Recent heartbeat { - PlainActionFuture.get(f -> heartbeatStore.writeHeartbeat(new Heartbeat(1, fakeClock.get()), f)); + safeAwait((ActionListener l) -> heartbeatStore.writeHeartbeat(new Heartbeat(1, fakeClock.get()), l)); AtomicBoolean noRecentLeaderFound = new AtomicBoolean(); heartbeatService.checkLeaderHeartbeatAndRun(() -> noRecentLeaderFound.set(true), hb -> {}); @@ -248,7 +248,7 @@ protected long absoluteTimeInMillis() { // Stale heartbeat { - PlainActionFuture.get(f -> heartbeatStore.writeHeartbeat(new Heartbeat(1, fakeClock.get()), f)); + safeAwait((ActionListener l) -> heartbeatStore.writeHeartbeat(new Heartbeat(1, fakeClock.get()), l)); fakeClock.set(maxTimeSinceLastHeartbeat.millis() + 1); AtomicBoolean noRecentLeaderFound = new AtomicBoolean(); @@ -258,7 +258,7 @@ protected long absoluteTimeInMillis() { // Failing store { - PlainActionFuture.get(f -> heartbeatStore.writeHeartbeat(new Heartbeat(1, fakeClock.get()), f)); + safeAwait((ActionListener l) -> heartbeatStore.writeHeartbeat(new Heartbeat(1, fakeClock.get()), l)); fakeClock.set(maxTimeSinceLastHeartbeat.millis() + 1); failReadingHeartbeat.set(true); @@ -309,7 +309,7 @@ protected long absoluteTimeInMillis() { retryTask.v2().run(); - Heartbeat firstHeartbeat = PlainActionFuture.get(heartbeatStore::readLatestHeartbeat); + Heartbeat firstHeartbeat = safeAwait(heartbeatStore::readLatestHeartbeat); assertThat(firstHeartbeat, is(notNullValue())); assertThat(firstHeartbeat.term(), is(equalTo(1L))); diff --git a/server/src/test/java/org/elasticsearch/cluster/features/NodeFeaturesFixupListenerTests.java b/server/src/test/java/org/elasticsearch/cluster/features/NodeFeaturesFixupListenerTests.java new file mode 100644 index 0000000000000..30d4c85e8fb67 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/features/NodeFeaturesFixupListenerTests.java @@ -0,0 +1,245 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.cluster.features; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.features.NodeFeatures; +import org.elasticsearch.action.admin.cluster.node.features.NodesFeaturesRequest; +import org.elasticsearch.action.admin.cluster.node.features.NodesFeaturesResponse; +import org.elasticsearch.action.admin.cluster.node.features.TransportNodesFeaturesAction; +import org.elasticsearch.client.internal.ClusterAdminClient; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.features.NodeFeaturesFixupListener.NodesFeaturesTask; +import org.elasticsearch.cluster.features.NodeFeaturesFixupListener.NodesFeaturesUpdater; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.node.VersionInformation; +import org.elasticsearch.cluster.service.ClusterStateTaskExecutorUtils; +import org.elasticsearch.cluster.service.MasterServiceTaskQueue; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.Scheduler; +import org.mockito.ArgumentCaptor; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executor; + +import static org.elasticsearch.test.LambdaMatchers.transformedMatch; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.hamcrest.MockitoHamcrest.argThat; + +public class NodeFeaturesFixupListenerTests extends ESTestCase { + + @SuppressWarnings("unchecked") + private static MasterServiceTaskQueue newMockTaskQueue() { + return mock(MasterServiceTaskQueue.class); + } + + private static DiscoveryNodes nodes(Version... versions) { + var builder = DiscoveryNodes.builder(); + for (int i = 0; i < versions.length; i++) { + builder.add(DiscoveryNodeUtils.create("node" + i, new TransportAddress(TransportAddress.META_ADDRESS, 9200 + i), versions[i])); + } + builder.localNodeId("node0").masterNodeId("node0"); + return builder.build(); + } + + private static DiscoveryNodes nodes(VersionInformation... versions) { + var builder = DiscoveryNodes.builder(); + for (int i = 0; i < versions.length; i++) { + builder.add( + DiscoveryNodeUtils.builder("node" + i) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9200 + i)) + .version(versions[i]) + .build() + ); + } + builder.localNodeId("node0").masterNodeId("node0"); + return builder.build(); + } + + @SafeVarargs + private static Map> features(Set... nodeFeatures) { + Map> features = new HashMap<>(); + for (int i = 0; i < nodeFeatures.length; i++) { + features.put("node" + i, nodeFeatures[i]); + } + return features; + } + + private static NodesFeaturesResponse getResponse(Map> responseData) { + return new NodesFeaturesResponse( + ClusterName.DEFAULT, + responseData.entrySet() + .stream() + .map( + e -> new NodeFeatures( + e.getValue(), + DiscoveryNodeUtils.create(e.getKey(), new TransportAddress(TransportAddress.META_ADDRESS, 9200)) + ) + ) + .toList(), + List.of() + ); + } + + public void testNothingDoneWhenNothingToFix() { + MasterServiceTaskQueue taskQueue = newMockTaskQueue(); + ClusterAdminClient client = mock(ClusterAdminClient.class); + + ClusterState testState = ClusterState.builder(ClusterState.EMPTY_STATE) + .nodes(nodes(Version.CURRENT, Version.CURRENT)) + .nodeFeatures(features(Set.of("f1", "f2"), Set.of("f1", "f2"))) + .build(); + + NodeFeaturesFixupListener listener = new NodeFeaturesFixupListener(taskQueue, client, null, null); + listener.clusterChanged(new ClusterChangedEvent("test", testState, ClusterState.EMPTY_STATE)); + + verify(taskQueue, never()).submitTask(anyString(), any(), any()); + } + + public void testFeaturesFixedAfterNewMaster() throws Exception { + MasterServiceTaskQueue taskQueue = newMockTaskQueue(); + ClusterAdminClient client = mock(ClusterAdminClient.class); + Set features = Set.of("f1", "f2"); + + ClusterState testState = ClusterState.builder(ClusterState.EMPTY_STATE) + .nodes(nodes(Version.CURRENT, Version.CURRENT, Version.CURRENT)) + .nodeFeatures(features(features, Set.of(), Set.of())) + .build(); + + ArgumentCaptor> action = ArgumentCaptor.captor(); + ArgumentCaptor task = ArgumentCaptor.captor(); + + NodeFeaturesFixupListener listener = new NodeFeaturesFixupListener(taskQueue, client, null, null); + listener.clusterChanged(new ClusterChangedEvent("test", testState, ClusterState.EMPTY_STATE)); + verify(client).execute( + eq(TransportNodesFeaturesAction.TYPE), + argThat(transformedMatch(NodesFeaturesRequest::nodesIds, arrayContainingInAnyOrder("node1", "node2"))), + action.capture() + ); + + action.getValue().onResponse(getResponse(Map.of("node1", features, "node2", features))); + verify(taskQueue).submitTask(anyString(), task.capture(), any()); + + ClusterState newState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful( + testState, + new NodesFeaturesUpdater(), + List.of(task.getValue()) + ); + + assertThat(newState.clusterFeatures().allNodeFeatures(), containsInAnyOrder("f1", "f2")); + } + + public void testFeaturesFetchedOnlyForUpdatedNodes() { + MasterServiceTaskQueue taskQueue = newMockTaskQueue(); + ClusterAdminClient client = mock(ClusterAdminClient.class); + + ClusterState testState = ClusterState.builder(ClusterState.EMPTY_STATE) + .nodes( + nodes( + VersionInformation.CURRENT, + VersionInformation.CURRENT, + new VersionInformation(Version.V_8_12_0, IndexVersion.current(), IndexVersion.current()) + ) + ) + .nodeFeatures(features(Set.of("f1", "f2"), Set.of(), Set.of())) + .build(); + + ArgumentCaptor> action = ArgumentCaptor.captor(); + + NodeFeaturesFixupListener listener = new NodeFeaturesFixupListener(taskQueue, client, null, null); + listener.clusterChanged(new ClusterChangedEvent("test", testState, ClusterState.EMPTY_STATE)); + verify(client).execute( + eq(TransportNodesFeaturesAction.TYPE), + argThat(transformedMatch(NodesFeaturesRequest::nodesIds, arrayContainingInAnyOrder("node1"))), + action.capture() + ); + } + + public void testConcurrentChangesDoNotOverlap() { + MasterServiceTaskQueue taskQueue = newMockTaskQueue(); + ClusterAdminClient client = mock(ClusterAdminClient.class); + Set features = Set.of("f1", "f2"); + + ClusterState testState1 = ClusterState.builder(ClusterState.EMPTY_STATE) + .nodes(nodes(Version.CURRENT, Version.CURRENT, Version.CURRENT)) + .nodeFeatures(features(features, Set.of(), Set.of())) + .build(); + + NodeFeaturesFixupListener listeners = new NodeFeaturesFixupListener(taskQueue, client, null, null); + listeners.clusterChanged(new ClusterChangedEvent("test", testState1, ClusterState.EMPTY_STATE)); + verify(client).execute( + eq(TransportNodesFeaturesAction.TYPE), + argThat(transformedMatch(NodesFeaturesRequest::nodesIds, arrayContainingInAnyOrder("node1", "node2"))), + any() + ); + // don't send back the response yet + + ClusterState testState2 = ClusterState.builder(ClusterState.EMPTY_STATE) + .nodes(nodes(Version.CURRENT, Version.CURRENT, Version.CURRENT)) + .nodeFeatures(features(features, features, Set.of())) + .build(); + // should not send any requests + listeners.clusterChanged(new ClusterChangedEvent("test", testState2, testState1)); + verifyNoMoreInteractions(client); + } + + public void testFailedRequestsAreRetried() { + MasterServiceTaskQueue taskQueue = newMockTaskQueue(); + ClusterAdminClient client = mock(ClusterAdminClient.class); + Scheduler scheduler = mock(Scheduler.class); + Executor executor = mock(Executor.class); + Set features = Set.of("f1", "f2"); + + ClusterState testState = ClusterState.builder(ClusterState.EMPTY_STATE) + .nodes(nodes(Version.CURRENT, Version.CURRENT, Version.CURRENT)) + .nodeFeatures(features(features, Set.of(), Set.of())) + .build(); + + ArgumentCaptor> action = ArgumentCaptor.captor(); + ArgumentCaptor retry = ArgumentCaptor.forClass(Runnable.class); + + NodeFeaturesFixupListener listener = new NodeFeaturesFixupListener(taskQueue, client, scheduler, executor); + listener.clusterChanged(new ClusterChangedEvent("test", testState, ClusterState.EMPTY_STATE)); + verify(client).execute( + eq(TransportNodesFeaturesAction.TYPE), + argThat(transformedMatch(NodesFeaturesRequest::nodesIds, arrayContainingInAnyOrder("node1", "node2"))), + action.capture() + ); + + action.getValue().onFailure(new RuntimeException("failure")); + verify(scheduler).schedule(retry.capture(), any(), same(executor)); + + // running the retry should cause another call + retry.getValue().run(); + verify(client, times(2)).execute( + eq(TransportNodesFeaturesAction.TYPE), + argThat(transformedMatch(NodesFeaturesRequest::nodesIds, arrayContainingInAnyOrder("node1", "node2"))), + action.capture() + ); + } +} diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/BatchedRerouteServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/BatchedRerouteServiceTests.java index 5666a2dd77a89..3bc9a936e4ee5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/BatchedRerouteServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/BatchedRerouteServiceTests.java @@ -310,12 +310,10 @@ public void testExceptionFidelity() { // Case 3: a NotMasterException - PlainActionFuture.get(future -> { - clusterService.getClusterApplierService().onNewClusterState("simulated", () -> { - final var state = clusterService.state(); - return ClusterState.builder(state).nodes(state.nodes().withMasterNodeId(null)).build(); - }, future); - }, 10, TimeUnit.SECONDS); + safeAwait((ActionListener listener) -> clusterService.getClusterApplierService().onNewClusterState("simulated", () -> { + final var state = clusterService.state(); + return ClusterState.builder(state).nodes(state.nodes().withMasterNodeId(null)).build(); + }, listener)); mockLog.addExpectation( new MockLog.SeenEventExpectation( diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java index 328777bfe28e7..7f9c69955adcd 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java @@ -583,20 +583,28 @@ public void testClusterAllActive3() { public void testRebalanceWithIgnoredUnassignedShards() { final AtomicBoolean allocateTest1 = new AtomicBoolean(false); - AllocationService strategy = createAllocationService(Settings.EMPTY, new TestGatewayAllocator() { - @Override - public void allocateUnassigned( - ShardRouting shardRouting, - RoutingAllocation allocation, - UnassignedAllocationHandler unassignedAllocationHandler - ) { - if (allocateTest1.get() == false && "test1".equals(shardRouting.index().getName())) { - unassignedAllocationHandler.removeAndIgnore(UnassignedInfo.AllocationStatus.NO_ATTEMPT, allocation.changes()); - } else { - super.allocateUnassigned(shardRouting, allocation, unassignedAllocationHandler); + AllocationService strategy = createAllocationService( + Settings.builder() + .put( + ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), + ClusterRebalanceAllocationDecider.ClusterRebalanceType.INDICES_ALL_ACTIVE.toString() + ) + .build(), + new TestGatewayAllocator() { + @Override + public void allocateUnassigned( + ShardRouting shardRouting, + RoutingAllocation allocation, + UnassignedAllocationHandler unassignedAllocationHandler + ) { + if (allocateTest1.get() == false && "test1".equals(shardRouting.index().getName())) { + unassignedAllocationHandler.removeAndIgnore(UnassignedInfo.AllocationStatus.NO_ATTEMPT, allocation.changes()); + } else { + super.allocateUnassigned(shardRouting, allocation, unassignedAllocationHandler); + } } } - }); + ); Metadata metadata = Metadata.builder() .put(IndexMetadata.builder("test").settings(settings(IndexVersion.current())).numberOfShards(2).numberOfReplicas(0)) diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsTests.java index ad371ed239795..a21ef34b966ea 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsTests.java @@ -23,13 +23,7 @@ protected Writeable.Reader instanceReader() { @Override protected NodeAllocationStats createTestInstance() { - return new NodeAllocationStats( - randomIntBetween(0, 10000), - randomIntBetween(0, 1000), - randomDoubleBetween(0, 8, true), - randomNonNegativeLong(), - randomNonNegativeLong() - ); + return randomNodeAllocationStats(); } @Override @@ -73,4 +67,14 @@ protected NodeAllocationStats mutateInstance(NodeAllocationStats instance) throw default -> throw new RuntimeException("unreachable"); }; } + + public static NodeAllocationStats randomNodeAllocationStats() { + return new NodeAllocationStats( + randomIntBetween(0, 10000), + randomIntBetween(0, 1000), + randomDoubleBetween(0, 8, true), + randomNonNegativeLong(), + randomNonNegativeLong() + ); + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardChangesObserverTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardChangesObserverTests.java index 53c156671f540..2c5e588ec77b3 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardChangesObserverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardChangesObserverTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.AllocationId; import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; @@ -31,7 +32,7 @@ import static org.elasticsearch.cluster.routing.TestShardRouting.shardRoutingBuilder; import static org.elasticsearch.test.MockLog.assertThatLogger; -@TestLogging(value = "org.elasticsearch.cluster.routing.allocation.ShardChangesObserver:DEBUG", reason = "verifies debug level logging") +@TestLogging(value = "org.elasticsearch.cluster.routing.allocation.ShardChangesObserver:TRACE", reason = "verifies debug level logging") public class ShardChangesObserverTests extends ESAllocationTestCase { public void testLogShardStarting() { @@ -48,11 +49,17 @@ public void testLogShardStarting() { assertThatLogger( () -> applyStartedShardsUntilNoChange(clusterState, createAllocationService()), ShardChangesObserver.class, + new MockLog.SeenEventExpectation( + "Should log shard initializing", + ShardChangesObserver.class.getCanonicalName(), + Level.TRACE, + "[" + indexName + "][0][P] initializing from " + RecoverySource.Type.EMPTY_STORE + " on node [node-1]" + ), new MockLog.SeenEventExpectation( "Should log shard starting", ShardChangesObserver.class.getCanonicalName(), Level.DEBUG, - "[" + indexName + "][0][P] started on node [node-1]" + "[" + indexName + "][0][P] started from " + RecoverySource.Type.EMPTY_STORE + " on node [node-1]" ) ); } @@ -89,7 +96,7 @@ public void testLogShardMovement() { "Should log shard starting", ShardChangesObserver.class.getCanonicalName(), Level.DEBUG, - "[" + indexName + "][0][P] started on node [node-2]" + "[" + indexName + "][0][P] started from " + RecoverySource.Type.PEER + " on node [node-2]" ) ); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ContinuousComputationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ContinuousComputationTests.java index c644c0a1d1225..ca4aa5c6ae442 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ContinuousComputationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ContinuousComputationTests.java @@ -17,7 +17,6 @@ import org.junit.BeforeClass; import java.util.Arrays; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; @@ -60,26 +59,13 @@ protected void processInput(Integer input) { } }; - final Thread[] threads = new Thread[between(1, 5)]; - final int[] valuePerThread = new int[threads.length]; - final CountDownLatch startLatch = new CountDownLatch(1); - for (int i = 0; i < threads.length; i++) { - final int threadIndex = i; - valuePerThread[threadIndex] = randomInt(); - threads[threadIndex] = new Thread(() -> { - safeAwait(startLatch); - for (int j = 1000; j >= 0; j--) { - computation.onNewInput(valuePerThread[threadIndex] = valuePerThread[threadIndex] + j); - } - }, "submit-thread-" + threadIndex); - threads[threadIndex].start(); - } - - startLatch.countDown(); - - for (Thread thread : threads) { - thread.join(); - } + final int threads = between(1, 5); + final int[] valuePerThread = new int[threads]; + startInParallel(threads, threadIndex -> { + for (int j = 1000; j >= 0; j--) { + computation.onNewInput(valuePerThread[threadIndex] = valuePerThread[threadIndex] + j); + } + }); assertBusy(() -> assertFalse(computation.isActive())); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java index e5b3393723ab1..130d2e41aa374 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java @@ -11,7 +11,6 @@ import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionTestUtils; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterInfo; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -818,6 +817,6 @@ public ShardAllocationDecision decideShardAllocation(ShardRouting shard, Routing } private static void rerouteAndWait(AllocationService service, ClusterState clusterState, String reason) { - PlainActionFuture.get(f -> service.reroute(clusterState, reason, f), 10, TimeUnit.SECONDS); + safeAwait((ActionListener listener) -> service.reroute(clusterState, reason, listener)); } } diff --git a/server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobContainerTests.java b/server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobContainerTests.java index b4ddc02aeb2d2..e849f82e169cc 100644 --- a/server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobContainerTests.java +++ b/server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobContainerTests.java @@ -11,7 +11,7 @@ import org.apache.lucene.tests.mockfile.FilterSeekableByteChannel; import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.OptionalBytesReference; import org.elasticsearch.common.bytes.BytesArray; @@ -43,7 +43,6 @@ import java.util.Locale; import java.util.Set; import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -53,6 +52,7 @@ import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.oneOf; import static org.hamcrest.Matchers.startsWith; @@ -177,7 +177,9 @@ private static BytesReference getBytesAsync(Consumer T getAsync(Consumer> consumer) { - return PlainActionFuture.get(consumer::accept, 0, TimeUnit.SECONDS); + final var listener = SubscribableListener.newForked(consumer::accept); + assertTrue(listener.isDone()); + return safeAwait(listener); } public void testCompareAndExchange() throws Exception { @@ -235,9 +237,12 @@ public void testCompareAndExchange() throws Exception { } container.writeBlob(randomPurpose(), key, new BytesArray(new byte[17]), false); - expectThrows( - IllegalStateException.class, - () -> getBytesAsync(l -> container.compareAndExchangeRegister(randomPurpose(), key, expectedValue.get(), BytesArray.EMPTY, l)) + assertThat( + safeAwaitFailure( + OptionalBytesReference.class, + l -> container.compareAndExchangeRegister(randomPurpose(), key, expectedValue.get(), BytesArray.EMPTY, l) + ), + instanceOf(IllegalStateException.class) ); } @@ -256,8 +261,8 @@ public void testRegisterContention() throws Exception { final var finalValue = randomValueOtherThan(startValue, () -> new BytesArray(randomByteArrayOfLength(8))); final var p = randomPurpose(); - assertTrue(PlainActionFuture.get(l -> container.compareAndSetRegister(p, contendedKey, BytesArray.EMPTY, startValue, l))); - assertTrue(PlainActionFuture.get(l -> container.compareAndSetRegister(p, uncontendedKey, BytesArray.EMPTY, startValue, l))); + assertTrue(safeAwait(l -> container.compareAndSetRegister(p, contendedKey, BytesArray.EMPTY, startValue, l))); + assertTrue(safeAwait(l -> container.compareAndSetRegister(p, uncontendedKey, BytesArray.EMPTY, startValue, l))); final var threads = new Thread[between(2, 5)]; final var startBarrier = new CyclicBarrier(threads.length + 1); @@ -268,7 +273,7 @@ public void testRegisterContention() throws Exception { // first thread does an uncontended write, which must succeed ? () -> { safeAwait(startBarrier); - final OptionalBytesReference result = PlainActionFuture.get( + final OptionalBytesReference result = safeAwait( l -> container.compareAndExchangeRegister(p, uncontendedKey, startValue, finalValue, l) ); // NB calling .bytesReference() asserts that the result is present, there was no contention @@ -278,7 +283,7 @@ public void testRegisterContention() throws Exception { : () -> { safeAwait(startBarrier); while (casSucceeded.get() == false) { - final OptionalBytesReference result = PlainActionFuture.get( + final OptionalBytesReference result = safeAwait( l -> container.compareAndExchangeRegister(p, contendedKey, startValue, finalValue, l) ); if (result.isPresent() && result.bytesReference().equals(startValue)) { @@ -296,7 +301,7 @@ public void testRegisterContention() throws Exception { for (var key : new String[] { contendedKey, uncontendedKey }) { // NB calling .bytesReference() asserts that the read did not experience contention assertThat( - PlainActionFuture.get(l -> container.getRegister(p, key, l)).bytesReference(), + safeAwait((ActionListener l) -> container.getRegister(p, key, l)).bytesReference(), oneOf(startValue, finalValue) ); } @@ -309,7 +314,7 @@ public void testRegisterContention() throws Exception { for (var key : new String[] { contendedKey, uncontendedKey }) { assertEquals( finalValue, - PlainActionFuture.get(l -> container.getRegister(p, key, l)).bytesReference() + safeAwait((ActionListener l) -> container.getRegister(p, key, l)).bytesReference() ); } } diff --git a/server/src/test/java/org/elasticsearch/common/cache/CacheTests.java b/server/src/test/java/org/elasticsearch/common/cache/CacheTests.java index 0fb40583a4d79..fe302d2e5fec1 100644 --- a/server/src/test/java/org/elasticsearch/common/cache/CacheTests.java +++ b/server/src/test/java/org/elasticsearch/common/cache/CacheTests.java @@ -326,32 +326,19 @@ protected long now() { assertEquals(numberOfEntries, cache.stats().getEvictions()); } - public void testComputeIfAbsentDeadlock() { - final int numberOfThreads = randomIntBetween(2, 32); + public void testComputeIfAbsentDeadlock() throws InterruptedException { final Cache cache = CacheBuilder.builder() .setExpireAfterAccess(TimeValue.timeValueNanos(1)) .build(); - - final CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads); - for (int i = 0; i < numberOfThreads; i++) { - final Thread thread = new Thread(() -> { - safeAwait(barrier); - for (int j = 0; j < numberOfEntries; j++) { - try { - cache.computeIfAbsent(0, k -> Integer.toString(k)); - } catch (final ExecutionException e) { - throw new AssertionError(e); - } + startInParallel(randomIntBetween(2, 32), i -> { + for (int j = 0; j < numberOfEntries; j++) { + try { + cache.computeIfAbsent(0, k -> Integer.toString(k)); + } catch (final ExecutionException e) { + throw new AssertionError(e); } - safeAwait(barrier); - }); - thread.start(); - } - - // wait for all threads to be ready - safeAwait(barrier); - // wait for all threads to finish - safeAwait(barrier); + } + }); } // randomly promote some entries, step the clock forward, then check that the promoted entries remain and the @@ -596,41 +583,26 @@ public void testComputeIfAbsentLoadsSuccessfully() { } } - public void testComputeIfAbsentCallsOnce() { - int numberOfThreads = randomIntBetween(2, 32); + public void testComputeIfAbsentCallsOnce() throws InterruptedException { final Cache cache = CacheBuilder.builder().build(); AtomicReferenceArray flags = new AtomicReferenceArray<>(numberOfEntries); for (int j = 0; j < numberOfEntries; j++) { flags.set(j, false); } - CopyOnWriteArrayList failures = new CopyOnWriteArrayList<>(); - - CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads); - for (int i = 0; i < numberOfThreads; i++) { - Thread thread = new Thread(() -> { - safeAwait(barrier); - for (int j = 0; j < numberOfEntries; j++) { - try { - cache.computeIfAbsent(j, key -> { - assertTrue(flags.compareAndSet(key, false, true)); - return Integer.toString(key); - }); - } catch (ExecutionException e) { - failures.add(e); - break; - } + startInParallel(randomIntBetween(2, 32), i -> { + for (int j = 0; j < numberOfEntries; j++) { + try { + cache.computeIfAbsent(j, key -> { + assertTrue(flags.compareAndSet(key, false, true)); + return Integer.toString(key); + }); + } catch (ExecutionException e) { + failures.add(e); + break; } - safeAwait(barrier); - }); - thread.start(); - } - - // wait for all threads to be ready - safeAwait(barrier); - // wait for all threads to finish - safeAwait(barrier); - + } + }); assertThat(failures, is(empty())); } @@ -751,111 +723,70 @@ public int hashCode() { assertFalse("deadlock", deadlock.get()); } - public void testCachePollution() { + public void testCachePollution() throws InterruptedException { int numberOfThreads = randomIntBetween(2, 32); final Cache cache = CacheBuilder.builder().build(); - - CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads); - - for (int i = 0; i < numberOfThreads; i++) { - Thread thread = new Thread(() -> { - safeAwait(barrier); - Random random = new Random(random().nextLong()); - for (int j = 0; j < numberOfEntries; j++) { - Integer key = random.nextInt(numberOfEntries); - boolean first; - boolean second; - do { - first = random.nextBoolean(); - second = random.nextBoolean(); - } while (first && second); - if (first) { - try { - cache.computeIfAbsent(key, k -> { - if (random.nextBoolean()) { - return Integer.toString(k); - } else { - throw new Exception("testCachePollution"); - } - }); - } catch (ExecutionException e) { - assertNotNull(e.getCause()); - assertThat(e.getCause(), instanceOf(Exception.class)); - assertEquals(e.getCause().getMessage(), "testCachePollution"); - } - } else if (second) { - cache.invalidate(key); - } else { - cache.get(key); + startInParallel(numberOfThreads, i -> { + Random random = new Random(random().nextLong()); + for (int j = 0; j < numberOfEntries; j++) { + Integer key = random.nextInt(numberOfEntries); + boolean first; + boolean second; + do { + first = random.nextBoolean(); + second = random.nextBoolean(); + } while (first && second); + if (first) { + try { + cache.computeIfAbsent(key, k -> { + if (random.nextBoolean()) { + return Integer.toString(k); + } else { + throw new Exception("testCachePollution"); + } + }); + } catch (ExecutionException e) { + assertNotNull(e.getCause()); + assertThat(e.getCause(), instanceOf(Exception.class)); + assertEquals(e.getCause().getMessage(), "testCachePollution"); } + } else if (second) { + cache.invalidate(key); + } else { + cache.get(key); } - safeAwait(barrier); - }); - thread.start(); - } - - // wait for all threads to be ready - safeAwait(barrier); - // wait for all threads to finish - safeAwait(barrier); + } + }); } - public void testExceptionThrownDuringConcurrentComputeIfAbsent() { - int numberOfThreads = randomIntBetween(2, 32); + public void testExceptionThrownDuringConcurrentComputeIfAbsent() throws InterruptedException { final Cache cache = CacheBuilder.builder().build(); - - CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads); - final String key = randomAlphaOfLengthBetween(2, 32); - for (int i = 0; i < numberOfThreads; i++) { - Thread thread = new Thread(() -> { - safeAwait(barrier); - for (int j = 0; j < numberOfEntries; j++) { - try { - String value = cache.computeIfAbsent(key, k -> { throw new RuntimeException("failed to load"); }); - fail("expected exception but got: " + value); - } catch (ExecutionException e) { - assertNotNull(e.getCause()); - assertThat(e.getCause(), instanceOf(RuntimeException.class)); - assertEquals(e.getCause().getMessage(), "failed to load"); - } + startInParallel(randomIntBetween(2, 32), i -> { + for (int j = 0; j < numberOfEntries; j++) { + try { + String value = cache.computeIfAbsent(key, k -> { throw new RuntimeException("failed to load"); }); + fail("expected exception but got: " + value); + } catch (ExecutionException e) { + assertNotNull(e.getCause()); + assertThat(e.getCause(), instanceOf(RuntimeException.class)); + assertEquals(e.getCause().getMessage(), "failed to load"); } - safeAwait(barrier); - }); - thread.start(); - } - - // wait for all threads to be ready - safeAwait(barrier); - // wait for all threads to finish - safeAwait(barrier); + } + }); } // test that the cache is not corrupted under lots of concurrent modifications, even hitting the same key // here be dragons: this test did catch one subtle bug during development; do not remove lightly - public void testTorture() { - int numberOfThreads = randomIntBetween(2, 32); + public void testTorture() throws InterruptedException { final Cache cache = CacheBuilder.builder().setMaximumWeight(1000).weigher((k, v) -> 2).build(); - - CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads); - for (int i = 0; i < numberOfThreads; i++) { - Thread thread = new Thread(() -> { - safeAwait(barrier); - Random random = new Random(random().nextLong()); - for (int j = 0; j < numberOfEntries; j++) { - Integer key = random.nextInt(numberOfEntries); - cache.put(key, Integer.toString(j)); - } - safeAwait(barrier); - }); - thread.start(); - } - - // wait for all threads to be ready - safeAwait(barrier); - // wait for all threads to finish - safeAwait(barrier); - + startInParallel(randomIntBetween(2, 32), i -> { + Random random = new Random(random().nextLong()); + for (int j = 0; j < numberOfEntries; j++) { + Integer key = random.nextInt(numberOfEntries); + cache.put(key, Integer.toString(j)); + } + }); cache.refresh(); assertEquals(500, cache.count()); } diff --git a/server/src/test/java/org/elasticsearch/common/collect/IteratorsTests.java b/server/src/test/java/org/elasticsearch/common/collect/IteratorsTests.java index 67f74df78e256..a3573d081397a 100644 --- a/server/src/test/java/org/elasticsearch/common/collect/IteratorsTests.java +++ b/server/src/test/java/org/elasticsearch/common/collect/IteratorsTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.common.collect; import org.elasticsearch.common.Randomness; +import org.elasticsearch.core.Assertions; import org.elasticsearch.core.Tuple; import org.elasticsearch.test.ESTestCase; @@ -23,6 +24,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiPredicate; +import java.util.function.Predicate; import java.util.function.ToIntFunction; import java.util.stream.IntStream; @@ -219,6 +221,27 @@ public void testMap() { assertEquals(array.length, index.get()); } + public void testFilter() { + assertSame(Collections.emptyIterator(), Iterators.filter(Collections.emptyIterator(), i -> fail(null, "not called"))); + + final var array = randomIntegerArray(); + assertSame(Collections.emptyIterator(), Iterators.filter(Iterators.forArray(array), i -> false)); + + final var threshold = array.length > 0 && randomBoolean() ? randomFrom(array) : randomIntBetween(0, 1000); + final Predicate predicate = i -> i <= threshold; + final var expectedResults = Arrays.stream(array).filter(predicate).toList(); + final var index = new AtomicInteger(); + Iterators.filter(Iterators.forArray(array), predicate) + .forEachRemaining(i -> assertEquals(expectedResults.get(index.getAndIncrement()), i)); + + if (Assertions.ENABLED) { + final var predicateCalled = new AtomicBoolean(); + final var inputIterator = Iterators.forArray(new Object[] { null }); + expectThrows(AssertionError.class, () -> Iterators.filter(inputIterator, i -> predicateCalled.compareAndSet(false, true))); + assertFalse(predicateCalled.get()); + } + } + public void testFailFast() { final var array = randomIntegerArray(); assertEmptyIterator(Iterators.failFast(Iterators.forArray(array), () -> true)); diff --git a/server/src/test/java/org/elasticsearch/common/component/LifecycleTests.java b/server/src/test/java/org/elasticsearch/common/component/LifecycleTests.java index bea074b100609..c6e2b72e79e6b 100644 --- a/server/src/test/java/org/elasticsearch/common/component/LifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/common/component/LifecycleTests.java @@ -8,8 +8,8 @@ package org.elasticsearch.common.component; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.RefCountingListener; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; @@ -99,8 +99,8 @@ private static class ThreadSafetyTestHarness implements Releasable { void testTransition(BooleanSupplier doTransition) { final var transitioned = new AtomicBoolean(); - PlainActionFuture.get(fut -> { - try (var listeners = new RefCountingListener(fut)) { + safeAwait((ActionListener listener) -> { + try (var listeners = new RefCountingListener(listener)) { for (int i = 0; i < threads; i++) { executor.execute(ActionRunnable.run(listeners.acquire(), () -> { safeAwait(barrier); diff --git a/server/src/test/java/org/elasticsearch/common/compress/DeflateCompressTests.java b/server/src/test/java/org/elasticsearch/common/compress/DeflateCompressTests.java index 2fb31cad12051..2909b22347b93 100644 --- a/server/src/test/java/org/elasticsearch/common/compress/DeflateCompressTests.java +++ b/server/src/test/java/org/elasticsearch/common/compress/DeflateCompressTests.java @@ -23,7 +23,6 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Random; -import java.util.concurrent.CountDownLatch; import java.util.zip.ZipException; import static org.hamcrest.Matchers.equalTo; @@ -45,34 +44,17 @@ public void testRandom() throws IOException { } public void testRandomThreads() throws Exception { - final Random r = random(); - int threadCount = TestUtil.nextInt(r, 2, 6); - Thread[] threads = new Thread[threadCount]; - final CountDownLatch startingGun = new CountDownLatch(1); - for (int tid = 0; tid < threadCount; tid++) { - final long seed = r.nextLong(); - threads[tid] = new Thread() { - @Override - public void run() { - try { - Random r = new Random(seed); - startingGun.await(); - for (int i = 0; i < 10; i++) { - byte bytes[] = new byte[TestUtil.nextInt(r, 1, 100000)]; - r.nextBytes(bytes); - doTest(bytes); - } - } catch (Exception e) { - throw new RuntimeException(e); - } + startInParallel(randomIntBetween(2, 6), tid -> { + try { + for (int i = 0; i < 10; i++) { + byte[] bytes = new byte[randomIntBetween(1, 100000)]; + randomBytesBetween(bytes, Byte.MIN_VALUE, Byte.MAX_VALUE); + doTest(bytes); } - }; - threads[tid].start(); - } - startingGun.countDown(); - for (Thread t : threads) { - t.join(); - } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); } public void testLineDocs() throws IOException { @@ -91,40 +73,24 @@ public void testLineDocs() throws IOException { } public void testLineDocsThreads() throws Exception { - final Random r = random(); - int threadCount = TestUtil.nextInt(r, 2, 6); - Thread[] threads = new Thread[threadCount]; - final CountDownLatch startingGun = new CountDownLatch(1); - for (int tid = 0; tid < threadCount; tid++) { - final long seed = r.nextLong(); - threads[tid] = new Thread() { - @Override - public void run() { - try { - Random r = new Random(seed); - startingGun.await(); - LineFileDocs lineFileDocs = new LineFileDocs(r); - for (int i = 0; i < 10; i++) { - int numDocs = TestUtil.nextInt(r, 1, 200); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - for (int j = 0; j < numDocs; j++) { - String s = lineFileDocs.nextDoc().get("body"); - bos.write(s.getBytes(StandardCharsets.UTF_8)); - } - doTest(bos.toByteArray()); - } - lineFileDocs.close(); - } catch (Exception e) { - throw new RuntimeException(e); + int threadCount = randomIntBetween(2, 6); + startInParallel(threadCount, tid -> { + try { + LineFileDocs lineFileDocs = new LineFileDocs(random()); + for (int i = 0; i < 10; i++) { + int numDocs = randomIntBetween(1, 200); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + for (int j = 0; j < numDocs; j++) { + String s = lineFileDocs.nextDoc().get("body"); + bos.write(s.getBytes(StandardCharsets.UTF_8)); } + doTest(bos.toByteArray()); } - }; - threads[tid].start(); - } - startingGun.countDown(); - for (Thread t : threads) { - t.join(); - } + lineFileDocs.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); } public void testRepetitionsL() throws IOException { @@ -151,48 +117,32 @@ public void testRepetitionsL() throws IOException { } public void testRepetitionsLThreads() throws Exception { - final Random r = random(); - int threadCount = TestUtil.nextInt(r, 2, 6); - Thread[] threads = new Thread[threadCount]; - final CountDownLatch startingGun = new CountDownLatch(1); - for (int tid = 0; tid < threadCount; tid++) { - final long seed = r.nextLong(); - threads[tid] = new Thread() { - @Override - public void run() { - try { - Random r = new Random(seed); - startingGun.await(); - for (int i = 0; i < 10; i++) { - int numLongs = TestUtil.nextInt(r, 1, 10000); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - long theValue = r.nextLong(); - for (int j = 0; j < numLongs; j++) { - if (r.nextInt(10) == 0) { - theValue = r.nextLong(); - } - bos.write((byte) (theValue >>> 56)); - bos.write((byte) (theValue >>> 48)); - bos.write((byte) (theValue >>> 40)); - bos.write((byte) (theValue >>> 32)); - bos.write((byte) (theValue >>> 24)); - bos.write((byte) (theValue >>> 16)); - bos.write((byte) (theValue >>> 8)); - bos.write((byte) theValue); - } - doTest(bos.toByteArray()); + int threadCount = randomIntBetween(2, 6); + startInParallel(threadCount, tid -> { + try { + for (int i = 0; i < 10; i++) { + int numLongs = randomIntBetween(1, 10000); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + long theValue = randomLong(); + for (int j = 0; j < numLongs; j++) { + if (randomInt(10) == 0) { + theValue = randomLong(); } - } catch (Exception e) { - throw new RuntimeException(e); + bos.write((byte) (theValue >>> 56)); + bos.write((byte) (theValue >>> 48)); + bos.write((byte) (theValue >>> 40)); + bos.write((byte) (theValue >>> 32)); + bos.write((byte) (theValue >>> 24)); + bos.write((byte) (theValue >>> 16)); + bos.write((byte) (theValue >>> 8)); + bos.write((byte) theValue); } + doTest(bos.toByteArray()); } - }; - threads[tid].start(); - } - startingGun.countDown(); - for (Thread t : threads) { - t.join(); - } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); } public void testRepetitionsI() throws IOException { @@ -215,44 +165,28 @@ public void testRepetitionsI() throws IOException { } public void testRepetitionsIThreads() throws Exception { - final Random r = random(); - int threadCount = TestUtil.nextInt(r, 2, 6); - Thread[] threads = new Thread[threadCount]; - final CountDownLatch startingGun = new CountDownLatch(1); - for (int tid = 0; tid < threadCount; tid++) { - final long seed = r.nextLong(); - threads[tid] = new Thread() { - @Override - public void run() { - try { - Random r = new Random(seed); - startingGun.await(); - for (int i = 0; i < 10; i++) { - int numInts = TestUtil.nextInt(r, 1, 20000); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - int theValue = r.nextInt(); - for (int j = 0; j < numInts; j++) { - if (r.nextInt(10) == 0) { - theValue = r.nextInt(); - } - bos.write((byte) (theValue >>> 24)); - bos.write((byte) (theValue >>> 16)); - bos.write((byte) (theValue >>> 8)); - bos.write((byte) theValue); - } - doTest(bos.toByteArray()); + int threadCount = randomIntBetween(2, 6); + startInParallel(threadCount, tid -> { + try { + for (int i = 0; i < 10; i++) { + int numInts = randomIntBetween(1, 20000); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + int theValue = randomInt(); + for (int j = 0; j < numInts; j++) { + if (randomInt(10) == 0) { + theValue = randomInt(); } - } catch (Exception e) { - throw new RuntimeException(e); + bos.write((byte) (theValue >>> 24)); + bos.write((byte) (theValue >>> 16)); + bos.write((byte) (theValue >>> 8)); + bos.write((byte) theValue); } + doTest(bos.toByteArray()); } - }; - threads[tid].start(); - } - startingGun.countDown(); - for (Thread t : threads) { - t.join(); - } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); } public void testRepetitionsS() throws IOException { @@ -330,42 +264,26 @@ private void addBytes(Random r, ByteArrayOutputStream bos) throws IOException { } public void testRepetitionsSThreads() throws Exception { - final Random r = random(); - int threadCount = TestUtil.nextInt(r, 2, 6); - Thread[] threads = new Thread[threadCount]; - final CountDownLatch startingGun = new CountDownLatch(1); - for (int tid = 0; tid < threadCount; tid++) { - final long seed = r.nextLong(); - threads[tid] = new Thread() { - @Override - public void run() { - try { - Random r = new Random(seed); - startingGun.await(); - for (int i = 0; i < 10; i++) { - int numShorts = TestUtil.nextInt(r, 1, 40000); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - short theValue = (short) r.nextInt(65535); - for (int j = 0; j < numShorts; j++) { - if (r.nextInt(10) == 0) { - theValue = (short) r.nextInt(65535); - } - bos.write((byte) (theValue >>> 8)); - bos.write((byte) theValue); - } - doTest(bos.toByteArray()); + int threadCount = randomIntBetween(2, 6); + startInParallel(threadCount, tid -> { + try { + for (int i = 0; i < 10; i++) { + int numShorts = randomIntBetween(1, 40000); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + short theValue = (short) randomInt(65535); + for (int j = 0; j < numShorts; j++) { + if (randomInt(10) == 0) { + theValue = (short) randomInt(65535); } - } catch (Exception e) { - throw new RuntimeException(e); + bos.write((byte) (theValue >>> 8)); + bos.write((byte) theValue); } + doTest(bos.toByteArray()); } - }; - threads[tid].start(); - } - startingGun.countDown(); - for (Thread t : threads) { - t.join(); - } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); } public void testCompressUncompressWithCorruptions() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/CountingFilterInputStreamTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/CountingFilterInputStreamTests.java new file mode 100644 index 0000000000000..344a36e03faf7 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/io/stream/CountingFilterInputStreamTests.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.io.stream; + +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.test.ESTestCase; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.equalTo; + +public class CountingFilterInputStreamTests extends ESTestCase { + + public void testBytesCounting() throws IOException { + final byte[] input = randomByteArrayOfLength(between(500, 1000)); + final var in = new CountingFilterInputStream(new ByteArrayInputStream(input)); + + assertThat(in.getBytesRead(), equalTo(0)); + + final CheckedConsumer readRandomly = (Integer length) -> { + switch (between(0, 3)) { + case 0 -> { + for (var i = 0; i < length; i++) { + final int bytesBefore = in.getBytesRead(); + final int result = in.read(); + assertThat((byte) result, equalTo(input[bytesBefore])); + assertThat(in.getBytesRead(), equalTo(bytesBefore + 1)); + } + } + case 1 -> { + final int bytesBefore = in.getBytesRead(); + final byte[] b; + if (randomBoolean()) { + b = in.readNBytes(length); + } else { + b = new byte[length]; + assertThat(in.read(b), equalTo(length)); + } + assertArrayEquals(Arrays.copyOfRange(input, bytesBefore, bytesBefore + length), b); + assertThat(in.getBytesRead(), equalTo(bytesBefore + length)); + } + case 2 -> { + final int bytesBefore = in.getBytesRead(); + final byte[] b = new byte[length * between(2, 5)]; + if (randomBoolean()) { + assertThat(in.read(b, length / 2, length), equalTo(length)); + } else { + assertThat(in.readNBytes(b, length / 2, length), equalTo(length)); + } + assertArrayEquals( + Arrays.copyOfRange(input, bytesBefore, bytesBefore + length), + Arrays.copyOfRange(b, length / 2, length / 2 + length) + ); + assertThat(in.getBytesRead(), equalTo(bytesBefore + length)); + } + case 3 -> { + final int bytesBefore = in.getBytesRead(); + if (randomBoolean()) { + assertThat((int) in.skip(length), equalTo(length)); + } else { + in.skipNBytes(length); + } + assertThat(in.getBytesRead(), equalTo(bytesBefore + length)); + } + default -> fail("unexpected"); + } + }; + + while (in.getBytesRead() < input.length - 50) { + readRandomly.accept(between(1, 30)); + } + + final int bytesBefore = in.getBytesRead(); + final byte[] remainingBytes = in.readAllBytes(); + assertThat(in.getBytesRead(), equalTo(bytesBefore + remainingBytes.length)); + assertThat(in.getBytesRead(), equalTo(input.length)); + + // Read beyond available data has no effect + assertThat(in.read(), equalTo(-1)); + final byte[] bytes = new byte[between(20, 30)]; + assertThat(in.read(bytes), equalTo(-1)); + IntStream.range(0, bytes.length).forEach(i -> assertThat(bytes[i], equalTo((byte) 0))); + assertThat(in.read(bytes, between(3, 5), between(5, 10)), equalTo(-1)); + IntStream.range(0, bytes.length).forEach(i -> assertThat(bytes[i], equalTo((byte) 0))); + assertThat(in.skip(between(10, 20)), equalTo(0L)); + + assertThat(in.getBytesRead(), equalTo(input.length)); + } +} diff --git a/server/src/test/java/org/elasticsearch/common/lucene/search/morelikethis/XMoreLikeThisTests.java b/server/src/test/java/org/elasticsearch/common/lucene/search/morelikethis/XMoreLikeThisTests.java index 3594bb276fe0a..95feefa623572 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/search/morelikethis/XMoreLikeThisTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/search/morelikethis/XMoreLikeThisTests.java @@ -8,6 +8,11 @@ package org.elasticsearch.common.lucene.search.morelikethis; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; @@ -15,20 +20,26 @@ import org.apache.lucene.queries.mlt.MoreLikeThis; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.similarities.ClassicSimilarity; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.analysis.MockTokenFilter; import org.apache.lucene.tests.analysis.MockTokenizer; import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.util.ArrayUtil; +import org.elasticsearch.common.lucene.search.XMoreLikeThis; import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.io.StringReader; import java.util.Arrays; import java.util.List; +import java.util.Locale; public class XMoreLikeThisTests extends ESTestCase { - private void addDoc(RandomIndexWriter writer, String[] texts) throws IOException { + private void addDoc(RandomIndexWriter writer, String... texts) throws IOException { Document doc = new Document(); for (String text : texts) { doc.add(newTextField("text", text, Field.Store.YES)); @@ -36,6 +47,97 @@ private void addDoc(RandomIndexWriter writer, String[] texts) throws IOException writer.addDocument(doc); } + // Copied from Lucene. See Lucene: https://issues.apache.org/jira/browse/LUCENE-8756 + public void testCustomFrequency() throws IOException { + Analyzer analyzer = new Analyzer() { + + @Override + protected TokenStreamComponents createComponents(String fieldName) { + MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false, 100); + MockTokenFilter filter = new MockTokenFilter(tokenizer, MockTokenFilter.EMPTY_STOPSET); + return new TokenStreamComponents(tokenizer, addCustomTokenFilter(filter)); + } + + TokenStream addCustomTokenFilter(TokenStream input) { + return new TokenFilter(input) { + final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + final TermFrequencyAttribute tfAtt = addAttribute(TermFrequencyAttribute.class); + + @Override + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + final char[] buffer = termAtt.buffer(); + final int length = termAtt.length(); + for (int i = 0; i < length; i++) { + if (buffer[i] == '|') { + termAtt.setLength(i); + i++; + tfAtt.setTermFrequency(ArrayUtil.parseInt(buffer, i, length - i)); + return true; + } + } + return true; + } + return false; + } + }; + } + }; + + Directory directory = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), directory); + + // Add series of docs with specific information for MoreLikeThis + addDoc(writer, "text", "lucene"); + addDoc(writer, "text", "lucene release"); + addDoc(writer, "text", "apache"); + addDoc(writer, "text", "apache lucene"); + + // one more time to increase the doc frequencies + addDoc(writer, "text", "lucene2"); + addDoc(writer, "text", "lucene2 release2"); + addDoc(writer, "text", "apache2"); + addDoc(writer, "text", "apache2 lucene2"); + + addDoc(writer, "text2", "lucene2"); + addDoc(writer, "text2", "lucene2 release2"); + addDoc(writer, "text2", "apache2"); + addDoc(writer, "text2", "apache2 lucene2"); + + IndexReader reader = writer.getReader(); + writer.close(); + XMoreLikeThis mlt = new XMoreLikeThis(reader, new ClassicSimilarity()); + mlt.setMinDocFreq(0); + mlt.setMinTermFreq(1); + mlt.setMinWordLen(1); + mlt.setAnalyzer(analyzer); + mlt.setFieldNames(new String[] { "text" }); + mlt.setBoost(true); + + final double boost10 = ((BooleanQuery) mlt.like("text", new StringReader("lucene|10 release|1"))).clauses() + .stream() + .map(BooleanClause::getQuery) + .map(BoostQuery.class::cast) + .filter(x -> ((TermQuery) x.getQuery()).getTerm().text().equals("lucene")) + .mapToDouble(BoostQuery::getBoost) + .sum(); + + final double boost1 = ((BooleanQuery) mlt.like("text", new StringReader("lucene|1 release|1"))).clauses() + .stream() + .map(BooleanClause::getQuery) + .map(BoostQuery.class::cast) + .filter(x -> ((TermQuery) x.getQuery()).getTerm().text().equals("lucene")) + .mapToDouble(BoostQuery::getBoost) + .sum(); + + // mlt should use the custom frequencies provided by the analyzer so "lucene|10" should be + // boosted more than "lucene|1" + assertTrue(String.format(Locale.ROOT, "%s should be greater than %s", boost10, boost1), boost10 > boost1); + analyzer.close(); + reader.close(); + directory.close(); + } + public void testTopN() throws Exception { int numDocs = 100; int topN = 25; diff --git a/server/src/test/java/org/elasticsearch/common/regex/RegexTests.java b/server/src/test/java/org/elasticsearch/common/regex/RegexTests.java index f010b233f40c2..bebc245172c1a 100644 --- a/server/src/test/java/org/elasticsearch/common/regex/RegexTests.java +++ b/server/src/test/java/org/elasticsearch/common/regex/RegexTests.java @@ -14,6 +14,7 @@ import java.io.IOException; import java.util.Locale; import java.util.Random; +import java.util.function.Predicate; import java.util.regex.Pattern; import static org.elasticsearch.test.LambdaMatchers.falseWith; @@ -236,4 +237,16 @@ public void testSimpleMatcher() { assertThat(Regex.simpleMatcher("a*c", "x*z"), falseWith("abd")); assertThat(Regex.simpleMatcher("a*c", "x*z"), falseWith("xyy")); } + + public void testThousandsAndLongPattern() throws IOException { + String[] patterns = new String[10000]; + for (int i = 0; i < patterns.length / 2; i++) { + patterns[i * 2] = randomAlphaOfLength(10); + patterns[i * 2 + 1] = patterns[i * 2] + ".*"; + } + Predicate predicate = Regex.simpleMatcher(patterns); + for (int i = 0; i < patterns.length / 2; i++) { + assertTrue(predicate.test(patterns[i])); + } + } } diff --git a/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java b/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java index e10cca58f8b78..a9b7cb74e548e 100644 --- a/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java +++ b/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java @@ -695,6 +695,23 @@ public void testMinMillis() { assertThat(javaFormatted, equalTo("-292275055-05-16T16:47:04.192Z")); } + public void testMinNanos() { + String javaFormatted = DateFormatter.forPattern("strict_date_optional_time").formatNanos(Long.MIN_VALUE); + assertThat(javaFormatted, equalTo("1677-09-21T00:12:43.145Z")); + + // Note - since this is a negative value, the nanoseconds are being subtracted, which is why we get this value. + javaFormatted = DateFormatter.forPattern("strict_date_optional_time_nanos").formatNanos(Long.MIN_VALUE); + assertThat(javaFormatted, equalTo("1677-09-21T00:12:43.145224192Z")); + } + + public void testMaxNanos() { + String javaFormatted = DateFormatter.forPattern("strict_date_optional_time").formatNanos(Long.MAX_VALUE); + assertThat(javaFormatted, equalTo("2262-04-11T23:47:16.854Z")); + + javaFormatted = DateFormatter.forPattern("strict_date_optional_time_nanos").formatNanos(Long.MAX_VALUE); + assertThat(javaFormatted, equalTo("2262-04-11T23:47:16.854775807Z")); + } + public void testYearParsing() { // this one is considered a year assertParses("1234", "strict_date_optional_time||epoch_millis"); diff --git a/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java b/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java index e3f2522de4813..10cd102dfbb82 100644 --- a/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java @@ -276,6 +276,36 @@ public void testFillFalseSingleWord() { } } + public void testFillTrueAfterArrayLength() { + try (BitArray bitArray = new BitArray(1, BigArrays.NON_RECYCLING_INSTANCE)) { + int from = 100; + int to = 200; + + bitArray.fill(from, to, true); + + for (int i = 0; i < to; i++) { + if (i < from) { + assertFalse(bitArray.get(i)); + } else { + assertTrue(bitArray.get(i)); + } + } + } + } + + public void testFillFalseAfterArrayLength() { + try (BitArray bitArray = new BitArray(1, BigArrays.NON_RECYCLING_INSTANCE)) { + int from = 100; + int to = 200; + + bitArray.fill(from, to, false); + + for (int i = 0; i < to; i++) { + assertFalse(bitArray.get(i)); + } + } + } + public void testSerialize() throws Exception { int initial = randomIntBetween(1, 100_000); BitArray bits1 = new BitArray(initial, BigArrays.NON_RECYCLING_INSTANCE); diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/RunOnceTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/RunOnceTests.java index 1fe436347a50f..eeaf41cc80569 100644 --- a/server/src/test/java/org/elasticsearch/common/util/concurrent/RunOnceTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/RunOnceTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ReachabilityChecker; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; public class RunOnceTests extends ESTestCase { @@ -33,26 +32,7 @@ public void testRunOnce() { public void testRunOnceConcurrently() throws InterruptedException { final AtomicInteger counter = new AtomicInteger(0); final RunOnce runOnce = new RunOnce(counter::incrementAndGet); - - final Thread[] threads = new Thread[between(3, 10)]; - final CountDownLatch latch = new CountDownLatch(1 + threads.length); - for (int i = 0; i < threads.length; i++) { - threads[i] = new Thread(() -> { - latch.countDown(); - try { - latch.await(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - runOnce.run(); - }); - threads[i].start(); - } - - latch.countDown(); - for (Thread thread : threads) { - thread.join(); - } + startInParallel(between(3, 10), i -> runOnce.run()); assertTrue(runOnce.hasRun()); assertEquals(1, counter.get()); } diff --git a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java index 247f60b7228e3..adab51a37d2bf 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java +++ b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java @@ -357,46 +357,23 @@ class Int { flipFlop[i] = new AtomicInteger(); } - Thread[] threads = new Thread[randomIntBetween(2, 5)]; - final CountDownLatch latch = new CountDownLatch(1); + final int threads = randomIntBetween(2, 5); final int iters = scaledRandomIntBetween(10000, 100000); - for (int i = 0; i < threads.length; i++) { - threads[i] = new Thread() { - @Override - public void run() { - try { - latch.await(); - } catch (InterruptedException e) { - fail(e.getMessage()); - } - for (int i = 0; i < iters; i++) { - int shard = randomIntBetween(0, counts.length - 1); - try { - try ( - ShardLock autoCloses = env.shardLock( - new ShardId("foo", "fooUUID", shard), - "1", - scaledRandomIntBetween(0, 10) - ) - ) { - counts[shard].value++; - countsAtomic[shard].incrementAndGet(); - assertEquals(flipFlop[shard].incrementAndGet(), 1); - assertEquals(flipFlop[shard].decrementAndGet(), 0); - } - } catch (ShardLockObtainFailedException ex) { - // ok - } + startInParallel(threads, tid -> { + for (int i = 0; i < iters; i++) { + int shard = randomIntBetween(0, counts.length - 1); + try { + try (ShardLock autoCloses = env.shardLock(new ShardId("foo", "fooUUID", shard), "1", scaledRandomIntBetween(0, 10))) { + counts[shard].value++; + countsAtomic[shard].incrementAndGet(); + assertEquals(flipFlop[shard].incrementAndGet(), 1); + assertEquals(flipFlop[shard].decrementAndGet(), 0); } + } catch (ShardLockObtainFailedException ex) { + // ok } - }; - threads[i].start(); - } - latch.countDown(); // fire the threads up - for (int i = 0; i < threads.length; i++) { - threads[i].join(); - } - + } + }); assertTrue("LockedShards: " + env.lockedShards(), env.lockedShards().isEmpty()); for (int i = 0; i < counts.length; i++) { assertTrue(counts[i].value > 0); diff --git a/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java b/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java index d49347a0dd3fc..5c871c8b912a0 100644 --- a/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java +++ b/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java @@ -12,7 +12,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.bytes.BytesArray; @@ -706,7 +705,7 @@ private static void writeContent(OutputStream bso, ChunkedRestResponseBodyPart c if (content.isLastPart()) { return; } - writeContent(bso, PlainActionFuture.get(content::getNextPart)); + writeContent(bso, safeAwait(content::getNextPart)); } }; diff --git a/server/src/test/java/org/elasticsearch/index/CompositeIndexEventListenerTests.java b/server/src/test/java/org/elasticsearch/index/CompositeIndexEventListenerTests.java index 81d1c21ac2751..8f57c733ba1a4 100644 --- a/server/src/test/java/org/elasticsearch/index/CompositeIndexEventListenerTests.java +++ b/server/src/test/java/org/elasticsearch/index/CompositeIndexEventListenerTests.java @@ -12,16 +12,14 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; -import org.elasticsearch.action.support.PlainActionFuture; -import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardTestCase; import org.elasticsearch.test.MockLog; import org.hamcrest.Matchers; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -72,16 +70,14 @@ private void runStep() { }).collect(Collectors.toList()) ); - final CheckedRunnable beforeIndexShardRecoveryRunner = () -> assertNull( - PlainActionFuture.get( - fut -> indexEventListener.beforeIndexShardRecovery(shard, shard.indexSettings(), fut), - 10, - TimeUnit.SECONDS - ) + final Consumer> beforeIndexShardRecoveryRunner = l -> indexEventListener.beforeIndexShardRecovery( + shard, + shard.indexSettings(), + l ); failAtStep.set(stepCount); - beforeIndexShardRecoveryRunner.run(); + assertNull(safeAwait(beforeIndexShardRecoveryRunner::accept)); assertEquals(stepCount, stepNumber.getAndSet(0)); if (stepCount > 0) { @@ -95,7 +91,9 @@ private void runStep() { ); failAtStep.set(between(0, stepCount - 1)); - final var rootCause = getRootCause(expectThrows(ElasticsearchException.class, beforeIndexShardRecoveryRunner::run)); + final var rootCause = getRootCause( + asInstanceOf(ElasticsearchException.class, safeAwaitFailure(beforeIndexShardRecoveryRunner)) + ); assertEquals("simulated failure at step " + failAtStep.get(), rootCause.getMessage()); assertEquals(failAtStep.get() + 1, stepNumber.getAndSet(0)); mockLog.assertAllExpectationsMatched(); @@ -138,12 +136,10 @@ private void runStep() { }).collect(Collectors.toList()) ); - final CheckedRunnable afterIndexShardRecoveryRunner = () -> assertNull( - PlainActionFuture.get(fut -> indexEventListener.afterIndexShardRecovery(shard, fut), 10, TimeUnit.SECONDS) - ); + final Consumer> afterIndexShardRecoveryRunner = l -> indexEventListener.afterIndexShardRecovery(shard, l); failAtStep.set(stepCount); - afterIndexShardRecoveryRunner.run(); + assertNull(safeAwait(afterIndexShardRecoveryRunner::accept)); assertEquals(stepCount, stepNumber.getAndSet(0)); if (stepCount > 0) { @@ -157,7 +153,9 @@ private void runStep() { ); failAtStep.set(between(0, stepCount - 1)); - final var rootCause = getRootCause(expectThrows(ElasticsearchException.class, afterIndexShardRecoveryRunner::run)); + final var rootCause = getRootCause( + asInstanceOf(ElasticsearchException.class, safeAwaitFailure(afterIndexShardRecoveryRunner)) + ); assertEquals("simulated failure at step " + failAtStep.get(), rootCause.getMessage()); assertEquals(failAtStep.get() + 1, stepNumber.getAndSet(0)); mockLog.assertAllExpectationsMatched(); diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index d2b2926af7d4c..379adc9ce517a 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -219,7 +219,7 @@ private Sort buildIndexSort(IndexSettings indexSettings, Map indexFieldDataService.getForField( ft, - new FieldDataContext("test", s, Set::of, MappedFieldType.FielddataOperation.SEARCH) + new FieldDataContext("test", indexSettings, s, Set::of, MappedFieldType.FielddataOperation.SEARCH) ) ); } diff --git a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java index caddc7d5ea5af..84a9682635c8c 100644 --- a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java @@ -16,7 +16,7 @@ public class LogsIndexModeTests extends ESTestCase { public void testLogsIndexModeSetting() { - assertThat(IndexSettings.MODE.get(buildSettings()), equalTo(IndexMode.LOGS)); + assertThat(IndexSettings.MODE.get(buildSettings()), equalTo(IndexMode.LOGSDB)); } public void testSortField() { @@ -25,9 +25,9 @@ public void testSortField() { .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "agent_id") .build(); final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings); - assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS)); + assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB)); final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - assertThat(settings.getMode(), equalTo(IndexMode.LOGS)); + assertThat(settings.getMode(), equalTo(IndexMode.LOGSDB)); assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey()))); } @@ -38,9 +38,9 @@ public void testSortMode() { .put(IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey(), "max") .build(); final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings); - assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS)); + assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB)); final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - assertThat(settings.getMode(), equalTo(IndexMode.LOGS)); + assertThat(settings.getMode(), equalTo(IndexMode.LOGSDB)); assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey()))); assertThat("max", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey()))); } @@ -52,9 +52,9 @@ public void testSortOrder() { .put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "desc") .build(); final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings); - assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS)); + assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB)); final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - assertThat(settings.getMode(), equalTo(IndexMode.LOGS)); + assertThat(settings.getMode(), equalTo(IndexMode.LOGSDB)); assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey()))); assertThat("desc", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey()))); } @@ -66,15 +66,15 @@ public void testSortMissing() { .put(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_last") .build(); final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings); - assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS)); + assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB)); final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - assertThat(settings.getMode(), equalTo(IndexMode.LOGS)); + assertThat(settings.getMode(), equalTo(IndexMode.LOGSDB)); assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey()))); assertThat("_last", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey()))); } private Settings buildSettings() { - return Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGS.getName()).build(); + return Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()).build(); } private String getIndexSetting(final IndexSettings settings, final String name) { diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index 6118a84814462..f9b39bd665abd 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -121,9 +121,10 @@ public void testSetDefaultTimeRangeValue() { public void testRequiredRouting() { Settings s = getSettings(); + var mapperService = new TestMapperServiceBuilder().settings(s).applyDefaultMapping(false).build(); Exception e = expectThrows( IllegalArgumentException.class, - () -> createMapperService(s, topMapping(b -> b.startObject("_routing").field("required", true).endObject())) + () -> withMapping(mapperService, topMapping(b -> b.startObject("_routing").field("required", true).endObject())) ); assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=time_series]")); } diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index 3c687f1792d0d..2c2af49e7d062 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -10,7 +10,6 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.IntField; @@ -73,7 +72,6 @@ public void testBestCompression() throws Exception { public void testLegacyDefault() throws Exception { Codec codec = createCodecService().codec("legacy_default"); - assertThat(codec, Matchers.instanceOf(Lucene99Codec.class)); assertThat(codec.storedFieldsFormat(), Matchers.instanceOf(Lucene90StoredFieldsFormat.class)); // Make sure the legacy codec is writable try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setCodec(codec))) { @@ -87,7 +85,6 @@ public void testLegacyDefault() throws Exception { public void testLegacyBestCompression() throws Exception { Codec codec = createCodecService().codec("legacy_best_compression"); - assertThat(codec, Matchers.instanceOf(Lucene99Codec.class)); assertThat(codec.storedFieldsFormat(), Matchers.instanceOf(Lucene90StoredFieldsFormat.class)); // Make sure the legacy codec is writable try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setCodec(codec))) { diff --git a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java index 525fa31673494..122e238fb6346 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java @@ -188,7 +188,7 @@ public void testUseTimeSeriesModeAndCodecEnabled() throws IOException { } public void testLogsIndexMode() throws IOException { - PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.LOGS, MAPPING_3); + PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.LOGSDB, MAPPING_3); assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true)); assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("hostname")), is(true)); assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("response_size")), is(true)); diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index c668cfbb502a2..cc636fb1bc995 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -1252,24 +1252,15 @@ public void testSyncTranslogConcurrently() throws Exception { assertThat(commitInfo.localCheckpoint(), equalTo(engine.getProcessedLocalCheckpoint())); } }; - final Thread[] threads = new Thread[randomIntBetween(2, 4)]; - final Phaser phaser = new Phaser(threads.length); globalCheckpoint.set(engine.getProcessedLocalCheckpoint()); - for (int i = 0; i < threads.length; i++) { - threads[i] = new Thread(() -> { - phaser.arriveAndAwaitAdvance(); - try { - engine.syncTranslog(); - checker.run(); - } catch (IOException e) { - throw new AssertionError(e); - } - }); - threads[i].start(); - } - for (Thread thread : threads) { - thread.join(); - } + startInParallel(randomIntBetween(2, 4), i -> { + try { + engine.syncTranslog(); + checker.run(); + } catch (IOException e) { + throw new AssertionError(e); + } + }); checker.run(); } @@ -2419,8 +2410,6 @@ public void testConcurrentGetAndSetOnPrimary() throws IOException, InterruptedEx MapperService mapperService = createMapperService(); MappingLookup mappingLookup = mapperService.mappingLookup(); DocumentParser documentParser = mapperService.documentParser(); - Thread[] thread = new Thread[randomIntBetween(3, 5)]; - CountDownLatch startGun = new CountDownLatch(thread.length); final int opsPerThread = randomIntBetween(10, 20); class OpAndVersion { final long version; @@ -2438,54 +2427,39 @@ class OpAndVersion { ParsedDocument doc = testParsedDocument("1", null, testDocument(), bytesArray(""), null); final BytesRef uidTerm = newUid(doc); engine.index(indexForDoc(doc)); - for (int i = 0; i < thread.length; i++) { - thread[i] = new Thread(() -> { - startGun.countDown(); - safeAwait(startGun); - for (int op = 0; op < opsPerThread; op++) { - Engine.Get engineGet = new Engine.Get(true, false, doc.id()); - try (Engine.GetResult get = engine.get(engineGet, mappingLookup, documentParser, randomSearcherWrapper())) { - FieldsVisitor visitor = new FieldsVisitor(true); - get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor); - List values = new ArrayList<>(Strings.commaDelimitedListToSet(visitor.source().utf8ToString())); - String removed = op % 3 == 0 && values.size() > 0 ? values.remove(0) : null; - String added = "v_" + idGenerator.incrementAndGet(); - values.add(added); - Engine.Index index = new Engine.Index( - uidTerm, - testParsedDocument( - "1", - null, - testDocument(), - bytesArray(Strings.collectionToCommaDelimitedString(values)), - null - ), - UNASSIGNED_SEQ_NO, - 2, - get.version(), - VersionType.INTERNAL, - PRIMARY, - System.currentTimeMillis(), - -1, - false, - UNASSIGNED_SEQ_NO, - 0 - ); - Engine.IndexResult indexResult = engine.index(index); - if (indexResult.getResultType() == Engine.Result.Type.SUCCESS) { - history.add(new OpAndVersion(indexResult.getVersion(), removed, added)); - } - - } catch (IOException e) { - throw new AssertionError(e); + startInParallel(randomIntBetween(3, 5), i -> { + for (int op = 0; op < opsPerThread; op++) { + Engine.Get engineGet = new Engine.Get(true, false, doc.id()); + try (Engine.GetResult get = engine.get(engineGet, mappingLookup, documentParser, randomSearcherWrapper())) { + FieldsVisitor visitor = new FieldsVisitor(true); + get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor); + List values = new ArrayList<>(Strings.commaDelimitedListToSet(visitor.source().utf8ToString())); + String removed = op % 3 == 0 && values.size() > 0 ? values.remove(0) : null; + String added = "v_" + idGenerator.incrementAndGet(); + values.add(added); + Engine.Index index = new Engine.Index( + uidTerm, + testParsedDocument("1", null, testDocument(), bytesArray(Strings.collectionToCommaDelimitedString(values)), null), + UNASSIGNED_SEQ_NO, + 2, + get.version(), + VersionType.INTERNAL, + PRIMARY, + System.currentTimeMillis(), + -1, + false, + UNASSIGNED_SEQ_NO, + 0 + ); + Engine.IndexResult indexResult = engine.index(index); + if (indexResult.getResultType() == Engine.Result.Type.SUCCESS) { + history.add(new OpAndVersion(indexResult.getVersion(), removed, added)); } + } catch (IOException e) { + throw new AssertionError(e); } - }); - thread[i].start(); - } - for (int i = 0; i < thread.length; i++) { - thread[i].join(); - } + } + }); List sortedHistory = new ArrayList<>(history); sortedHistory.sort(Comparator.comparing(o -> o.version)); Set currentValues = new HashSet<>(); @@ -3231,13 +3205,9 @@ public void testCurrentTranslogUUIIDIsCommitted() throws IOException { engine.syncTranslog(); // to advance persisted local checkpoint assertEquals(engine.getProcessedLocalCheckpoint(), engine.getPersistedLocalCheckpoint()); globalCheckpoint.set(engine.getPersistedLocalCheckpoint()); - expectThrows( + asInstanceOf( IllegalStateException.class, - () -> PlainActionFuture.get( - future -> engine.recoverFromTranslog(translogHandler, Long.MAX_VALUE, future), - 30, - TimeUnit.SECONDS - ) + safeAwaitFailure(Void.class, listener -> engine.recoverFromTranslog(translogHandler, Long.MAX_VALUE, listener)) ); Map userData = engine.getLastCommittedSegmentInfos().getUserData(); assertEquals(engine.getTranslog().getTranslogUUID(), userData.get(Translog.TRANSLOG_UUID_KEY)); @@ -4363,7 +4333,6 @@ public Engine.Index appendOnlyReplica(ParsedDocument doc, boolean retry, final l } public void testRetryConcurrently() throws InterruptedException, IOException { - Thread[] thread = new Thread[randomIntBetween(3, 5)]; int numDocs = randomIntBetween(1000, 10000); List docs = new ArrayList<>(); final boolean primary = randomBoolean(); @@ -4389,26 +4358,17 @@ public void testRetryConcurrently() throws InterruptedException, IOException { docs.add(retryIndex); } Collections.shuffle(docs, random()); - CountDownLatch startGun = new CountDownLatch(thread.length); AtomicInteger offset = new AtomicInteger(-1); - for (int i = 0; i < thread.length; i++) { - thread[i] = new Thread(() -> { - startGun.countDown(); - safeAwait(startGun); - int docOffset; - while ((docOffset = offset.incrementAndGet()) < docs.size()) { - try { - engine.index(docs.get(docOffset)); - } catch (IOException e) { - throw new AssertionError(e); - } + startInParallel(randomIntBetween(3, 5), i -> { + int docOffset; + while ((docOffset = offset.incrementAndGet()) < docs.size()) { + try { + engine.index(docs.get(docOffset)); + } catch (IOException e) { + throw new AssertionError(e); } - }); - thread[i].start(); - } - for (int i = 0; i < thread.length; i++) { - thread[i].join(); - } + } + }); engine.refresh("test"); try (Engine.Searcher searcher = engine.acquireSearcher("test")) { int count = searcher.count(new MatchAllDocsQuery()); diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 8c583fe3976fa..1289095ae8d2c 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -141,7 +141,10 @@ public void testGetForFieldRuntimeField() { return (IndexFieldData.Builder) (cache, breakerService) -> null; }); SearchLookup searchLookup = new SearchLookup(null, null, (ctx, doc) -> null); - ifdService.getForField(ft, new FieldDataContext("qualified", () -> searchLookup, null, MappedFieldType.FielddataOperation.SEARCH)); + ifdService.getForField( + ft, + new FieldDataContext("qualified", null, () -> searchLookup, null, MappedFieldType.FielddataOperation.SEARCH) + ); assertSame(searchLookup, searchLookupSetOnce.get().get()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java index aacd98f656b45..77b37b2bde860 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java @@ -167,7 +167,7 @@ public void testDefaultsForTimeSeriesIndex() throws IOException { var source = source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> { b.field("field", Base64.getEncoder().encodeToString(randomByteArrayOfLength(10))); - b.field("@timestamp", randomMillisUpToYear9999()); + b.field("@timestamp", "2000-10-10T23:40:53.384Z"); b.field("dimension", "dimension1"); }, null); ParsedDocument doc = mapper.parse(source); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 2f8af6c5521f9..d90517e4be274 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -152,6 +152,9 @@ public void testPostingsFormat() throws IOException { assertThat(codec, instanceOf(PerFieldMapperCodec.class)); assertThat(((PerFieldMapperCodec) codec).getPostingsFormatForField("field"), instanceOf(Completion99PostingsFormat.class)); } else { + if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) { + codec = deduplicateFieldInfosCodec.delegate(); + } assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class)); assertThat( ((LegacyPerFieldMapperCodec) codec).getPostingsFormatForField("field"), diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java index 162b45cef971c..70a8fc05510ed 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java @@ -343,7 +343,7 @@ public void testParseDocumentSubFieldAccess() throws IOException { SearchLookup searchLookup = new SearchLookup( mapperService::fieldType, (mft, lookupSupplier, fdo) -> mft.fielddataBuilder( - new FieldDataContext("test", lookupSupplier, mapperService.mappingLookup()::sourcePaths, fdo) + new FieldDataContext("test", null, lookupSupplier, mapperService.mappingLookup()::sourcePaths, fdo) ).build(null, null), SourceProvider.fromStoredFields() ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java index 633ffbf1c3a3a..9609c1ee8aed5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java @@ -327,6 +327,7 @@ public void testEmptyDocumentMapper() { .item(IgnoredFieldMapper.class) .item(IgnoredSourceFieldMapper.class) .item(IndexFieldMapper.class) + .item(IndexModeFieldMapper.class) .item(NestedPathFieldMapper.class) .item(ProvidedIdFieldMapper.class) .item(RoutingFieldMapper.class) @@ -345,6 +346,7 @@ public void testEmptyDocumentMapper() { .item(IgnoredFieldMapper.CONTENT_TYPE) .item(IgnoredSourceFieldMapper.NAME) .item(IndexFieldMapper.CONTENT_TYPE) + .item(IndexModeFieldMapper.CONTENT_TYPE) .item(NestedPathFieldMapper.NAME) .item(RoutingFieldMapper.CONTENT_TYPE) .item(SeqNoFieldMapper.CONTENT_TYPE) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexModeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexModeFieldTypeTests.java new file mode 100644 index 0000000000000..d7ecafc1b85b4 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexModeFieldTypeTests.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.index.mapper; + +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.query.SearchExecutionContext; + +import java.util.Collections; +import java.util.List; +import java.util.function.Predicate; + +public class IndexModeFieldTypeTests extends ConstantFieldTypeTestCase { + + public void testTermQuery() { + MappedFieldType ft = getMappedFieldType(); + for (IndexMode mode : IndexMode.values()) { + SearchExecutionContext context = createContext(mode); + for (IndexMode other : IndexMode.values()) { + Query query = ft.termQuery(other.getName(), context); + if (other.equals(mode)) { + assertEquals(new MatchAllDocsQuery(), query); + } else { + assertEquals(new MatchNoDocsQuery(), query); + } + } + } + } + + public void testWildcardQuery() { + MappedFieldType ft = getMappedFieldType(); + + assertEquals(new MatchAllDocsQuery(), ft.wildcardQuery("stand*", null, createContext(IndexMode.STANDARD))); + assertEquals(new MatchNoDocsQuery(), ft.wildcardQuery("stand*", null, createContext(IndexMode.TIME_SERIES))); + assertEquals(new MatchNoDocsQuery(), ft.wildcardQuery("stand*", null, createContext(IndexMode.LOGSDB))); + + assertEquals(new MatchNoDocsQuery(), ft.wildcardQuery("time*", null, createContext(IndexMode.STANDARD))); + assertEquals(new MatchAllDocsQuery(), ft.wildcardQuery("time*", null, createContext(IndexMode.TIME_SERIES))); + assertEquals(new MatchNoDocsQuery(), ft.wildcardQuery("time*", null, createContext(IndexMode.LOGSDB))); + + assertEquals(new MatchNoDocsQuery(), ft.wildcardQuery("logs*", null, createContext(IndexMode.STANDARD))); + assertEquals(new MatchNoDocsQuery(), ft.wildcardQuery("logs*", null, createContext(IndexMode.TIME_SERIES))); + assertEquals(new MatchAllDocsQuery(), ft.wildcardQuery("logs*", null, createContext(IndexMode.LOGSDB))); + } + + @Override + public MappedFieldType getMappedFieldType() { + return IndexModeFieldMapper.IndexModeFieldType.INSTANCE; + } + + private SearchExecutionContext createContext(IndexMode mode) { + Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()); + if (mode != null) { + settings.put(IndexSettings.MODE.getKey(), mode); + } + if (mode == IndexMode.TIME_SERIES) { + settings.putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of("a,b,c")); + } + IndexMetadata indexMetadata = IndexMetadata.builder("index").settings(settings).numberOfShards(1).numberOfReplicas(0).build(); + IndexSettings indexSettings = new IndexSettings(indexMetadata, settings.build()); + + Predicate indexNameMatcher = pattern -> Regex.simpleMatch(pattern, "index"); + return new SearchExecutionContext( + 0, + 0, + indexSettings, + null, + null, + null, + MappingLookup.EMPTY, + null, + null, + parserConfig(), + writableRegistry(), + null, + null, + System::currentTimeMillis, + null, + indexNameMatcher, + () -> true, + null, + Collections.emptyMap(), + MapperMetrics.NOOP + ); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NodeMappingStatsTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NodeMappingStatsTests.java index 166e2c33fe263..563a860b084ea 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NodeMappingStatsTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NodeMappingStatsTests.java @@ -7,52 +7,62 @@ */ package org.elasticsearch.index.mapper; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.test.ESTestCase; import java.io.IOException; -public class NodeMappingStatsTests extends ESTestCase { - - public void testSerialize() throws IOException { - NodeMappingStats stats = randomNodeMappingStats(); - BytesStreamOutput out = new BytesStreamOutput(); - stats.writeTo(out); - StreamInput input = out.bytes().streamInput(); - NodeMappingStats read = new NodeMappingStats(input); - assertEquals(-1, input.read()); - assertEquals(stats.getTotalCount(), read.getTotalCount()); - assertEquals(stats.getTotalEstimatedOverhead(), read.getTotalEstimatedOverhead()); - } +public class NodeMappingStatsTests extends AbstractWireSerializingTestCase { - public void testEqualityAndHashCode() { - NodeMappingStats stats = randomNodeMappingStats(); - assertEquals(stats, stats); - assertEquals(stats.hashCode(), stats.hashCode()); + @Override + protected Writeable.Reader instanceReader() { + return NodeMappingStats::new; + } - NodeMappingStats stats1 = new NodeMappingStats(1L, 2L); - NodeMappingStats stats2 = new NodeMappingStats(3L, 5L); - NodeMappingStats stats3 = new NodeMappingStats(3L, 5L); + @Override + protected NodeMappingStats createTestInstance() { + return new NodeMappingStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()); + } - assertNotEquals(stats1, stats2); - assertNotEquals(stats1, stats3); - assertEquals(stats2, stats3); + @Override + protected NodeMappingStats mutateInstance(NodeMappingStats in) throws IOException { + return switch (between(0, 3)) { + case 0 -> new NodeMappingStats( + randomValueOtherThan(in.getTotalCount(), ESTestCase::randomNonNegativeLong), + in.getTotalEstimatedOverhead().getBytes(), + in.getTotalSegments(), + in.getTotalSegmentFields() + ); + case 1 -> new NodeMappingStats( + in.getTotalCount(), + randomValueOtherThan(in.getTotalCount(), ESTestCase::randomNonNegativeLong), + in.getTotalSegments(), + in.getTotalSegmentFields() + ); + case 2 -> new NodeMappingStats( + in.getTotalCount(), + in.getTotalEstimatedOverhead().getBytes(), + randomValueOtherThan(in.getTotalSegments(), ESTestCase::randomNonNegativeLong), + in.getTotalSegmentFields() + ); + case 3 -> new NodeMappingStats( + in.getTotalCount(), + in.getTotalEstimatedOverhead().getBytes(), + in.getTotalSegments(), + randomValueOtherThan(in.getTotalSegmentFields(), ESTestCase::randomNonNegativeLong) + ); + default -> throw new AssertionError("invalid option"); + }; } public void testAdd() { - NodeMappingStats stats1 = new NodeMappingStats(1L, 2L); - NodeMappingStats stats2 = new NodeMappingStats(2L, 3L); - NodeMappingStats stats3 = new NodeMappingStats(3L, 5L); + NodeMappingStats stats1 = new NodeMappingStats(1L, 2L, 4L, 6L); + NodeMappingStats stats2 = new NodeMappingStats(2L, 3L, 10L, 20L); + NodeMappingStats stats3 = new NodeMappingStats(3L, 5L, 14L, 26L); stats1.add(stats2); assertEquals(stats1, stats3); assertEquals(stats1.hashCode(), stats3.hashCode()); } - - private static NodeMappingStats randomNodeMappingStats() { - long totalCount = randomIntBetween(1, 100); - long estimatedOverhead = totalCount * 1024; - return new NodeMappingStats(totalCount, estimatedOverhead); - } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 8330cf1f5f794..4f7951c543909 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -294,7 +294,7 @@ public void testStoreParameterDefaults() throws IOException { var source = source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> { b.field("field", "1234"); if (timeSeriesIndexMode) { - b.field("@timestamp", randomMillisUpToYear9999()); + b.field("@timestamp", "2000-10-10T23:40:53.384Z"); b.field("dimension", "dimension1"); } }, null); @@ -599,7 +599,7 @@ public void testFielddata() throws IOException { Exception e = expectThrows( IllegalArgumentException.class, () -> disabledMapper.fieldType("field") - .fielddataBuilder(new FieldDataContext("index", null, null, MappedFieldType.FielddataOperation.SEARCH)) + .fielddataBuilder(new FieldDataContext("index", null, null, null, MappedFieldType.FielddataOperation.SEARCH)) ); assertThat( e.getMessage(), @@ -1309,7 +1309,7 @@ public void testEmpty() throws Exception { } : SourceProvider.fromStoredFields(); SearchLookup searchLookup = new SearchLookup(null, null, sourceProvider); IndexFieldData sfd = ft.fielddataBuilder( - new FieldDataContext("", () -> searchLookup, Set::of, MappedFieldType.FielddataOperation.SCRIPT) + new FieldDataContext("", null, () -> searchLookup, Set::of, MappedFieldType.FielddataOperation.SCRIPT) ).build(null, null); LeafFieldData lfd = sfd.load(getOnlyLeafReader(searcher.getIndexReader()).getContext()); TextDocValuesField scriptDV = (TextDocValuesField) lfd.getScriptFieldFactory("field"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index 87b107d5bd139..f05ec95fe84cb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -67,7 +67,7 @@ private DocumentMapper createDocumentMapper(String routingPath, XContentBuilder .put(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING.getKey(), 200) // Allow tests that use many dimensions .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-10-29T00:00:00Z") .build(), mappings ).documentMapper(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java index 5352bd446a80b..7ac17020f73cd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java @@ -44,7 +44,7 @@ private DocumentMapper createMapper(XContentBuilder mappings) throws IOException getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing path is required") .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-10-29T00:00:00Z") .build(), mappings ).documentMapper(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 3dd4e31b9ca3f..b044308218543 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -47,6 +47,7 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.search.lookup.SourceProvider; +import org.elasticsearch.search.vectors.VectorData; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; @@ -195,6 +196,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { .field("element_type", "bit") ) ); + // update for flat checker.registerUpdateCheck( b -> b.field("type", "dense_vector") .field("dims", dims) @@ -210,6 +212,21 @@ protected void registerParameters(ParameterChecker checker) throws IOException { .endObject(), m -> assertTrue(m.toString().contains("\"type\":\"int8_flat\"")) ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"int4_flat\"")) + ); checker.registerUpdateCheck( b -> b.field("type", "dense_vector") .field("dims", dims) @@ -240,6 +257,22 @@ protected void registerParameters(ParameterChecker checker) throws IOException { .endObject(), m -> assertTrue(m.toString().contains("\"type\":\"int8_hnsw\"")) ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"int4_hnsw\"")) + ); + // update for int8_flat checker.registerUpdateCheck( b -> b.field("type", "dense_vector") .field("dims", dims) @@ -270,6 +303,56 @@ protected void registerParameters(ParameterChecker checker) throws IOException { .endObject(), m -> assertTrue(m.toString().contains("\"type\":\"int8_hnsw\"")) ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"int4_hnsw\"")) + ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"int4_flat\"")) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_flat") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "flat") + .endObject() + ) + ); + // update for hnsw checker.registerUpdateCheck( b -> b.field("type", "dense_vector") .field("dims", dims) @@ -285,6 +368,37 @@ protected void registerParameters(ParameterChecker checker) throws IOException { .endObject(), m -> assertTrue(m.toString().contains("\"type\":\"int8_hnsw\"")) ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"int4_hnsw\"")) + ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .field("m", 100) + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"hnsw\"")) + ); checker.registerConflictCheck( "index_options", fieldMapping( @@ -304,6 +418,438 @@ protected void registerParameters(ParameterChecker checker) throws IOException { .endObject() ) ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .field("m", 32) + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .field("m", 16) + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_flat") + .endObject() + ) + ); + // update for int8_hnsw + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .field("m", 256) + .endObject(), + m -> assertTrue(m.toString().contains("\"m\":256")) + ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .field("m", 256) + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"int4_hnsw\"")) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .field("m", 32) + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .field("m", 16) + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "flat") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_flat") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject() + ) + ); + // update for int4_flat + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"int4_hnsw\"")) + ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"int8_hnsw\"")) + ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"hnsw\"")) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .field("m", 32) + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_flat") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .field("m", 32) + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "flat") + .endObject() + ) + ); + // update for int4_hnsw + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("m", 256) + .field("type", "int4_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"m\":256")) + ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .field("confidence_interval", 0.03) + .field("m", 4) + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .field("confidence_interval", 0.03) + .field("m", 100) + .endObject(), + m -> assertTrue(m.toString().contains("\"m\":100")) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .field("m", 32) + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .field("m", 16) + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .field("confidence_interval", 0.3) + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .field("m", 32) + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .field("m", 16) + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .field("m", 32) + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .field("m", 16) + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "flat") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int8_flat") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject() + ) + ); } @Override @@ -1141,7 +1687,7 @@ public void testByteVectorQueryBoundaries() throws IOException { Exception e = expectThrows( IllegalArgumentException.class, - () -> denseVectorFieldType.createKnnQuery(new float[] { 128, 0, 0 }, 3, 3, null, null, null) + () -> denseVectorFieldType.createKnnQuery(VectorData.fromFloats(new float[] { 128, 0, 0 }), 3, 3, null, null, null) ); assertThat( e.getMessage(), @@ -1150,7 +1696,7 @@ public void testByteVectorQueryBoundaries() throws IOException { e = expectThrows( IllegalArgumentException.class, - () -> denseVectorFieldType.createKnnQuery(new float[] { 0.0f, 0f, -129.0f }, 3, 3, null, null, null) + () -> denseVectorFieldType.createKnnQuery(VectorData.fromFloats(new float[] { 0.0f, 0f, -129.0f }), 3, 3, null, null, null) ); assertThat( e.getMessage(), @@ -1159,7 +1705,7 @@ public void testByteVectorQueryBoundaries() throws IOException { e = expectThrows( IllegalArgumentException.class, - () -> denseVectorFieldType.createKnnQuery(new float[] { 0.0f, 0.5f, 0.0f }, 3, 3, null, null, null) + () -> denseVectorFieldType.createKnnQuery(VectorData.fromFloats(new float[] { 0.0f, 0.5f, 0.0f }), 3, 3, null, null, null) ); assertThat( e.getMessage(), @@ -1168,7 +1714,7 @@ public void testByteVectorQueryBoundaries() throws IOException { e = expectThrows( IllegalArgumentException.class, - () -> denseVectorFieldType.createKnnQuery(new float[] { 0, 0.0f, -0.25f }, 3, 3, null, null, null) + () -> denseVectorFieldType.createKnnQuery(VectorData.fromFloats(new float[] { 0, 0.0f, -0.25f }), 3, 3, null, null, null) ); assertThat( e.getMessage(), @@ -1177,13 +1723,20 @@ public void testByteVectorQueryBoundaries() throws IOException { e = expectThrows( IllegalArgumentException.class, - () -> denseVectorFieldType.createKnnQuery(new float[] { Float.NaN, 0f, 0.0f }, 3, 3, null, null, null) + () -> denseVectorFieldType.createKnnQuery(VectorData.fromFloats(new float[] { Float.NaN, 0f, 0.0f }), 3, 3, null, null, null) ); assertThat(e.getMessage(), containsString("element_type [byte] vectors do not support NaN values but found [NaN] at dim [0];")); e = expectThrows( IllegalArgumentException.class, - () -> denseVectorFieldType.createKnnQuery(new float[] { Float.POSITIVE_INFINITY, 0f, 0.0f }, 3, 3, null, null, null) + () -> denseVectorFieldType.createKnnQuery( + VectorData.fromFloats(new float[] { Float.POSITIVE_INFINITY, 0f, 0.0f }), + 3, + 3, + null, + null, + null + ) ); assertThat( e.getMessage(), @@ -1192,7 +1745,14 @@ public void testByteVectorQueryBoundaries() throws IOException { e = expectThrows( IllegalArgumentException.class, - () -> denseVectorFieldType.createKnnQuery(new float[] { 0, Float.NEGATIVE_INFINITY, 0.0f }, 3, 3, null, null, null) + () -> denseVectorFieldType.createKnnQuery( + VectorData.fromFloats(new float[] { 0, Float.NEGATIVE_INFINITY, 0.0f }), + 3, + 3, + null, + null, + null + ) ); assertThat( e.getMessage(), @@ -1218,13 +1778,20 @@ public void testFloatVectorQueryBoundaries() throws IOException { Exception e = expectThrows( IllegalArgumentException.class, - () -> denseVectorFieldType.createKnnQuery(new float[] { Float.NaN, 0f, 0.0f }, 3, 3, null, null, null) + () -> denseVectorFieldType.createKnnQuery(VectorData.fromFloats(new float[] { Float.NaN, 0f, 0.0f }), 3, 3, null, null, null) ); assertThat(e.getMessage(), containsString("element_type [float] vectors do not support NaN values but found [NaN] at dim [0];")); e = expectThrows( IllegalArgumentException.class, - () -> denseVectorFieldType.createKnnQuery(new float[] { Float.POSITIVE_INFINITY, 0f, 0.0f }, 3, 3, null, null, null) + () -> denseVectorFieldType.createKnnQuery( + VectorData.fromFloats(new float[] { Float.POSITIVE_INFINITY, 0f, 0.0f }), + 3, + 3, + null, + null, + null + ) ); assertThat( e.getMessage(), @@ -1233,7 +1800,14 @@ public void testFloatVectorQueryBoundaries() throws IOException { e = expectThrows( IllegalArgumentException.class, - () -> denseVectorFieldType.createKnnQuery(new float[] { 0, Float.NEGATIVE_INFINITY, 0.0f }, 3, 3, null, null, null) + () -> denseVectorFieldType.createKnnQuery( + VectorData.fromFloats(new float[] { 0, Float.NEGATIVE_INFINITY, 0.0f }), + 3, + 3, + null, + null, + null + ) ); assertThat( e.getMessage(), @@ -1268,6 +1842,9 @@ public void testKnnVectorsFormat() throws IOException { assertThat(codec, instanceOf(PerFieldMapperCodec.class)); knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); } else { + if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) { + codec = deduplicateFieldInfosCodec.delegate(); + } assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class)); knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); } @@ -1303,6 +1880,9 @@ public void testKnnQuantizedFlatVectorsFormat() throws IOException { assertThat(codec, instanceOf(PerFieldMapperCodec.class)); knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); } else { + if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) { + codec = deduplicateFieldInfosCodec.delegate(); + } assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class)); knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); } @@ -1346,6 +1926,9 @@ public void testKnnQuantizedHNSWVectorsFormat() throws IOException { assertThat(codec, instanceOf(PerFieldMapperCodec.class)); knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); } else { + if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) { + codec = deduplicateFieldInfosCodec.delegate(); + } assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class)); knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); } @@ -1387,6 +1970,9 @@ public void testKnnHalfByteQuantizedHNSWVectorsFormat() throws IOException { assertThat(codec, instanceOf(PerFieldMapperCodec.class)); knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); } else { + if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) { + codec = deduplicateFieldInfosCodec.delegate(); + } assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class)); knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 2a4554091dc91..9ee895f6de003 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -120,11 +120,11 @@ public void testIsAggregatable() { public void testFielddataBuilder() { DenseVectorFieldType fft = createFloatFieldType(); - FieldDataContext fdc = new FieldDataContext("test", () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); + FieldDataContext fdc = new FieldDataContext("test", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); assertNotNull(fft.fielddataBuilder(fdc)); DenseVectorFieldType bft = createByteFieldType(); - FieldDataContext bdc = new FieldDataContext("test", () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); + FieldDataContext bdc = new FieldDataContext("test", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); assertNotNull(bft.fielddataBuilder(bdc)); } @@ -165,7 +165,7 @@ public void testCreateNestedKnnQuery() { for (int i = 0; i < dims; i++) { queryVector[i] = randomFloat(); } - Query query = field.createKnnQuery(queryVector, 10, 10, null, null, producer); + Query query = field.createKnnQuery(VectorData.fromFloats(queryVector), 10, 10, null, null, producer); assertThat(query, instanceOf(DiversifyingChildrenFloatKnnVectorQuery.class)); } { @@ -251,7 +251,7 @@ public void testFloatCreateKnnQuery() { ); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> unindexedField.createKnnQuery(new float[] { 0.3f, 0.1f, 1.0f, 0.0f }, 10, 10, null, null, null) + () -> unindexedField.createKnnQuery(VectorData.fromFloats(new float[] { 0.3f, 0.1f, 1.0f, 0.0f }), 10, 10, null, null, null) ); assertThat(e.getMessage(), containsString("to perform knn search on field [f], its mapping must have [index] set to [true]")); @@ -267,7 +267,7 @@ public void testFloatCreateKnnQuery() { ); e = expectThrows( IllegalArgumentException.class, - () -> dotProductField.createKnnQuery(new float[] { 0.3f, 0.1f, 1.0f, 0.0f }, 10, 10, null, null, null) + () -> dotProductField.createKnnQuery(VectorData.fromFloats(new float[] { 0.3f, 0.1f, 1.0f, 0.0f }), 10, 10, null, null, null) ); assertThat(e.getMessage(), containsString("The [dot_product] similarity can only be used with unit-length vectors.")); @@ -283,7 +283,7 @@ public void testFloatCreateKnnQuery() { ); e = expectThrows( IllegalArgumentException.class, - () -> cosineField.createKnnQuery(new float[] { 0.0f, 0.0f, 0.0f, 0.0f }, 10, 10, null, null, null) + () -> cosineField.createKnnQuery(VectorData.fromFloats(new float[] { 0.0f, 0.0f, 0.0f, 0.0f }), 10, 10, null, null, null) ); assertThat(e.getMessage(), containsString("The [cosine] similarity does not support vectors with zero magnitude.")); } @@ -304,7 +304,7 @@ public void testCreateKnnQueryMaxDims() { for (int i = 0; i < 4096; i++) { queryVector[i] = randomFloat(); } - Query query = fieldWith4096dims.createKnnQuery(queryVector, 10, 10, null, null, null); + Query query = fieldWith4096dims.createKnnQuery(VectorData.fromFloats(queryVector), 10, 10, null, null, null); assertThat(query, instanceOf(KnnFloatVectorQuery.class)); } @@ -342,7 +342,7 @@ public void testByteCreateKnnQuery() { ); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> unindexedField.createKnnQuery(new float[] { 0.3f, 0.1f, 1.0f }, 10, 10, null, null, null) + () -> unindexedField.createKnnQuery(VectorData.fromFloats(new float[] { 0.3f, 0.1f, 1.0f }), 10, 10, null, null, null) ); assertThat(e.getMessage(), containsString("to perform knn search on field [f], its mapping must have [index] set to [true]")); @@ -358,7 +358,7 @@ public void testByteCreateKnnQuery() { ); e = expectThrows( IllegalArgumentException.class, - () -> cosineField.createKnnQuery(new float[] { 0.0f, 0.0f, 0.0f }, 10, 10, null, null, null) + () -> cosineField.createKnnQuery(VectorData.fromFloats(new float[] { 0.0f, 0.0f, 0.0f }), 10, 10, null, null, null) ); assertThat(e.getMessage(), containsString("The [cosine] similarity does not support vectors with zero magnitude.")); diff --git a/server/src/test/java/org/elasticsearch/index/seqno/LocalCheckpointTrackerTests.java b/server/src/test/java/org/elasticsearch/index/seqno/LocalCheckpointTrackerTests.java index 3930a37df498c..0765872f892c2 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/LocalCheckpointTrackerTests.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/LocalCheckpointTrackerTests.java @@ -196,46 +196,29 @@ protected void doRun() throws Exception { } public void testConcurrentReplica() throws InterruptedException { - Thread[] threads = new Thread[randomIntBetween(2, 5)]; + final int threads = randomIntBetween(2, 5); final int opsPerThread = randomIntBetween(10, 20); - final int maxOps = opsPerThread * threads.length; + final int maxOps = opsPerThread * threads; final long unFinishedSeq = randomIntBetween(0, maxOps - 2); // make sure we always index the last seqNo to simplify maxSeq checks Set seqNos = IntStream.range(0, maxOps).boxed().collect(Collectors.toSet()); - final Integer[][] seqNoPerThread = new Integer[threads.length][]; - for (int t = 0; t < threads.length - 1; t++) { + final Integer[][] seqNoPerThread = new Integer[threads][]; + for (int t = 0; t < threads - 1; t++) { int size = Math.min(seqNos.size(), randomIntBetween(opsPerThread - 4, opsPerThread + 4)); seqNoPerThread[t] = randomSubsetOf(size, seqNos).toArray(new Integer[size]); seqNos.removeAll(Arrays.asList(seqNoPerThread[t])); } - seqNoPerThread[threads.length - 1] = seqNos.toArray(new Integer[seqNos.size()]); - logger.info("--> will run [{}] threads, maxOps [{}], unfinished seq no [{}]", threads.length, maxOps, unFinishedSeq); - final CyclicBarrier barrier = new CyclicBarrier(threads.length); - for (int t = 0; t < threads.length; t++) { - final int threadId = t; - threads[t] = new Thread(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - throw new ElasticsearchException("failure in background thread", e); + seqNoPerThread[threads - 1] = seqNos.toArray(new Integer[seqNos.size()]); + logger.info("--> will run [{}] threads, maxOps [{}], unfinished seq no [{}]", threads, maxOps, unFinishedSeq); + startInParallel(threads, threadId -> { + Integer[] ops = seqNoPerThread[threadId]; + for (int seqNo : ops) { + if (seqNo != unFinishedSeq) { + tracker.markSeqNoAsProcessed(seqNo); + logger.info("[t{}] completed [{}]", threadId, seqNo); } - - @Override - protected void doRun() throws Exception { - barrier.await(); - Integer[] ops = seqNoPerThread[threadId]; - for (int seqNo : ops) { - if (seqNo != unFinishedSeq) { - tracker.markSeqNoAsProcessed(seqNo); - logger.info("[t{}] completed [{}]", threadId, seqNo); - } - } - } - }, "testConcurrentReplica_" + threadId); - threads[t].start(); - } - for (Thread thread : threads) { - thread.join(); - } + } + }); assertThat(tracker.getMaxSeqNo(), equalTo(maxOps - 1L)); assertThat(tracker.getProcessedCheckpoint(), equalTo(unFinishedSeq - 1L)); assertThat(tracker.hasProcessed(unFinishedSeq), equalTo(false)); diff --git a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java index de8dbc3a1515e..0a167b677934e 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java @@ -29,7 +29,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -687,7 +686,7 @@ public void testUnnecessaryPersistenceOfRetentionLeases() throws IOException { * * @throws IOException if an I/O exception occurs loading the retention lease state file */ - public void testPersistRetentionLeasesUnderConcurrency() throws IOException { + public void testPersistRetentionLeasesUnderConcurrency() throws IOException, InterruptedException { final AllocationId allocationId = AllocationId.newInitializing(); long primaryTerm = randomLongBetween(1, Long.MAX_VALUE); final ReplicationTracker replicationTracker = new ReplicationTracker( @@ -719,35 +718,16 @@ public void testPersistRetentionLeasesUnderConcurrency() throws IOException { final Path path = createTempDir(); final int numberOfThreads = randomIntBetween(1, 2 * Runtime.getRuntime().availableProcessors()); - final CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads); - final Thread[] threads = new Thread[numberOfThreads]; - for (int i = 0; i < numberOfThreads; i++) { + startInParallel(numberOfThreads, i -> { final String id = Integer.toString(length + i); - threads[i] = new Thread(() -> { - try { - safeAwait(barrier); - final long retainingSequenceNumber = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); - replicationTracker.addRetentionLease(id, retainingSequenceNumber, "test-" + id, ActionListener.noop()); - replicationTracker.persistRetentionLeases(path); - safeAwait(barrier); - } catch (final WriteStateException e) { - throw new AssertionError(e); - } - }); - threads[i].start(); - } - - try { - // synchronize the threads invoking ReplicationTracker#persistRetentionLeases(Path path) - safeAwait(barrier); - // wait for all the threads to finish - safeAwait(barrier); - for (int i = 0; i < numberOfThreads; i++) { - threads[i].join(); + final long retainingSequenceNumber = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); + replicationTracker.addRetentionLease(id, retainingSequenceNumber, "test-" + id, ActionListener.noop()); + try { + replicationTracker.persistRetentionLeases(path); + } catch (WriteStateException e) { + throw new AssertionError(e); } - } catch (final InterruptedException e) { - throw new AssertionError(e); - } + }); assertThat(replicationTracker.loadRetentionLeases(path), equalTo(replicationTracker.getRetentionLeases())); } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java index 668a47645a17f..352013b6b6890 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.EsExecutors; @@ -524,24 +525,26 @@ public void testActiveOperationsCount() throws ExecutionException, InterruptedEx assertThat(permits.getActiveOperationsCount(), equalTo(0)); } + private Releasable acquirePermitImmediately() { + final var listener = SubscribableListener.newForked(l -> permits.acquire(l, threadPool.generic(), false)); + assertTrue(listener.isDone()); + return safeAwait(listener); + } + public void testAsyncBlockOperationsOnRejection() { final PlainActionFuture threadBlock = new PlainActionFuture<>(); - try (Releasable firstPermit = PlainActionFuture.get(f -> permits.acquire(f, threadPool.generic(), false), 0, TimeUnit.SECONDS)) { + try (Releasable firstPermit = acquirePermitImmediately()) { assertNotNull(firstPermit); final var rejectingExecutor = threadPool.executor(REJECTING_EXECUTOR); rejectingExecutor.execute(threadBlock::actionGet); - expectThrows( - EsRejectedExecutionException.class, - () -> PlainActionFuture.get( - f -> permits.blockOperations(f, 1, TimeUnit.HOURS, rejectingExecutor) - ) + assertThat( + safeAwaitFailure(Releasable.class, l -> permits.blockOperations(l, 1, TimeUnit.HOURS, rejectingExecutor)), + instanceOf(EsRejectedExecutionException.class) ); // ensure that the exception means no block was put in place - try ( - Releasable secondPermit = PlainActionFuture.get(f -> permits.acquire(f, threadPool.generic(), false), 0, TimeUnit.SECONDS) - ) { + try (Releasable secondPermit = acquirePermitImmediately()) { assertNotNull(secondPermit); } } finally { @@ -549,30 +552,26 @@ public void testAsyncBlockOperationsOnRejection() { } // ensure that another block can still be acquired - try (Releasable block = PlainActionFuture.get(f -> permits.blockOperations(f, 1, TimeUnit.HOURS, threadPool.generic()))) { + try (Releasable block = safeAwait(l -> permits.blockOperations(l, 1, TimeUnit.HOURS, threadPool.generic()))) { assertNotNull(block); } } public void testAsyncBlockOperationsOnTimeout() { final PlainActionFuture threadBlock = new PlainActionFuture<>(); - try (Releasable firstPermit = PlainActionFuture.get(f -> permits.acquire(f, threadPool.generic(), false), 0, TimeUnit.SECONDS)) { + try (Releasable firstPermit = acquirePermitImmediately()) { assertNotNull(firstPermit); assertEquals( "timeout while blocking operations after [0s]", - expectThrows( + asInstanceOf( ElasticsearchTimeoutException.class, - () -> PlainActionFuture.get( - f -> permits.blockOperations(f, 0, TimeUnit.SECONDS, threadPool.generic()) - ) + safeAwaitFailure(Releasable.class, f -> permits.blockOperations(f, 0, TimeUnit.SECONDS, threadPool.generic())) ).getMessage() ); // ensure that the exception means no block was put in place - try ( - Releasable secondPermit = PlainActionFuture.get(f -> permits.acquire(f, threadPool.generic(), false), 0, TimeUnit.SECONDS) - ) { + try (Releasable secondPermit = acquirePermitImmediately()) { assertNotNull(secondPermit); } @@ -581,7 +580,7 @@ public void testAsyncBlockOperationsOnTimeout() { } // ensure that another block can still be acquired - try (Releasable block = PlainActionFuture.get(f -> permits.blockOperations(f, 1, TimeUnit.HOURS, threadPool.generic()))) { + try (Releasable block = safeAwait(l -> permits.blockOperations(l, 1, TimeUnit.HOURS, threadPool.generic()))) { assertNotNull(block); } } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 29f39134d2bcf..142c03cdfa053 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -2098,9 +2098,17 @@ public void testShardCanNotBeMarkedAsRelocatedIfRelocationCancelled() throws IOE final ShardRouting relocationRouting = ShardRoutingHelper.relocate(originalRouting, "other_node"); IndexShardTestCase.updateRoutingEntry(shard, relocationRouting); IndexShardTestCase.updateRoutingEntry(shard, originalRouting); - expectThrows( + asInstanceOf( IllegalIndexShardStateException.class, - () -> blockingCallRelocated(shard, relocationRouting, (primaryContext, listener) -> fail("should not be called")) + safeAwaitFailure( + Void.class, + listener -> shard.relocated( + relocationRouting.relocatingNodeId(), + relocationRouting.getTargetRelocatingShard().allocationId().getId(), + (primaryContext, l) -> fail("should not be called"), + listener + ) + ) ); closeShards(shard); } @@ -2121,7 +2129,14 @@ public void onFailure(Exception e) { @Override protected void doRun() throws Exception { cyclicBarrier.await(); - blockingCallRelocated(shard, relocationRouting, (primaryContext, listener) -> listener.onResponse(null)); + final var relocatedCompleteLatch = new CountDownLatch(1); + shard.relocated( + relocationRouting.relocatingNodeId(), + relocationRouting.getTargetRelocatingShard().allocationId().getId(), + (primaryContext, listener) -> listener.onResponse(null), + ActionListener.releaseAfter(ActionListener.wrap(r -> {}, relocationException::set), relocatedCompleteLatch::countDown) + ); + safeAwait(relocatedCompleteLatch); } }); relocationThread.start(); @@ -2177,9 +2192,17 @@ public void testRelocateMismatchedTarget() throws Exception { final AtomicBoolean relocated = new AtomicBoolean(); - final IllegalIndexShardStateException wrongNodeException = expectThrows( + final IllegalIndexShardStateException wrongNodeException = asInstanceOf( IllegalIndexShardStateException.class, - () -> blockingCallRelocated(shard, wrongTargetNodeShardRouting, (ctx, listener) -> relocated.set(true)) + safeAwaitFailure( + Void.class, + listener -> shard.relocated( + wrongTargetNodeShardRouting.relocatingNodeId(), + wrongTargetNodeShardRouting.getTargetRelocatingShard().allocationId().getId(), + (ctx, l) -> relocated.set(true), + listener + ) + ) ); assertThat( wrongNodeException.getMessage(), @@ -2187,9 +2210,17 @@ public void testRelocateMismatchedTarget() throws Exception { ); assertFalse(relocated.get()); - final IllegalStateException wrongTargetIdException = expectThrows( + final IllegalStateException wrongTargetIdException = asInstanceOf( IllegalStateException.class, - () -> blockingCallRelocated(shard, wrongTargetAllocationIdShardRouting, (ctx, listener) -> relocated.set(true)) + safeAwaitFailure( + Void.class, + listener -> shard.relocated( + wrongTargetAllocationIdShardRouting.relocatingNodeId(), + wrongTargetAllocationIdShardRouting.getTargetRelocatingShard().allocationId().getId(), + (ctx, l) -> relocated.set(true), + listener + ) + ) ); assertThat( wrongTargetIdException.getMessage(), @@ -2981,7 +3012,7 @@ public void testShardActiveDuringInternalRecovery() throws IOException { // Shard is still inactive since we haven't started recovering yet assertFalse(shard.isActive()); shard.recoveryState().getIndex().setFileDetailsComplete(); - PlainActionFuture.get(shard::openEngineAndRecoverFromTranslog, 30, TimeUnit.SECONDS); + safeAwait(shard::openEngineAndRecoverFromTranslog); // Shard should now be active since we did recover: assertTrue(shard.isActive()); closeShards(shard); @@ -3849,7 +3880,6 @@ public void testIsSearchIdle() throws Exception { closeShards(primary); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/101008") @TestIssueLogging( issueUrl = "https://github.com/elastic/elasticsearch/issues/101008", value = "org.elasticsearch.index.shard.IndexShard:TRACE" @@ -3914,8 +3944,8 @@ public void testScheduledRefresh() throws Exception { }); latch.await(); - // Index a document while shard is search active and ensure scheduleRefresh(...) makes documen visible: - logger.info("--> index doc while shard search active"); + // Index a document while shard is search is idle and ensure scheduleRefresh(...) returns false: + logger.info("--> index doc while shard search is idle"); indexDoc(primary, "_doc", "2", "{\"foo\" : \"bar\"}"); logger.info("--> scheduledRefresh(future4)"); PlainActionFuture future4 = new PlainActionFuture<>(); @@ -3925,11 +3955,14 @@ public void testScheduledRefresh() throws Exception { logger.info("--> ensure search idle"); assertTrue(primary.isSearchIdle()); assertTrue(primary.searchIdleTime() >= TimeValue.ZERO.millis()); + long periodicFlushesBefore = primary.flushStats().getPeriodic(); primary.flushOnIdle(0); + assertBusy(() -> assertThat(primary.flushStats().getPeriodic(), greaterThan(periodicFlushesBefore))); + + long externalRefreshesBefore = primary.refreshStats().getExternalTotal(); logger.info("--> scheduledRefresh(future5)"); - PlainActionFuture future5 = new PlainActionFuture<>(); - primary.scheduledRefresh(future5); - assertTrue(future5.actionGet()); // make sure we refresh once the shard is inactive + primary.scheduledRefresh(ActionListener.noop()); + assertBusy(() -> assertThat(primary.refreshStats().getExternalTotal(), equalTo(externalRefreshesBefore + 1))); try (Engine.Searcher searcher = primary.acquireSearcher("test")) { assertEquals(3, searcher.getIndexReader().numDocs()); } @@ -5019,8 +5052,13 @@ private static void blockingCallRelocated( ShardRouting routing, BiConsumer> consumer ) { - PlainActionFuture.get( - f -> indexShard.relocated(routing.relocatingNodeId(), routing.getTargetRelocatingShard().allocationId().getId(), consumer, f) + safeAwait( + (ActionListener listener) -> indexShard.relocated( + routing.relocatingNodeId(), + routing.getTargetRelocatingShard().allocationId().getId(), + consumer, + listener + ) ); } } diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java index c173a22dcdf57..628ff4b99b133 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; +import org.elasticsearch.index.mapper.IndexModeFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; @@ -85,6 +86,7 @@ public Map getMetadataMappers() { TimeSeriesIdFieldMapper.NAME, TimeSeriesRoutingHashFieldMapper.NAME, IndexFieldMapper.NAME, + IndexModeFieldMapper.NAME, SourceFieldMapper.NAME, IgnoredSourceFieldMapper.NAME, NestedPathFieldMapper.NAME, diff --git a/server/src/test/java/org/elasticsearch/indices/ShardLimitValidatorTests.java b/server/src/test/java/org/elasticsearch/indices/ShardLimitValidatorTests.java index e1b3bb4fe9c49..0eea536ddbff1 100644 --- a/server/src/test/java/org/elasticsearch/indices/ShardLimitValidatorTests.java +++ b/server/src/test/java/org/elasticsearch/indices/ShardLimitValidatorTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.shards.ShardCounts; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; @@ -79,7 +80,8 @@ private void testOverShardLimit(CheckShardLimitMethod targetMethod, String group + maxShards + "] maximum " + group - + " shards open", + + " shards open; for more information, see " + + ReferenceDocs.MAX_SHARDS_PER_NODE, ShardLimitValidator.errorMessageFrom(shardLimitsResult) ); assertEquals(shardLimitsResult.maxShardsInCluster(), maxShards); @@ -151,7 +153,9 @@ public void testValidateShardLimitOpenIndices() { + maxShards + "] maximum " + group - + " shards open;", + + " shards open; for more information, see " + + ReferenceDocs.MAX_SHARDS_PER_NODE + + ";", exception.getMessage() ); } @@ -179,7 +183,9 @@ public void testValidateShardLimitUpdateReplicas() { + shardsPerNode * nodesInCluster + "] maximum " + group - + " shards open;", + + " shards open; for more information, see " + + ReferenceDocs.MAX_SHARDS_PER_NODE + + ";", exception.getMessage() ); } diff --git a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java index ff2f55c791dd3..c228cc067b20b 100644 --- a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java @@ -56,7 +56,6 @@ public class HierarchyCircuitBreakerServiceTests extends ESTestCase { public void testThreadedUpdatesToChildBreaker() throws Exception { final int NUM_THREADS = scaledRandomIntBetween(3, 15); final int BYTES_PER_THREAD = scaledRandomIntBetween(500, 4500); - final Thread[] threads = new Thread[NUM_THREADS]; final AtomicBoolean tripped = new AtomicBoolean(false); final AtomicReference lastException = new AtomicReference<>(null); @@ -87,31 +86,21 @@ public void checkParentLimit(long newBytesReserved, String label) throws Circuit CircuitBreaker.REQUEST ); breakerRef.set(breaker); - - for (int i = 0; i < NUM_THREADS; i++) { - threads[i] = new Thread(() -> { - for (int j = 0; j < BYTES_PER_THREAD; j++) { - try { - breaker.addEstimateBytesAndMaybeBreak(1L, "test"); - } catch (CircuitBreakingException e) { - if (tripped.get()) { - assertThat("tripped too many times", true, equalTo(false)); - } else { - assertThat(tripped.compareAndSet(false, true), equalTo(true)); - } - } catch (Exception e) { - lastException.set(e); + runInParallel(NUM_THREADS, i -> { + for (int j = 0; j < BYTES_PER_THREAD; j++) { + try { + breaker.addEstimateBytesAndMaybeBreak(1L, "test"); + } catch (CircuitBreakingException e) { + if (tripped.get()) { + assertThat("tripped too many times", true, equalTo(false)); + } else { + assertThat(tripped.compareAndSet(false, true), equalTo(true)); } + } catch (Exception e) { + lastException.set(e); } - }); - - threads[i].start(); - } - - for (Thread t : threads) { - t.join(); - } - + } + }); assertThat("no other exceptions were thrown", lastException.get(), equalTo(null)); assertThat("breaker was tripped", tripped.get(), equalTo(true)); assertThat("breaker was tripped at least once", breaker.getTrippedCount(), greaterThanOrEqualTo(1L)); @@ -122,7 +111,6 @@ public void testThreadedUpdatesToChildBreakerWithParentLimit() throws Exception final int BYTES_PER_THREAD = scaledRandomIntBetween(500, 4500); final int parentLimit = (BYTES_PER_THREAD * NUM_THREADS) - 2; final int childLimit = parentLimit + 10; - final Thread[] threads = new Thread[NUM_THREADS]; final AtomicInteger tripped = new AtomicInteger(0); final AtomicReference lastException = new AtomicReference<>(null); @@ -165,21 +153,6 @@ public void checkParentLimit(long newBytesReserved, String label) throws Circuit CircuitBreaker.REQUEST ); breakerRef.set(breaker); - - for (int i = 0; i < NUM_THREADS; i++) { - threads[i] = new Thread(() -> { - for (int j = 0; j < BYTES_PER_THREAD; j++) { - try { - breaker.addEstimateBytesAndMaybeBreak(1L, "test"); - } catch (CircuitBreakingException e) { - tripped.incrementAndGet(); - } catch (Exception e) { - lastException.set(e); - } - } - }); - } - logger.info( "--> NUM_THREADS: [{}], BYTES_PER_THREAD: [{}], TOTAL_BYTES: [{}], PARENT_LIMIT: [{}], CHILD_LIMIT: [{}]", NUM_THREADS, @@ -190,13 +163,17 @@ public void checkParentLimit(long newBytesReserved, String label) throws Circuit ); logger.info("--> starting threads..."); - for (Thread t : threads) { - t.start(); - } - - for (Thread t : threads) { - t.join(); - } + runInParallel(NUM_THREADS, i -> { + for (int j = 0; j < BYTES_PER_THREAD; j++) { + try { + breaker.addEstimateBytesAndMaybeBreak(1L, "test"); + } catch (CircuitBreakingException e) { + tripped.incrementAndGet(); + } catch (Exception e) { + lastException.set(e); + } + } + }); logger.info("--> child breaker: used: {}, limit: {}", breaker.getUsed(), breaker.getLimit()); logger.info("--> parent tripped: {}, total trip count: {} (expecting 1-2 for each)", parentTripped.get(), tripped.get()); @@ -401,29 +378,15 @@ void overLimitTriggered(boolean leader) { }); logger.trace("black hole [{}]", data.hashCode()); - int threadCount = randomIntBetween(1, 10); - CyclicBarrier barrier = new CyclicBarrier(threadCount + 1); - List threads = new ArrayList<>(threadCount); - for (int i = 0; i < threadCount; ++i) { - threads.add(new Thread(() -> { - try { - safeAwait(barrier); - service.checkParentLimit(0, "test-thread"); - } catch (CircuitBreakingException e) { - // very rare - logger.info("Thread got semi-unexpected circuit breaking exception", e); - } - })); - } - - threads.forEach(Thread::start); - barrier.await(20, TimeUnit.SECONDS); - - for (Thread thread : threads) { - thread.join(10000); - } - threads.forEach(thread -> assertFalse(thread.isAlive())); + startInParallel(threadCount, i -> { + try { + service.checkParentLimit(0, "test-thread"); + } catch (CircuitBreakingException e) { + // very rare + logger.info("Thread got semi-unexpected circuit breaking exception", e); + } + }); assertThat(leaderTriggerCount.get(), equalTo(2)); } diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index 86c111d1c7145..23efd21e32e69 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -103,7 +103,6 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -807,20 +806,21 @@ public void testCancellationsDoesNotLeakPrimaryPermits() throws Exception { Thread cancelingThread = new Thread(() -> cancellableThreads.cancel("test")); cancelingThread.start(); - try { - PlainActionFuture.get( - future -> RecoverySourceHandler.runUnderPrimaryPermit( - listener -> listener.onResponse(null), - shard, - cancellableThreads, - future - ), - 10, - TimeUnit.SECONDS - ); - } catch (CancellableThreads.ExecutionCancelledException e) { - // expected. - } + safeAwait( + runListener -> RecoverySourceHandler.runUnderPrimaryPermit( + permitListener -> permitListener.onResponse(null), + shard, + cancellableThreads, + runListener.delegateResponse((l, e) -> { + if (e instanceof CancellableThreads.ExecutionCancelledException) { + // expected. + l.onResponse(null); + } else { + l.onFailure(e); + } + }) + ) + ); cancelingThread.join(); // we have to use assert busy as we may be interrupted while acquiring the permit, if so we want to check // that the permit is released. diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index b2b19f14cfd4b..5621ed468f557 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -1175,7 +1175,7 @@ public void testExecuteBulkRequestCallsDocumentSizeObserver() { AtomicInteger parsedValueWasUsed = new AtomicInteger(0); DocumentParsingProvider documentParsingProvider = new DocumentParsingProvider() { @Override - public DocumentSizeObserver newDocumentSizeObserver() { + public DocumentSizeObserver newDocumentSizeObserver(DocWriteRequest request) { return new DocumentSizeObserver() { @Override public XContentParser wrapParser(XContentParser xContentParser) { @@ -1188,6 +1188,7 @@ public long normalisedBytesParsed() { parsedValueWasUsed.incrementAndGet(); return 0; } + }; } }; @@ -1825,9 +1826,9 @@ public void testBulkRequestExecution() throws Exception { for (int i = 0; i < numRequest; i++) { IndexRequest indexRequest = new IndexRequest("_index").id("_id").setPipeline(pipelineId).setFinalPipeline("_none"); indexRequest.source(xContentType, "field1", "value1"); - boolean shouldListExecutedPiplines = randomBoolean(); - executedPipelinesExpected.add(shouldListExecutedPiplines); - indexRequest.setListExecutedPipelines(shouldListExecutedPiplines); + boolean shouldListExecutedPipelines = randomBoolean(); + executedPipelinesExpected.add(shouldListExecutedPipelines); + indexRequest.setListExecutedPipelines(shouldListExecutedPipelines); bulkRequest.add(indexRequest); } diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java index ce732a3b95a34..45cd944a4a926 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java @@ -10,7 +10,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.util.TestUtil; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.routing.RecoverySource; @@ -171,8 +170,8 @@ public void testSnapshotWithConflictingName() throws Exception { new SnapshotId(snapshot.getSnapshotId().getName(), "_uuid2") ); final ShardGenerations shardGenerations = ShardGenerations.builder().put(indexId, 0, shardGen).build(); - PlainActionFuture.get( - f -> repository.finalizeSnapshot( + final RepositoryData ignoredRepositoryData = safeAwait( + listener -> repository.finalizeSnapshot( new FinalizeSnapshotContext( shardGenerations, RepositoryData.EMPTY_REPO_GEN, @@ -192,7 +191,7 @@ public void testSnapshotWithConflictingName() throws Exception { Collections.emptyMap() ), IndexVersion.current(), - f, + listener, info -> {} ) ) diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java index ac23f646e5c52..29c858d49a0b6 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -87,6 +87,7 @@ import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.nullValue; @@ -233,7 +234,7 @@ public void testCorruptIndexLatestFile() throws Exception { } } - public void testRepositoryDataConcurrentModificationNotAllowed() throws Exception { + public void testRepositoryDataConcurrentModificationNotAllowed() { final BlobStoreRepository repository = setupRepo(); // write to index generational file @@ -244,7 +245,20 @@ public void testRepositoryDataConcurrentModificationNotAllowed() throws Exceptio // write repo data again to index generational file, errors because we already wrote to the // N+1 generation from which this repository data instance was created final RepositoryData fresherRepositoryData = repositoryData.withGenId(startingGeneration + 1); - expectThrows(RepositoryException.class, () -> writeIndexGen(repository, fresherRepositoryData, repositoryData.getGenId())); + + assertThat( + safeAwaitFailure( + RepositoryData.class, + listener -> repository.writeIndexGen( + fresherRepositoryData, + repositoryData.getGenId(), + IndexVersion.current(), + Function.identity(), + listener + ) + ), + instanceOf(RepositoryException.class) + ); } public void testBadChunksize() { @@ -330,9 +344,15 @@ public void testRepositoryDataDetails() throws Exception { snapshotDetailsAsserter.accept(AbstractSnapshotIntegTestCase.getRepositoryData(repository).getSnapshotDetails(snapshotId)); } - private static void writeIndexGen(BlobStoreRepository repository, RepositoryData repositoryData, long generation) throws Exception { - PlainActionFuture.get( - f -> repository.writeIndexGen(repositoryData, generation, IndexVersion.current(), Function.identity(), f) + private static void writeIndexGen(BlobStoreRepository repository, RepositoryData repositoryData, long generation) { + safeAwait( + (ActionListener listener) -> repository.writeIndexGen( + repositoryData, + generation, + IndexVersion.current(), + Function.identity(), + listener + ) ); } diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java index db8af818f1c52..9167a97c4b5c1 100644 --- a/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Releasable; -import org.elasticsearch.reservedstate.NonStateTransformResult; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.reservedstate.TransformState; import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction; @@ -38,9 +37,7 @@ import org.mockito.ArgumentMatchers; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -48,19 +45,17 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; -import java.util.stream.Collectors; -import static org.elasticsearch.reservedstate.service.ReservedStateUpdateTask.checkMetadataVersion; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.sameInstance; import static org.hamcrest.Matchers.startsWith; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; @@ -80,6 +75,65 @@ private static MasterServiceTaskQueue mo return (MasterServiceTaskQueue) mock(MasterServiceTaskQueue.class); } + private static class TestTaskContext implements ClusterStateTaskExecutor.TaskContext { + private final T task; + + private TestTaskContext(T task) { + this.task = task; + } + + @Override + public T getTask() { + return task; + } + + @Override + public void success(Runnable onPublicationSuccess) { + onPublicationSuccess.run(); + } + + @Override + public void success(Consumer publishedStateConsumer) {} + + @Override + public void success(Runnable onPublicationSuccess, ClusterStateAckListener clusterStateAckListener) {} + + @Override + public void success(Consumer publishedStateConsumer, ClusterStateAckListener clusterStateAckListener) {} + + @Override + public void onFailure(Exception failure) {} + + @Override + public Releasable captureResponseHeaders() { + return null; + } + } + + private static class TestStateHandler implements ReservedClusterStateHandler> { + private final String name; + + private TestStateHandler(String name) { + this.name = name; + } + + @Override + public String name() { + return name; + } + + @Override + public TransformState transform(Object source, TransformState prevState) throws Exception { + ClusterState newState = new ClusterState.Builder(prevState.state()).build(); + return new TransformState(newState, prevState.keys()); + } + + @Override + public Map fromXContent(XContentParser parser) throws IOException { + return parser.map(); + } + } + public void testOperatorController() throws IOException { ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); ClusterService clusterService = mock(ClusterService.class); @@ -153,8 +207,7 @@ public void testInitEmptyTask() { // grab the update task when it gets given to us when(clusterService.createTaskQueue(ArgumentMatchers.contains("reserved state update"), any(), any())).thenAnswer(i -> { - @SuppressWarnings("unchecked") - MasterServiceTaskQueue queue = mock(MasterServiceTaskQueue.class); + MasterServiceTaskQueue queue = mockTaskQueue(); doNothing().when(queue).submitTask(any(), updateTask.capture(), any()); return queue; }); @@ -182,39 +235,17 @@ public void testUpdateStateTasks() throws Exception { AtomicBoolean successCalled = new AtomicBoolean(false); ReservedStateUpdateTask task = spy( - new ReservedStateUpdateTask("test", null, List.of(), Map.of(), Set.of(), errorState -> {}, ActionListener.noop()) + new ReservedStateUpdateTask("test", null, Map.of(), Set.of(), errorState -> {}, ActionListener.noop()) ); doReturn(state).when(task).execute(any()); - ClusterStateTaskExecutor.TaskContext taskContext = new ClusterStateTaskExecutor.TaskContext<>() { - @Override - public ReservedStateUpdateTask getTask() { - return task; - } - + ClusterStateTaskExecutor.TaskContext taskContext = new TestTaskContext<>(task) { @Override public void success(Runnable onPublicationSuccess) { - onPublicationSuccess.run(); + super.success(onPublicationSuccess); successCalled.set(true); } - - @Override - public void success(Consumer publishedStateConsumer) {} - - @Override - public void success(Runnable onPublicationSuccess, ClusterStateAckListener clusterStateAckListener) {} - - @Override - public void success(Consumer publishedStateConsumer, ClusterStateAckListener clusterStateAckListener) {} - - @Override - public void onFailure(Exception failure) {} - - @Override - public Releasable captureResponseHeaders() { - return null; - } }; ClusterState newState = taskExecutor.execute( @@ -233,8 +264,7 @@ public void testUpdateErrorState() { ClusterState state = ClusterState.builder(new ClusterName("test")).build(); ArgumentCaptor updateTask = ArgumentCaptor.captor(); - @SuppressWarnings("unchecked") - MasterServiceTaskQueue errorQueue = mock(MasterServiceTaskQueue.class); + MasterServiceTaskQueue errorQueue = mockTaskQueue(); doNothing().when(errorQueue).submitTask(any(), updateTask.capture(), any()); // grab the update task when it gets given to us @@ -282,40 +312,8 @@ public void testErrorStateTask() throws Exception { ) ); - ReservedStateErrorTaskExecutor.TaskContext taskContext = - new ReservedStateErrorTaskExecutor.TaskContext<>() { - @Override - public ReservedStateErrorTask getTask() { - return task; - } - - @Override - public void success(Runnable onPublicationSuccess) { - onPublicationSuccess.run(); - } - - @Override - public void success(Consumer publishedStateConsumer) {} - - @Override - public void success(Runnable onPublicationSuccess, ClusterStateAckListener clusterStateAckListener) {} - - @Override - public void success(Consumer publishedStateConsumer, ClusterStateAckListener clusterStateAckListener) {} - - @Override - public void onFailure(Exception failure) {} - - @Override - public Releasable captureResponseHeaders() { - return null; - } - }; - - ReservedStateErrorTaskExecutor executor = new ReservedStateErrorTaskExecutor(); - - ClusterState newState = executor.execute( - new ClusterStateTaskExecutor.BatchExecutionContext<>(state, List.of(taskContext), () -> null) + ClusterState newState = new ReservedStateErrorTaskExecutor().execute( + new ClusterStateTaskExecutor.BatchExecutionContext<>(state, List.of(new TestTaskContext<>(task)), () -> null) ); verify(task, times(1)).execute(any()); @@ -330,39 +328,12 @@ public Releasable captureResponseHeaders() { } public void testUpdateTaskDuplicateError() { - ReservedClusterStateHandler> newStateMaker = new ReservedClusterStateHandler<>() { - @Override - public String name() { - return "maker"; - } - - @Override - public TransformState transform(Object source, TransformState prevState) throws Exception { - ClusterState newState = new ClusterState.Builder(prevState.state()).build(); - return new TransformState(newState, prevState.keys()); - } - - @Override - public Map fromXContent(XContentParser parser) throws IOException { - return parser.map(); - } - }; - - ReservedClusterStateHandler> exceptionThrower = new ReservedClusterStateHandler<>() { - @Override - public String name() { - return "one"; - } - + ReservedClusterStateHandler> newStateMaker = new TestStateHandler("maker"); + ReservedClusterStateHandler> exceptionThrower = new TestStateHandler("one") { @Override public TransformState transform(Object source, TransformState prevState) throws Exception { throw new Exception("anything"); } - - @Override - public Map fromXContent(XContentParser parser) throws IOException { - return parser.map(); - } }; ReservedStateHandlerMetadata hmOne = new ReservedStateHandlerMetadata("one", Set.of("a", "b")); @@ -395,7 +366,6 @@ public Map fromXContent(XContentParser parser) throws IOExceptio ReservedStateUpdateTask task = new ReservedStateUpdateTask( "namespace_one", chunk, - List.of(), Map.of(exceptionThrower.name(), exceptionThrower, newStateMaker.name(), newStateMaker), orderedHandlers, errorState -> assertFalse(ReservedStateErrorTask.isNewError(operatorMetadata, errorState.version())), @@ -407,9 +377,8 @@ public Map fromXContent(XContentParser parser) throws IOExceptio new ReservedClusterStateService(clusterService, mock(RerouteService.class), List.of(newStateMaker, exceptionThrower)) ); - var trialRunResult = controller.trialRun("namespace_one", state, chunk, new LinkedHashSet<>(orderedHandlers)); - assertThat(trialRunResult.nonStateTransforms(), empty()); - assertThat(trialRunResult.errors(), contains(containsString("Error processing one state change:"))); + var trialRunErrors = controller.trialRun("namespace_one", state, chunk, new LinkedHashSet<>(orderedHandlers)); + assertThat(trialRunErrors, contains(containsString("Error processing one state change:"))); // We exit on duplicate errors before we update the cluster state error metadata assertThat( @@ -443,22 +412,40 @@ public Map fromXContent(XContentParser parser) throws IOExceptio public void testCheckMetadataVersion() { ReservedStateMetadata operatorMetadata = ReservedStateMetadata.builder("test").version(123L).build(); - assertTrue(checkMetadataVersion("operator", operatorMetadata, new ReservedStateVersion(124L, Version.CURRENT))); + ClusterState state = ClusterState.builder(new ClusterName("test")).metadata(Metadata.builder().put(operatorMetadata)).build(); + ReservedStateUpdateTask task = new ReservedStateUpdateTask( + "test", + new ReservedStateChunk(Map.of(), new ReservedStateVersion(124L, Version.CURRENT)), + Map.of(), + List.of(), + e -> {}, + ActionListener.noop() + ); + assertThat("Cluster state should be modified", task.execute(state), not(sameInstance(state))); - assertFalse(checkMetadataVersion("operator", operatorMetadata, new ReservedStateVersion(123L, Version.CURRENT))); + task = new ReservedStateUpdateTask( + "test", + new ReservedStateChunk(Map.of(), new ReservedStateVersion(123L, Version.CURRENT)), + Map.of(), + List.of(), + e -> {}, + ActionListener.noop() + ); + assertThat("Cluster state should not be modified", task.execute(state), sameInstance(state)); - assertFalse( - checkMetadataVersion("operator", operatorMetadata, new ReservedStateVersion(124L, Version.fromId(Version.CURRENT.id + 1))) + task = new ReservedStateUpdateTask( + "test", + new ReservedStateChunk(Map.of(), new ReservedStateVersion(124L, Version.fromId(Version.CURRENT.id + 1))), + Map.of(), + List.of(), + e -> {}, + ActionListener.noop() ); + assertThat("Cluster state should not be modified", task.execute(state), sameInstance(state)); } - private ReservedClusterStateHandler> makeHandlerHelper(final String name, final List deps) { - return new ReservedClusterStateHandler<>() { - @Override - public String name() { - return name; - } - + private ReservedClusterStateHandler> makeHandlerHelper(String name, List deps) { + return new TestStateHandler(name) { @Override public TransformState transform(Object source, TransformState prevState) throws Exception { return null; @@ -468,11 +455,6 @@ public TransformState transform(Object source, TransformState prevState) throws public Collection dependencies() { return deps; } - - @Override - public Map fromXContent(XContentParser parser) throws IOException { - return parser.map(); - } }; } @@ -527,7 +509,12 @@ public void testDuplicateHandlerNames() { () -> new ReservedClusterStateService( clusterService, mock(RerouteService.class), - List.of(new ReservedClusterSettingsAction(clusterSettings), new TestHandler()) + List.of(new ReservedClusterSettingsAction(clusterSettings), new TestStateHandler(ReservedClusterSettingsAction.NAME) { + @Override + public TransformState transform(Object source, TransformState prevState) throws Exception { + return prevState; + } + }) ) ).getMessage(), startsWith("Duplicate key cluster_settings") @@ -553,42 +540,11 @@ public void testCheckAndReportError() { } public void testTrialRunExtractsNonStateActions() { - ReservedClusterStateHandler> newStateMaker = new ReservedClusterStateHandler<>() { - @Override - public String name() { - return "maker"; - } - - @Override - public TransformState transform(Object source, TransformState prevState) throws Exception { - ClusterState newState = new ClusterState.Builder(prevState.state()).build(); - return new TransformState(newState, prevState.keys()); - } - - @Override - public Map fromXContent(XContentParser parser) throws IOException { - return parser.map(); - } - }; - - ReservedClusterStateHandler> exceptionThrower = new ReservedClusterStateHandler<>() { - @Override - public String name() { - return "non-state"; - } - + ReservedClusterStateHandler> newStateMaker = new TestStateHandler("maker"); + ReservedClusterStateHandler> exceptionThrower = new TestStateHandler("non-state") { @Override public TransformState transform(Object source, TransformState prevState) { - return new TransformState(prevState.state(), prevState.keys(), this::internalKeys); - } - - private void internalKeys(ActionListener listener) { - listener.onResponse(new NonStateTransformResult(name(), Set.of("key non-state"))); - } - - @Override - public Map fromXContent(XContentParser parser) throws IOException { - return parser.map(); + return new TransformState(prevState.state(), prevState.keys()); } }; @@ -616,122 +572,7 @@ public Map fromXContent(XContentParser parser) throws IOExceptio new ReservedClusterStateService(clusterService, mock(RerouteService.class), List.of(newStateMaker, exceptionThrower)) ); - var trialRunResult = controller.trialRun("namespace_one", state, chunk, new LinkedHashSet<>(orderedHandlers)); - - assertThat(trialRunResult.nonStateTransforms(), hasSize(1)); - assertThat(trialRunResult.errors(), empty()); - trialRunResult.nonStateTransforms().get(0).accept(new ActionListener<>() { - @Override - public void onResponse(NonStateTransformResult nonStateTransformResult) { - assertThat(nonStateTransformResult.updatedKeys(), containsInAnyOrder("key non-state")); - assertThat(nonStateTransformResult.handlerName(), is("non-state")); - } - - @Override - public void onFailure(Exception e) { - fail("Should not reach here"); - } - }); - } - - public void testExecuteNonStateTransformationSteps() { - int count = randomInt(10); - var handlers = new ArrayList>(); - var i = 0; - var builder = ReservedStateMetadata.builder("namespace_one").version(1L); - var chunkMap = new HashMap(); - - while (i < count) { - final var key = i++; - var handler = new ReservedClusterStateHandler<>() { - @Override - public String name() { - return "non-state:" + key; - } - - @Override - public TransformState transform(Object source, TransformState prevState) { - return new TransformState(prevState.state(), prevState.keys(), this::internalKeys); - } - - private void internalKeys(ActionListener listener) { - listener.onResponse(new NonStateTransformResult(name(), Set.of("key non-state:" + key))); - } - - @Override - public Map fromXContent(XContentParser parser) throws IOException { - return parser.map(); - } - }; - - builder.putHandler(new ReservedStateHandlerMetadata(handler.name(), Set.of("a", "b"))); - handlers.add(handler); - chunkMap.put(handler.name(), i); - } - - final ReservedStateMetadata operatorMetadata = ReservedStateMetadata.builder("namespace_one").version(1L).build(); - - Metadata metadata = Metadata.builder().put(operatorMetadata).build(); - ClusterState state = ClusterState.builder(new ClusterName("test")).metadata(metadata).build(); - - var chunk = new ReservedStateChunk(chunkMap, new ReservedStateVersion(2L, Version.CURRENT)); - - ClusterService clusterService = mock(ClusterService.class); - final var controller = spy(new ReservedClusterStateService(clusterService, mock(RerouteService.class), handlers)); - - var trialRunResult = controller.trialRun( - "namespace_one", - state, - chunk, - handlers.stream().map(ReservedClusterStateHandler::name).collect(Collectors.toCollection(LinkedHashSet::new)) - ); - - assertThat(trialRunResult.nonStateTransforms(), hasSize(count)); - ReservedClusterStateService.executeNonStateTransformationSteps(trialRunResult.nonStateTransforms(), new ActionListener<>() { - @Override - public void onResponse(Collection nonStateTransformResults) { - assertEquals(count, nonStateTransformResults.size()); - var expectedHandlers = new ArrayList(); - var expectedValues = new ArrayList(); - for (int i = 0; i < count; i++) { - expectedHandlers.add("non-state:" + i); - expectedValues.add("key non-state:" + i); - } - assertThat( - nonStateTransformResults.stream().map(NonStateTransformResult::handlerName).collect(Collectors.toSet()), - containsInAnyOrder(expectedHandlers.toArray()) - ); - assertThat( - nonStateTransformResults.stream() - .map(NonStateTransformResult::updatedKeys) - .flatMap(Set::stream) - .collect(Collectors.toSet()), - containsInAnyOrder(expectedValues.toArray()) - ); - } - - @Override - public void onFailure(Exception e) { - fail("Shouldn't reach here"); - } - }); - } - - static class TestHandler implements ReservedClusterStateHandler> { - - @Override - public String name() { - return ReservedClusterSettingsAction.NAME; - } - - @Override - public TransformState transform(Object source, TransformState prevState) { - return prevState; - } - - @Override - public Map fromXContent(XContentParser parser) throws IOException { - return parser.map(); - } + var trialRunErrors = controller.trialRun("namespace_one", state, chunk, new LinkedHashSet<>(orderedHandlers)); + assertThat(trialRunErrors, empty()); } } diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTaskTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTaskTests.java index d887d7edb19f2..72d2310a098cf 100644 --- a/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTaskTests.java +++ b/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedStateUpdateTaskTests.java @@ -22,7 +22,7 @@ public class ReservedStateUpdateTaskTests extends ESTestCase { public void testBlockedClusterState() { - var task = new ReservedStateUpdateTask("dummy", null, List.of(), Map.of(), List.of(), e -> {}, ActionListener.noop()); + var task = new ReservedStateUpdateTask("dummy", null, Map.of(), List.of(), e -> {}, ActionListener.noop()); ClusterState notRecoveredClusterState = ClusterState.builder(ClusterName.DEFAULT) .blocks(ClusterBlocks.builder().addGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) .build(); diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionTests.java index 55096f38fece2..86847a19f9ba9 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ClusterStatsLevel; import org.elasticsearch.action.NodeStatsLevel; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.test.ESTestCase; @@ -34,7 +35,7 @@ public void setUp() throws Exception { action = new RestNodesStatsAction(); } - public void testUnrecognizedMetric() throws IOException { + public void testUnrecognizedMetric() { final HashMap params = new HashMap<>(); final String metric = randomAlphaOfLength(64); params.put("metric", metric); @@ -46,7 +47,7 @@ public void testUnrecognizedMetric() throws IOException { assertThat(e, hasToString(containsString("request [/_nodes/stats] contains unrecognized metric: [" + metric + "]"))); } - public void testUnrecognizedMetricDidYouMean() throws IOException { + public void testUnrecognizedMetricDidYouMean() { final HashMap params = new HashMap<>(); params.put("metric", "os,transprot,unrecognized"); final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_nodes/stats").withParams(params).build(); @@ -64,9 +65,9 @@ public void testUnrecognizedMetricDidYouMean() throws IOException { ); } - public void testAllRequestWithOtherMetrics() throws IOException { + public void testAllRequestWithOtherMetrics() { final HashMap params = new HashMap<>(); - final String metric = randomSubsetOf(1, RestNodesStatsAction.METRICS.keySet()).get(0); + final String metric = randomFrom(Metric.ALL_NAMES); params.put("metric", "_all," + metric); final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_nodes/stats").withParams(params).build(); final IllegalArgumentException e = expectThrows( @@ -108,9 +109,9 @@ public void testUnrecognizedIndexMetricDidYouMean() { ); } - public void testIndexMetricsRequestWithoutIndicesMetric() throws IOException { + public void testIndexMetricsRequestWithoutIndicesMetric() { final HashMap params = new HashMap<>(); - final Set metrics = new HashSet<>(RestNodesStatsAction.METRICS.keySet()); + final Set metrics = new HashSet<>(Metric.ALL_NAMES); metrics.remove("indices"); params.put("metric", randomSubsetOf(1, metrics).get(0)); final String indexMetric = randomSubsetOf(1, RestNodesStatsAction.FLAGS.keySet()).get(0); @@ -128,7 +129,7 @@ public void testIndexMetricsRequestWithoutIndicesMetric() throws IOException { ); } - public void testIndexMetricsRequestOnAllRequest() throws IOException { + public void testIndexMetricsRequestOnAllRequest() { final HashMap params = new HashMap<>(); params.put("metric", "_all"); final String indexMetric = randomSubsetOf(1, RestNodesStatsAction.FLAGS.keySet()).get(0); diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java index 59ab7ec719cf4..1270c23227756 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; @@ -74,7 +75,8 @@ public void stubValidateQueryAction() { final TransportAction transportAction = new TransportAction<>( ValidateQueryAction.NAME, new ActionFilters(Collections.emptySet()), - taskManager + taskManager, + EsExecutors.DIRECT_EXECUTOR_SERVICE ) { @Override protected void doExecute(Task task, ActionRequest request, ActionListener listener) {} diff --git a/server/src/test/java/org/elasticsearch/rest/action/info/RestClusterInfoActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/info/RestClusterInfoActionTests.java index f0473ae344a79..2c9cc50031cab 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/info/RestClusterInfoActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/info/RestClusterInfoActionTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.rest.action.info; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.ClusterName; @@ -28,7 +29,7 @@ import java.util.stream.IntStream; import java.util.stream.Stream; -import static org.elasticsearch.rest.action.info.RestClusterInfoAction.AVAILABLE_TARGETS; +import static org.elasticsearch.rest.action.info.RestClusterInfoAction.AVAILABLE_TARGET_NAMES; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasToString; import static org.mockito.Mockito.mock; @@ -69,7 +70,7 @@ public void testAllTargetAlone() throws IOException { } public void testMultiTargetRequest() throws IOException { - var target = String.join(",", AVAILABLE_TARGETS); + var target = String.join(",", AVAILABLE_TARGET_NAMES); var request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_info/").withParams(Map.of("target", target)).build(); action.prepareRequest(request, mock(NodeClient.class)); @@ -79,7 +80,7 @@ public void testHttpResponseMapper() { var nodeStats = IntStream.range(1, randomIntBetween(2, 20)).mapToObj(this::randomNodeStatsWithOnlyHttpStats).toList(); var response = new NodesStatsResponse(new ClusterName("cluster-name"), nodeStats, List.of()); - var httpStats = (HttpStats) RestClusterInfoAction.RESPONSE_MAPPER.get("http").apply(response); + var httpStats = (HttpStats) RestClusterInfoAction.RESPONSE_MAPPER.get(Metric.HTTP).apply(response); final Map httpRouteStatsMap = new HashMap<>(); for (var ns : nodeStats) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index 788249fee1187..27f0b21d2767f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -329,6 +329,65 @@ public void testStringShardMinDocCount() throws IOException { } } + public void testStringShardZeroMinDocCount() throws IOException { + MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType("string", true, true, Collections.emptyMap()); + for (TermsAggregatorFactory.ExecutionMode executionMode : TermsAggregatorFactory.ExecutionMode.values()) { + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name").field("string") + .executionHint(executionMode.toString()) + .size(2) + .minDocCount(0) + .executionHint("map") + .excludeDeletedDocs(true) + .order(BucketOrder.key(true)); + + { + boolean delete = randomBoolean(); + // force single shard/segment + testCase(iw -> { + // force single shard/segment + iw.addDocuments(Arrays.asList(doc(fieldType, "a"), doc(fieldType, "b"), doc(fieldType, "c"), doc(fieldType, "d"))); + if (delete) { + iw.deleteDocuments(new TermQuery(new Term("string", "b"))); + } + }, (InternalTerms result) -> { + assertEquals(2, result.getBuckets().size()); + assertEquals("a", result.getBuckets().get(0).getKeyAsString()); + assertEquals(0L, result.getBuckets().get(0).getDocCount()); + if (delete) { + assertEquals("c", result.getBuckets().get(1).getKeyAsString()); + } else { + assertEquals("b", result.getBuckets().get(1).getKeyAsString()); + } + assertEquals(0L, result.getBuckets().get(1).getDocCount()); + }, new AggTestConfig(aggregationBuilder, fieldType).withQuery(new TermQuery(new Term("string", "e")))); + } + + { + boolean delete = randomBoolean(); + // force single shard/segment + testCase(iw -> { + // force single shard/segment + iw.addDocuments( + Arrays.asList(doc(fieldType, "a"), doc(fieldType, "c", "d"), doc(fieldType, "b", "d"), doc(fieldType, "b")) + ); + if (delete) { + iw.deleteDocuments(new TermQuery(new Term("string", "b"))); + } + }, (InternalTerms result) -> { + assertEquals(2, result.getBuckets().size()); + assertEquals("a", result.getBuckets().get(0).getKeyAsString()); + assertEquals(0L, result.getBuckets().get(0).getDocCount()); + if (delete) { + assertEquals("c", result.getBuckets().get(1).getKeyAsString()); + } else { + assertEquals("b", result.getBuckets().get(1).getKeyAsString()); + } + assertEquals(0L, result.getBuckets().get(1).getDocCount()); + }, new AggTestConfig(aggregationBuilder, fieldType).withQuery(new TermQuery(new Term("string", "e")))); + } + } + } + public void testManyTerms() throws Exception { MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType("string", randomBoolean(), true, Collections.emptyMap()); TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name").executionHint(randomHint()).field("string"); diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilderTests.java index d2a5859ae981f..a558081c2d16f 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilderTests.java @@ -56,7 +56,7 @@ protected KnnScoreDocQueryBuilder doCreateTestQueryBuilder() { return new KnnScoreDocQueryBuilder( scoreDocs.toArray(new ScoreDoc[0]), randomBoolean() ? "field" : null, - randomBoolean() ? randomVector(10) : null + randomBoolean() ? VectorData.fromFloats(randomVector(10)) : null ); } @@ -65,7 +65,7 @@ public void testValidOutput() { KnnScoreDocQueryBuilder query = new KnnScoreDocQueryBuilder( new ScoreDoc[] { new ScoreDoc(0, 4.25f), new ScoreDoc(5, 1.6f) }, "field", - new float[] { 1.0f, 2.0f } + VectorData.fromFloats(new float[] { 1.0f, 2.0f }) ); String expected = """ { @@ -155,7 +155,7 @@ public void testRewriteToMatchNone() throws IOException { KnnScoreDocQueryBuilder queryBuilder = new KnnScoreDocQueryBuilder( new ScoreDoc[0], randomBoolean() ? "field" : null, - randomBoolean() ? randomVector(10) : null + randomBoolean() ? VectorData.fromFloats(randomVector(10)) : null ); QueryRewriteContext context = randomBoolean() ? new InnerHitsRewriteContext(createSearchExecutionContext().getParserConfig(), System::currentTimeMillis) @@ -169,7 +169,7 @@ public void testRewriteForInnerHits() throws IOException { KnnScoreDocQueryBuilder queryBuilder = new KnnScoreDocQueryBuilder( new ScoreDoc[] { new ScoreDoc(0, 4.25f), new ScoreDoc(5, 1.6f) }, randomAlphaOfLength(10), - randomVector(10) + VectorData.fromFloats(randomVector(10)) ); queryBuilder.boost(randomFloat()); queryBuilder.queryName(randomAlphaOfLength(10)); @@ -218,7 +218,11 @@ public void testScoreDocQueryWeightCount() throws IOException { } ScoreDoc[] scoreDocs = scoreDocsList.toArray(new ScoreDoc[0]); - KnnScoreDocQueryBuilder queryBuilder = new KnnScoreDocQueryBuilder(scoreDocs, "field", randomVector(10)); + KnnScoreDocQueryBuilder queryBuilder = new KnnScoreDocQueryBuilder( + scoreDocs, + "field", + VectorData.fromFloats(randomVector(10)) + ); Query query = queryBuilder.doToQuery(context); final Weight w = query.createWeight(searcher, ScoreMode.TOP_SCORES, 1.0f); for (LeafReaderContext leafReaderContext : searcher.getLeafContexts()) { @@ -261,7 +265,11 @@ public void testScoreDocQuery() throws IOException { } ScoreDoc[] scoreDocs = scoreDocsList.toArray(new ScoreDoc[0]); - KnnScoreDocQueryBuilder queryBuilder = new KnnScoreDocQueryBuilder(scoreDocs, "field", randomVector(10)); + KnnScoreDocQueryBuilder queryBuilder = new KnnScoreDocQueryBuilder( + scoreDocs, + "field", + VectorData.fromFloats(randomVector(10)) + ); final Query query = queryBuilder.doToQuery(context); final Weight w = query.createWeight(searcher, ScoreMode.TOP_SCORES, 1.0f); diff --git a/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java index a7496e36955c3..7d502f3c41da0 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.snapshots; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ESAllocationTestCase; @@ -381,11 +382,9 @@ public IndexShardSnapshotStatus.Copy getShardSnapshotStatus(SnapshotId snapshotI } private void applyClusterState(final String reason, final Function applier) { - PlainActionFuture.get( - future -> clusterService.getClusterApplierService() - .onNewClusterState(reason, () -> applier.apply(clusterService.state()), future), - 10, - TimeUnit.SECONDS + safeAwait( + (ActionListener listener) -> clusterService.getClusterApplierService() + .onNewClusterState(reason, () -> applier.apply(clusterService.state()), listener) ); } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 8c9cd8cd54500..54051f8311967 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -1378,7 +1378,7 @@ public TransportRequestHandler interceptHandler( .anyMatch( e -> e.snapshot().getSnapshotId().getName().equals(cloneName) && e.isClone() - && e.shardsByRepoShardId().isEmpty() == false + && e.shardSnapshotStatusByRepoShardId().isEmpty() == false ) ).addListener(l); client.admin() diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index bcc7a23bbec53..71c041d21a825 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -315,7 +315,8 @@ public void testCompletedSnapshotStartsClone() throws Exception { assertThat(completedClone.state(), is(SnapshotsInProgress.State.SUCCESS)); final SnapshotsInProgress.Entry startedSnapshot = snapshotsInProgress.forRepo(repoName).get(1); assertThat(startedSnapshot.state(), is(SnapshotsInProgress.State.STARTED)); - final SnapshotsInProgress.ShardSnapshotStatus shardCloneStatus = startedSnapshot.shardsByRepoShardId().get(repositoryShardId); + final SnapshotsInProgress.ShardSnapshotStatus shardCloneStatus = startedSnapshot.shardSnapshotStatusByRepoShardId() + .get(repositoryShardId); assertThat(shardCloneStatus.state(), is(SnapshotsInProgress.ShardState.INIT)); assertThat(shardCloneStatus.nodeId(), is(updatedClusterState.nodes().getLocalNodeId())); assertIsNoop(updatedClusterState, completeShard); @@ -397,7 +398,7 @@ public void testCompletedCloneStartsNextClone() throws Exception { assertThat(completedClone.state(), is(SnapshotsInProgress.State.SUCCESS)); final SnapshotsInProgress.Entry startedSnapshot = snapshotsInProgress.forRepo(repoName).get(1); assertThat(startedSnapshot.state(), is(SnapshotsInProgress.State.STARTED)); - assertThat(startedSnapshot.shardsByRepoShardId().get(shardId1).state(), is(SnapshotsInProgress.ShardState.INIT)); + assertThat(startedSnapshot.shardSnapshotStatusByRepoShardId().get(shardId1).state(), is(SnapshotsInProgress.ShardState.INIT)); assertIsNoop(updatedClusterState, completeShardClone); } diff --git a/server/src/test/java/org/elasticsearch/tasks/TaskManagerTests.java b/server/src/test/java/org/elasticsearch/tasks/TaskManagerTests.java index 05150cd5dd362..d15eae47968e4 100644 --- a/server/src/test/java/org/elasticsearch/tasks/TaskManagerTests.java +++ b/server/src/test/java/org/elasticsearch/tasks/TaskManagerTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.network.CloseableChannel; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.core.TimeValue; @@ -335,7 +336,12 @@ public void testRegisterAndExecuteStartsAndStopsTracing() { final Task task = taskManager.registerAndExecute( "testType", - new TransportAction("actionName", new ActionFilters(Set.of()), taskManager) { + new TransportAction( + "actionName", + new ActionFilters(Set.of()), + taskManager, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ) { @Override protected void doExecute(Task task, ActionRequest request, ActionListener listener) { listener.onResponse(new ActionResponse() { diff --git a/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java b/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java index bc13d3fde7e31..e97fb3220923d 100644 --- a/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java +++ b/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java @@ -119,7 +119,7 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti validatedConnectionRef.set(c); l.onResponse(null); }; - PlainActionFuture.get(fut -> connectionManager.connectToNode(node, connectionProfile, validator, fut.map(x -> null))); + safeAwait(listener -> connectionManager.connectToNode(node, connectionProfile, validator, listener.map(x -> null))); assertFalse(connection.isClosed()); assertTrue(connectionManager.nodeConnected(node)); @@ -166,9 +166,9 @@ public void testDisconnectLogging() { final ConnectionManager.ConnectionValidator validator = (c, p, l) -> l.onResponse(null); final AtomicReference toClose = new AtomicReference<>(); - PlainActionFuture.get(f -> connectionManager.connectToNode(remoteClose, connectionProfile, validator, f.map(x -> null))); - PlainActionFuture.get(f -> connectionManager.connectToNode(shutdownClose, connectionProfile, validator, f.map(x -> null))); - PlainActionFuture.get(f -> connectionManager.connectToNode(localClose, connectionProfile, validator, f.map(toClose::getAndSet))); + safeAwait(l -> connectionManager.connectToNode(remoteClose, connectionProfile, validator, l.map(x -> null))); + safeAwait(l -> connectionManager.connectToNode(shutdownClose, connectionProfile, validator, l.map(x -> null))); + safeAwait(l -> connectionManager.connectToNode(localClose, connectionProfile, validator, l.map(toClose::getAndSet))); final Releasable localConnectionRef = toClose.getAndSet(null); assertThat(localConnectionRef, notNullValue()); @@ -431,15 +431,13 @@ public void onFailure(Exception e) { assertTrue(pendingConnectionPermits.tryAcquire(10, TimeUnit.SECONDS)); // ... and then send a connection attempt through the system to ensure that the lagging has started Releasables.closeExpectNoException( - PlainActionFuture.get( - fut -> connectionManager.connectToNode( + safeAwait( + (ActionListener listener) -> connectionManager.connectToNode( DiscoveryNodeUtils.create("", new TransportAddress(InetAddress.getLoopbackAddress(), 0)), connectionProfile, validator, - fut - ), - 30, - TimeUnit.SECONDS + listener + ) ) ); diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterAwareClientTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterAwareClientTests.java index 863bb60f0acc7..53ea6803b24a1 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterAwareClientTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterAwareClientTests.java @@ -194,17 +194,15 @@ public void testSearchShards() throws Exception { randomBoolean(), null ); - final SearchShardsResponse searchShardsResponse = PlainActionFuture.get( - future -> client.execute( + final SearchShardsResponse searchShardsResponse = safeAwait( + listener -> client.execute( TransportSearchShardsAction.REMOTE_TYPE, searchShardsRequest, ActionListener.runBefore( - future, + listener, () -> assertTrue(Thread.currentThread().getName().contains('[' + TEST_THREAD_POOL_NAME + ']')) ) - ), - 10, - TimeUnit.SECONDS + ) ); assertThat(searchShardsResponse.getNodes(), equalTo(knownNodes)); } diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java index bb18420276190..d2e885f8da4be 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.transport; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; @@ -40,6 +41,7 @@ import static org.elasticsearch.transport.AbstractSimpleTransportTestCase.IGNORE_DESERIALIZATION_ERRORS_SETTING; import static org.elasticsearch.transport.RemoteClusterConnectionTests.startTransport; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; public class RemoteClusterClientTests extends ESTestCase { @@ -99,25 +101,26 @@ public void testConnectAndExecuteRequest() throws Exception { threadPool.executor(TEST_THREAD_POOL_NAME), randomFrom(RemoteClusterService.DisconnectedStrategy.values()) ); - ClusterStateResponse clusterStateResponse = PlainActionFuture.get( - future -> client.execute( + ClusterStateResponse clusterStateResponse = safeAwait( + listener -> client.execute( ClusterStateAction.REMOTE_TYPE, new ClusterStateRequest(), ActionListener.runBefore( - future, + listener, () -> assertTrue(Thread.currentThread().getName().contains('[' + TEST_THREAD_POOL_NAME + ']')) ) - ), - 10, - TimeUnit.SECONDS + ) ); assertNotNull(clusterStateResponse); assertEquals("foo_bar_cluster", clusterStateResponse.getState().getClusterName().value()); // also test a failure, there is no handler for scroll registered - ActionNotFoundTransportException ex = expectThrows( + ActionNotFoundTransportException ex = asInstanceOf( ActionNotFoundTransportException.class, - () -> PlainActionFuture.get( - future -> client.execute(TransportSearchScrollAction.REMOTE_TYPE, new SearchScrollRequest(""), future) + ExceptionsHelper.unwrapCause( + safeAwaitFailure( + SearchResponse.class, + listener -> client.execute(TransportSearchScrollAction.REMOTE_TYPE, new SearchScrollRequest(""), listener) + ) ) ); assertEquals("No handler for action [indices:data/read/scroll]", ex.getMessage()); @@ -180,8 +183,8 @@ public void testEnsureWeReconnect() throws Exception { RemoteClusterService.DisconnectedStrategy.RECONNECT_UNLESS_SKIP_UNAVAILABLE ) ); - ClusterStateResponse clusterStateResponse = PlainActionFuture.get( - f -> client.execute(ClusterStateAction.REMOTE_TYPE, new ClusterStateRequest(), f) + ClusterStateResponse clusterStateResponse = safeAwait( + listener -> client.execute(ClusterStateAction.REMOTE_TYPE, new ClusterStateRequest(), listener) ); assertNotNull(clusterStateResponse); assertEquals("foo_bar_cluster", clusterStateResponse.getState().getClusterName().value()); @@ -267,11 +270,12 @@ public void testQuicklySkipUnavailableClusters() throws Exception { assertFalse(remoteClusterService.isRemoteNodeConnected("test", remoteNode)); // check that we quickly fail - expectThrows( - ConnectTransportException.class, - () -> PlainActionFuture.get( - f -> client.execute(ClusterStateAction.REMOTE_TYPE, new ClusterStateRequest(), f) - ) + ESTestCase.assertThat( + safeAwaitFailure( + ClusterStateResponse.class, + listener -> client.execute(ClusterStateAction.REMOTE_TYPE, new ClusterStateRequest(), listener) + ), + instanceOf(ConnectTransportException.class) ); } finally { service.clearAllRules(); @@ -279,14 +283,10 @@ public void testQuicklySkipUnavailableClusters() throws Exception { } assertBusy(() -> { - try { - PlainActionFuture.get( - f -> client.execute(ClusterStateAction.REMOTE_TYPE, new ClusterStateRequest(), f) - ); - } catch (ConnectTransportException e) { - // keep retrying on this exception, the goal is to check that we eventually reconnect - throw new AssertionError(e); - } + ClusterStateResponse ignored = safeAwait( + listener -> client.execute(ClusterStateAction.REMOTE_TYPE, new ClusterStateRequest(), listener) + ); + // keep retrying on an exception, the goal is to check that we eventually reconnect }); assertTrue(remoteClusterService.isRemoteNodeConnected("test", remoteNode)); } diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 77a57bf1110fb..23f6246e9191d 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -922,7 +922,7 @@ public void testGetConnection() throws Exception { RemoteClusterCredentialsManager.EMPTY ) ) { - PlainActionFuture.get(fut -> connection.ensureConnected(fut.map(x -> null))); + safeAwait(listener -> connection.ensureConnected(listener.map(x -> null))); for (int i = 0; i < 10; i++) { // always a direct connection as the remote node is already connected Transport.Connection remoteConnection = connection.getConnection(seedNode); diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteConnectionStrategyTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteConnectionStrategyTests.java index ca9986ba5eb1f..d40a244f40ebc 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteConnectionStrategyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteConnectionStrategyTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.EnumSerializationTestUtils; import static org.mockito.Mockito.mock; @@ -156,6 +157,14 @@ public void testTransportProfile() { } } + public void testConnectionStrategySerialization() { + EnumSerializationTestUtils.assertEnumSerialization( + RemoteConnectionStrategy.ConnectionStrategy.class, + RemoteConnectionStrategy.ConnectionStrategy.SNIFF, + RemoteConnectionStrategy.ConnectionStrategy.PROXY + ); + } + private static class FakeConnectionStrategy extends RemoteConnectionStrategy { private final ConnectionStrategy strategy; diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java index c5034f51d1e26..6f02c354b7485 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.Version; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.VersionInformation; @@ -21,6 +20,7 @@ import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.PageCacheRecycler; +import org.elasticsearch.core.Releasable; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexVersion; @@ -147,7 +147,7 @@ public void testConnectToNodeLight() { TestProfiles.LIGHT_PROFILE ) ) { - DiscoveryNode connectedNode = PlainActionFuture.get(fut -> transportServiceA.handshake(connection, timeout, fut)); + DiscoveryNode connectedNode = safeAwait(listener -> transportServiceA.handshake(connection, timeout, listener)); assertNotNull(connectedNode); // the name and version should be updated assertEquals(connectedNode.getName(), "TS_B"); @@ -177,21 +177,23 @@ public void testMismatchedClusterName() { .roles(emptySet()) .version(Version.CURRENT.minimumCompatibilityVersion(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current()) .build(); - IllegalStateException ex = expectThrows(IllegalStateException.class, () -> { - try ( - Transport.Connection connection = AbstractSimpleTransportTestCase.openConnection( - transportServiceA, - discoveryNode, - TestProfiles.LIGHT_PROFILE + try ( + Transport.Connection connection = AbstractSimpleTransportTestCase.openConnection( + transportServiceA, + discoveryNode, + TestProfiles.LIGHT_PROFILE + ) + ) { + assertThat( + asInstanceOf( + IllegalStateException.class, + safeAwaitFailure(DiscoveryNode.class, listener -> transportServiceA.handshake(connection, timeout, listener)) + ).getMessage(), + containsString( + "handshake with [" + discoveryNode + "] failed: remote cluster name [b] does not match local cluster name [a]" ) - ) { - PlainActionFuture.get(fut -> transportServiceA.handshake(connection, timeout, fut.map(x -> null))); - } - }); - assertThat( - ex.getMessage(), - containsString("handshake with [" + discoveryNode + "] failed: remote cluster name [b] does not match local cluster name [a]") - ); + ); + } assertFalse(transportServiceA.nodeConnected(discoveryNode)); } @@ -220,29 +222,29 @@ public void testIncompatibleNodeVersions() { .roles(emptySet()) .version(Version.CURRENT.minimumCompatibilityVersion(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current()) .build(); - IllegalStateException ex = expectThrows(IllegalStateException.class, () -> { - try ( - Transport.Connection connection = AbstractSimpleTransportTestCase.openConnection( - transportServiceA, - discoveryNode, - TestProfiles.LIGHT_PROFILE - ) - ) { - PlainActionFuture.get(fut -> transportServiceA.handshake(connection, timeout, fut.map(x -> null))); - } - }); - assertThat( - ex.getMessage(), - containsString( - "handshake with [" - + discoveryNode - + "] failed: remote node version [" - + transportServiceB.getLocalNode().getVersion() - + "] is incompatible with local node version [" - + Version.CURRENT - + "]" + try ( + Transport.Connection connection = AbstractSimpleTransportTestCase.openConnection( + transportServiceA, + discoveryNode, + TestProfiles.LIGHT_PROFILE ) - ); + ) { + assertThat( + asInstanceOf( + IllegalStateException.class, + safeAwaitFailure(DiscoveryNode.class, listener -> transportServiceA.handshake(connection, timeout, listener)) + ).getMessage(), + containsString( + "handshake with [" + + discoveryNode + + "] failed: remote node version [" + + transportServiceB.getLocalNode().getVersion() + + "] is incompatible with local node version [" + + Version.CURRENT + + "]" + ) + ); + } assertFalse(transportServiceA.nodeConnected(discoveryNode)); } @@ -267,17 +269,13 @@ public void testIncompatibleTransportVersions() { .roles(emptySet()) .version(Version.CURRENT.minimumCompatibilityVersion(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current()) .build(); - expectThrows(ConnectTransportException.class, () -> { - try ( - Transport.Connection connection = AbstractSimpleTransportTestCase.openConnection( - transportServiceA, - discoveryNode, - TestProfiles.LIGHT_PROFILE - ) - ) { - PlainActionFuture.get(fut -> transportServiceA.handshake(connection, timeout, fut.map(x -> null))); - } - }); + assertThat( + safeAwaitFailure( + Transport.Connection.class, + listener -> transportServiceA.openConnection(discoveryNode, TestProfiles.LIGHT_PROFILE, listener) + ), + instanceOf(ConnectTransportException.class) + ); // the error is exposed as a general connection exception, the actual message is in the logs assertFalse(transportServiceA.nodeConnected(discoveryNode)); } @@ -303,12 +301,14 @@ public void testNodeConnectWithDifferentNodeId() { .roles(emptySet()) .version(transportServiceB.getLocalNode().getVersionInformation()) .build(); - ConnectTransportException ex = expectThrows( - ConnectTransportException.class, - () -> AbstractSimpleTransportTestCase.connectToNode(transportServiceA, discoveryNode, TestProfiles.LIGHT_PROFILE) - ); assertThat( - ex.getMessage(), + asInstanceOf( + ConnectTransportException.class, + safeAwaitFailure( + Releasable.class, + listener -> transportServiceA.connectToNode(discoveryNode, TestProfiles.LIGHT_PROFILE, listener) + ) + ).getMessage(), allOf( containsString("Connecting to [" + discoveryNode.getAddress() + "] failed"), containsString("expected to connect to [" + discoveryNode.descriptionWithoutAttributes() + "]"), @@ -350,21 +350,24 @@ public void testRejectsMismatchedBuildHash() { .roles(emptySet()) .version(Version.CURRENT.minimumCompatibilityVersion(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current()) .build(); - TransportSerializationException ex = expectThrows(TransportSerializationException.class, () -> { - try ( - Transport.Connection connection = AbstractSimpleTransportTestCase.openConnection( - transportServiceA, - discoveryNode, - TestProfiles.LIGHT_PROFILE - ) - ) { - PlainActionFuture.get(fut -> transportServiceA.handshake(connection, timeout, fut.map(x -> null))); - } - }); - assertThat( - ExceptionsHelper.unwrap(ex, IllegalArgumentException.class).getMessage(), - containsString("which has an incompatible wire format") - ); + try ( + Transport.Connection connection = AbstractSimpleTransportTestCase.openConnection( + transportServiceA, + discoveryNode, + TestProfiles.LIGHT_PROFILE + ) + ) { + assertThat( + ExceptionsHelper.unwrap( + asInstanceOf( + TransportSerializationException.class, + safeAwaitFailure(DiscoveryNode.class, listener -> transportServiceA.handshake(connection, timeout, listener)) + ), + IllegalArgumentException.class + ).getMessage(), + containsString("which has an incompatible wire format") + ); + } assertFalse(transportServiceA.nodeConnected(discoveryNode)); } diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java index 78fffa5f84097..97b40dfeee52a 100644 --- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java +++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java @@ -16,7 +16,9 @@ import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.WarningsHandler; +import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; @@ -48,8 +50,10 @@ import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.hamcrest.Matchers.any; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.matchesRegex; /** * Tests that run ESQL queries that have, in the past, used so much memory they @@ -97,6 +101,78 @@ public void testSortByManyLongsTooMuchMemory() throws IOException { assertCircuitBreaks(() -> sortByManyLongs(5000)); } + /** + * This should record an async response with a {@link CircuitBreakingException}. + */ + public void testSortByManyLongsTooMuchMemoryAsync() throws IOException { + initManyLongs(); + Request request = new Request("POST", "/_query/async"); + request.addParameter("error_trace", ""); + request.setJsonEntity(makeSortByManyLongs(5000).toString().replace("\n", "\\n")); + request.setOptions( + RequestOptions.DEFAULT.toBuilder() + .setRequestConfig(RequestConfig.custom().setSocketTimeout(Math.toIntExact(TimeValue.timeValueMinutes(6).millis())).build()) + .setWarningsHandler(WarningsHandler.PERMISSIVE) + ); + logger.info("--> test {} started async", getTestName()); + Response response = runQuery(() -> { + Response r = client().performRequest(request); + Map map = responseAsMap(r); + assertMap(map, matchesMap().extraOk().entry("is_running", true).entry("id", any(String.class))); + String id = map.get("id").toString(); + Request fetch = new Request("GET", "/_query/async/" + id); + long endTime = System.nanoTime() + TimeValue.timeValueMinutes(5).nanos(); + while (System.nanoTime() < endTime) { + Response resp; + try { + resp = client().performRequest(fetch); + } catch (ResponseException e) { + if (e.getResponse().getStatusLine().getStatusCode() == 404) { + logger.error("polled for results got 404"); + continue; + } + if (e.getResponse().getStatusLine().getStatusCode() == 503) { + logger.error("polled for results got 503"); + continue; + } + if (e.getResponse().getStatusLine().getStatusCode() == 429) { + // This is what we were going to for - a CircuitBreakerException + return e.getResponse(); + } + throw e; + } + Map m = responseAsMap(resp); + logger.error("polled for results {}", m); + boolean isRunning = (boolean) m.get("is_running"); + if (isRunning == false) { + return resp; + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + throw new IOException("timed out"); + }); + assertThat(response.getStatusLine().getStatusCode(), equalTo(429)); + Map map = responseAsMap(response); + assertMap( + map, + matchesMap().entry("status", 429) + .entry( + "error", + matchesMap().extraOk() + .entry("bytes_wanted", greaterThan(1000)) + .entry("reason", matchesRegex("\\[request] Data too large, data for \\[topn] would .+")) + .entry("durability", "TRANSIENT") + .entry("type", "circuit_breaking_exception") + .entry("bytes_limit", greaterThan(1000)) + .entry("root_cause", matchesList().item(any(Map.class))) + ) + ); + } + private void assertCircuitBreaks(ThrowingRunnable r) throws IOException { ResponseException e = expectThrows(ResponseException.class, r); Map map = responseAsMap(e.getResponse()); @@ -109,13 +185,17 @@ private void assertCircuitBreaks(ThrowingRunnable r) throws IOException { private Response sortByManyLongs(int count) throws IOException { logger.info("sorting by {} longs", count); + return query(makeSortByManyLongs(count).toString(), null); + } + + private StringBuilder makeSortByManyLongs(int count) { StringBuilder query = makeManyLongs(count); query.append("| SORT a, b, i0"); for (int i = 1; i < count; i++) { query.append(", i").append(i); } query.append("\\n| KEEP a, b | LIMIT 10000\"}"); - return query(query.toString(), null); + return query; } /** @@ -295,12 +375,16 @@ private Response query(String query, String filterPath) throws IOException { if (filterPath != null) { request.addParameter("filter_path", filterPath); } - request.setJsonEntity(query.toString().replace("\n", "\\n")); + request.setJsonEntity(query.replace("\n", "\\n")); request.setOptions( RequestOptions.DEFAULT.toBuilder() .setRequestConfig(RequestConfig.custom().setSocketTimeout(Math.toIntExact(TimeValue.timeValueMinutes(6).millis())).build()) .setWarningsHandler(WarningsHandler.PERMISSIVE) ); + return runQuery(() -> client().performRequest(request)); + } + + private Response runQuery(CheckedSupplier run) throws IOException { logger.info("--> test {} started querying", getTestName()); final ThreadPool testThreadPool = new TestThreadPool(getTestName()); final long startedTimeInNanos = System.nanoTime(); @@ -321,11 +405,10 @@ protected void doRun() throws Exception { RequestConfig requestConfig = RequestConfig.custom() .setSocketTimeout(Math.toIntExact(TimeValue.timeValueMinutes(2).millis())) .build(); - request.setOptions(RequestOptions.DEFAULT.toBuilder().setRequestConfig(requestConfig)); client().performRequest(triggerOOM); } }, TimeValue.timeValueMinutes(5), testThreadPool.executor(ThreadPool.Names.GENERIC)); - Response resp = client().performRequest(request); + Response resp = run.get(); logger.info("--> test {} completed querying", getTestName()); return resp; } finally { diff --git a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpFixture.java b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpFixture.java index 2e80bdecbad8d..71c635972d62e 100644 --- a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpFixture.java +++ b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpFixture.java @@ -8,42 +8,140 @@ package fixture.azure; import com.sun.net.httpserver.HttpServer; +import com.sun.net.httpserver.HttpsConfigurator; +import com.sun.net.httpserver.HttpsServer; +import org.elasticsearch.common.ssl.KeyStoreUtil; +import org.elasticsearch.common.ssl.PemUtils; +import org.elasticsearch.test.ESTestCase; import org.junit.rules.ExternalResource; import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.SecureRandom; +import java.security.cert.Certificate; +import java.util.List; +import java.util.Objects; +import java.util.function.Predicate; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.SSLContext; + +import static org.elasticsearch.test.ESTestCase.assertThat; +import static org.elasticsearch.test.ESTestCase.fail; +import static org.hamcrest.Matchers.hasSize; public class AzureHttpFixture extends ExternalResource { - private final boolean enabled; + private final Protocol protocol; private final String account; private final String container; + private final Predicate authHeaderPredicate; + private HttpServer server; - public AzureHttpFixture(boolean enabled, String account, String container) { - this.enabled = enabled; + public enum Protocol { + NONE, + HTTP, + HTTPS + } + + /** + * @param account The name of the Azure Blob Storage account against which the request should be authorized.. + * @return a predicate that matches the {@code Authorization} HTTP header that the Azure SDK sends when using shared key auth (i.e. + * using a key or SAS token). + * @see Azure docs on shared key auth + */ + public static Predicate sharedKeyForAccountPredicate(String account) { + return new Predicate<>() { + @Override + public boolean test(String s) { + return s.startsWith("SharedKey " + account + ":"); + } + + @Override + public String toString() { + return "SharedKey[" + account + "]"; + } + }; + } + + public AzureHttpFixture(Protocol protocol, String account, String container, Predicate authHeaderPredicate) { + this.protocol = protocol; this.account = account; this.container = container; + this.authHeaderPredicate = authHeaderPredicate; + } + + private String scheme() { + return switch (protocol) { + case NONE -> fail(null, "fixture is disabled"); + case HTTP -> "http"; + case HTTPS -> "https"; + }; } public String getAddress() { - return "http://" + server.getAddress().getHostString() + ":" + server.getAddress().getPort() + "/" + account; + return scheme() + "://" + server.getAddress().getHostString() + ":" + server.getAddress().getPort() + "/" + account; } @Override - protected void before() throws IOException { - if (enabled) { - this.server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); - server.createContext("/" + account, new AzureHttpHandler(account, container)); - server.start(); + protected void before() { + try { + switch (protocol) { + case NONE -> { + } + case HTTP -> { + server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + server.createContext("/" + account, new AzureHttpHandler(account, container, authHeaderPredicate)); + server.start(); + } + case HTTPS -> { + final var httpsServer = HttpsServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + this.server = httpsServer; + final var tmpdir = ESTestCase.createTempDir(); + final var certificates = PemUtils.readCertificates(List.of(copyResource(tmpdir, "azure-http-fixture.pem"))); + assertThat(certificates, hasSize(1)); + final SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init( + new KeyManager[] { + KeyStoreUtil.createKeyManager( + new Certificate[] { certificates.get(0) }, + PemUtils.readPrivateKey(copyResource(tmpdir, "azure-http-fixture.key"), () -> null), + null + ) }, + null, + new SecureRandom() + ); + httpsServer.setHttpsConfigurator(new HttpsConfigurator(sslContext)); + httpsServer.createContext("/" + account, new AzureHttpHandler(account, container, authHeaderPredicate)); + httpsServer.start(); + } + } + } catch (Exception e) { + throw new AssertionError("unexpected", e); + } + } + + private Path copyResource(Path tmpdir, String name) throws IOException { + try ( + var stream = Objects.requireNonNullElseGet( + getClass().getResourceAsStream(name), + () -> ESTestCase.fail(null, "resource [%s] not found", name) + ) + ) { + final var path = tmpdir.resolve(name); + Files.write(path, stream.readAllBytes()); + return path; } } @Override protected void after() { - if (enabled) { + if (protocol != Protocol.NONE) { server.stop(0); } } diff --git a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java index e49941efa1e70..d46afdcf93cd2 100644 --- a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java +++ b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java @@ -15,9 +15,12 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.RestUtils; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -32,6 +35,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -45,15 +49,67 @@ public class AzureHttpHandler implements HttpHandler { private final Map blobs; private final String account; private final String container; + private final Predicate authHeaderPredicate; - public AzureHttpHandler(final String account, final String container) { + public AzureHttpHandler(final String account, final String container, @Nullable Predicate authHeaderPredicate) { this.account = Objects.requireNonNull(account); this.container = Objects.requireNonNull(container); + this.authHeaderPredicate = authHeaderPredicate; this.blobs = new ConcurrentHashMap<>(); } + private static List getAuthHeader(HttpExchange exchange) { + return exchange.getRequestHeaders().get("Authorization"); + } + + private boolean isValidAuthHeader(HttpExchange exchange) { + if (authHeaderPredicate == null) { + return true; + } + + final var authHeader = getAuthHeader(exchange); + if (authHeader == null) { + return false; + } + + if (authHeader.size() != 1) { + return false; + } + + return authHeaderPredicate.test(authHeader.get(0)); + } + @Override public void handle(final HttpExchange exchange) throws IOException { + if (isValidAuthHeader(exchange) == false) { + try (exchange; var builder = XContentBuilder.builder(XContentType.JSON.xContent())) { + builder.startObject(); + builder.field("method", exchange.getRequestMethod()); + builder.field("uri", exchange.getRequestURI().toString()); + builder.field("predicate", authHeaderPredicate.toString()); + builder.field("authorization", Objects.toString(getAuthHeader(exchange))); + builder.startObject("headers"); + for (final var header : exchange.getRequestHeaders().entrySet()) { + if (header.getValue() == null) { + builder.nullField(header.getKey()); + } else { + builder.startArray(header.getKey()); + for (final var value : header.getValue()) { + builder.value(value); + } + builder.endArray(); + } + } + builder.endObject(); + builder.endObject(); + final var responseBytes = BytesReference.bytes(builder); + exchange.getResponseHeaders().add("Content-Type", "application/json; charset=utf-8"); + exchange.sendResponseHeaders(RestStatus.FORBIDDEN.getStatus(), responseBytes.length()); + responseBytes.writeTo(exchange.getResponseBody()); + return; + } + } + final String request = exchange.getRequestMethod() + " " + exchange.getRequestURI().toString(); if (request.startsWith("GET") || request.startsWith("HEAD") || request.startsWith("DELETE")) { int read = exchange.getRequestBody().read(); @@ -131,6 +187,13 @@ public void handle(final HttpExchange exchange) throws IOException { final long start = Long.parseLong(matcher.group(1)); final long end = Long.parseLong(matcher.group(2)); + + if (blob.length() <= start) { + exchange.getResponseHeaders().add("Content-Type", "application/octet-stream"); + exchange.sendResponseHeaders(RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus(), -1); + return; + } + var responseBlob = blob.slice(Math.toIntExact(start), Math.toIntExact(Math.min(end - start + 1, blob.length() - start))); exchange.getResponseHeaders().add("Content-Type", "application/octet-stream"); diff --git a/test/fixtures/azure-fixture/src/main/resources/fixture/azure/azure-http-fixture.key b/test/fixtures/azure-fixture/src/main/resources/fixture/azure/azure-http-fixture.key new file mode 100644 index 0000000000000..29efd3a6913a0 --- /dev/null +++ b/test/fixtures/azure-fixture/src/main/resources/fixture/azure/azure-http-fixture.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCswbyZDtaghZXs +Phs1+lqCnq5HmRT2P6Drrs9bJlABeql29IhzdHOGLr+lTMhKOUpHuphgC31qbf/G +vQLS65qdOzTjNfLv93+Jj0gp4S7Q6eRZvn1ihUgzECHazTYwIlzVs4sFPm5i2fQb +DK7W6zQm+h9r6GjCYj01OeIAe7rbRI9Ar+svuHGfZnaQHzLZlfYkkM2bCaXBgKWV +wmEUmwMW+IMOPCrVm+gk1MDbGnu9KtY/LqrJcddsqOdkK8qJ0Lpchg3zlP4qIzbm +WRyTUIy1USbcazjuC/vMmN4fr/Xr0Jrhi4Rw8l2LGdyA8qnqtKYTqMzo3uv1ESlE +R8EAZDUbAgMBAAECggEAY8lYRdSTVo8y5Q2OrCQa6b38jvC2cfKY4enMbir4JZKT +lllzA7VtEUGpgzKRsoXbCQmYAEpCvBojlskQe4KJgW50gxVjaQa9zVhM55vhbdzc +AJaOWD0CUjRsSbUlKrJ+ixW1JGdGXaTlYkZ2K0AalLT/N1Y8RKN4FWmEyKCvcvz4 +0XzOIVmG+HqcNURamXTxMKbj1yzi5goue2/iP2kMFo8sHxRsGvvV4PWo6JrktE2y +47oiH42lpSIcpLSE8z/csLbMTw/Q/pPQAYqyvEJHU22uhac1XmMqFHWNSpQZq6gr +46t8YQ3FJSN8UrZf1h1mdvLlK/sgPEvCQa6TrCq4GQKBgQDbl0M/4gJZhgpvBuCC +aratamWcFoa/pu1/JoQhPXLv6uGwB/cFhJUVr0ZoI5KPFJr9SG4kQ/eEPywkY2qT +mGPVlVmGOeJa1VK8TRUTzkEFTYWytUepACM2//LiWvzABciO8SxWgNZrmUKghQTN +d989b8edy0ti6y7lHpkTyawVXQKBgQDJZo7X6K+6cyuXORApv98cU5C3vEVtBp/c +QfU/rRj/YXAbYFhKIS5rF/Gfg2YoDa5YguMxGsIPzYwdTI5gGGgSgolapz3fr22q +edCPaFg8qO64pIii+Ar4lx4k1IyNtpJ+nvlam7sI9yGzksrVazsWhpaSKX8xGd7r +9ZSr/c8U1wKBgGWl+pJay52nR7MnasvUHCXgR5LedpfG7M9cA/PjHw5iGwDCXx2l +xuFX1m6kcNZcwnYWji2pbK1CFOvvPUl/VE9tKBjTOK21a+wQfn5BjqWmwgn8kmRv +1N1D06nmVnOI+dL5Xv3X++mo80ec66E1KRimYq/viEEM/xM+e7vGMitdAoGAUAUe +pix+fa8615fFk0D37bJKIqZ8Uyg5pfLS9ZzZ/MYDG+14xuNOJSDbUMyNb0aYSfSf +PihqiIrbq9x6CTZJS2lwF4Oxcsmp4f0KX6BOxrM8PkKpQ08YVNL+GBYXTksHA6Y4 +XsbXVmWSj124l3lGfdm1w5cXQTQNPWVSz89FUvsCgYEArl27gbXndTLpZ4dNkeBS +0JMe84nPPrrwHbNwcqkoDiux5Ln+AZE8jYSlp4kUfd1A7XDTxXWXIxeD0YLiabf3 +5+/QzJ6j1qi77JoyadomnL7CFI5f2FotKt029PeVxAUOohao94p8J5OuaRMPvkGC +CNhjfRAIBhfm9kdyjPmwVLU= +-----END PRIVATE KEY----- diff --git a/test/fixtures/azure-fixture/src/main/resources/fixture/azure/azure-http-fixture.pem b/test/fixtures/azure-fixture/src/main/resources/fixture/azure/azure-http-fixture.pem new file mode 100644 index 0000000000000..b291aaa9362de --- /dev/null +++ b/test/fixtures/azure-fixture/src/main/resources/fixture/azure/azure-http-fixture.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDRTCCAi2gAwIBAgIVAJpxxIbXWyvdd6/rIFXPgWe6fyvTMA0GCSqGSIb3DQEB +CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu +ZXJhdGVkIENBMB4XDTE4MTIyMTA3NDY1NVoXDTQ2MDUwNzA3NDY1NVowGTEXMBUG +A1UEAxMObGRhcC10ZXN0LWNhc2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQCswbyZDtaghZXsPhs1+lqCnq5HmRT2P6Drrs9bJlABeql29IhzdHOGLr+l +TMhKOUpHuphgC31qbf/GvQLS65qdOzTjNfLv93+Jj0gp4S7Q6eRZvn1ihUgzECHa +zTYwIlzVs4sFPm5i2fQbDK7W6zQm+h9r6GjCYj01OeIAe7rbRI9Ar+svuHGfZnaQ +HzLZlfYkkM2bCaXBgKWVwmEUmwMW+IMOPCrVm+gk1MDbGnu9KtY/LqrJcddsqOdk +K8qJ0Lpchg3zlP4qIzbmWRyTUIy1USbcazjuC/vMmN4fr/Xr0Jrhi4Rw8l2LGdyA +8qnqtKYTqMzo3uv1ESlER8EAZDUbAgMBAAGjaTBnMB0GA1UdDgQWBBQaiCDScfBa +jHOSk04XOymffbLBxTAfBgNVHSMEGDAWgBROJaHRWe17um5rqqYn10aqedr55DAa +BgNVHREEEzARgglsb2NhbGhvc3SHBH8AAAEwCQYDVR0TBAIwADANBgkqhkiG9w0B +AQsFAAOCAQEAXBovNqVg+VQ1LR0PfEMpbgbQlekky8qY2y1tz7J0ntGepAq+Np6n +7J9En6ty1ELZUvgPUCF2btQqZbv8uyHz/C+rojKC5xzHN5qbZ31o5/0I/kNase1Z +NbXuNJe3wAXuz+Mj5rtuOGZvlFsbtocuoydVYOclfqjUXcoZtqCcRamSvye7vGl2 +CHPqDi0uK8d75nE9Jrnmz/BNNV7CjPg636PJmCUrLL21+t69ZFL1eGAFtLBmmjcw +cMkyv9bJirjZbjt/9UB+fW9XzV3RVLAzfrIHtToupXmWc4+hTOnlbKfFwqB9fa7Y +XcCfGrZoJg9di1HbJrSJmv5QgRTM+/zkrA== +-----END CERTIFICATE----- diff --git a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java index 7f9c50204b0a7..43894814c7a60 100644 --- a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java +++ b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java @@ -144,6 +144,13 @@ public void handle(final HttpExchange exchange) throws IOException { offset = Long.parseLong(matcher.group(1)); end = Long.parseLong(matcher.group(2)); } + + if (offset >= blob.length()) { + exchange.getResponseHeaders().add("Content-Type", "application/octet-stream"); + exchange.sendResponseHeaders(RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus(), -1); + return; + } + BytesReference response = blob; exchange.getResponseHeaders().add("Content-Type", "application/octet-stream"); final int bufferedLength = response.length(); diff --git a/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/EnterpriseGeoIpHttpFixture.java b/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/EnterpriseGeoIpHttpFixture.java new file mode 100644 index 0000000000000..5932890dd8459 --- /dev/null +++ b/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/EnterpriseGeoIpHttpFixture.java @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package fixture.geoip; + +import com.sun.net.httpserver.HttpServer; + +import org.elasticsearch.common.hash.MessageDigests; +import org.junit.rules.ExternalResource; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; + +/** + * This fixture is used to simulate a maxmind-provided server for downloading maxmind geoip database files from the + * EnterpriseGeoIpDownloader. It can be used by integration tests so that they don't actually hit maxmind servers. + */ +public class EnterpriseGeoIpHttpFixture extends ExternalResource { + + private final Path source; + private final String[] databaseTypes; + private HttpServer server; + + /* + * The values in databaseTypes must be in DatabaseConfiguration.MAXMIND_NAMES, and must be one of the databases copied in the + * copyFiles method of thisi class. + */ + public EnterpriseGeoIpHttpFixture(String... databaseTypes) { + this.databaseTypes = databaseTypes; + try { + this.source = Files.createTempDirectory("source"); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public String getAddress() { + return "http://" + server.getAddress().getHostString() + ":" + server.getAddress().getPort() + "/"; + } + + @Override + protected void before() throws Throwable { + copyFiles(); + this.server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + + // for expediency reasons, it is handy to have this test fixture be able to serve the dual purpose of actually stubbing + // out the download protocol for downloading files from maxmind (see the looped context creation after this stanza), as + // we as to serve an empty response for the geoip.elastic.co service here + this.server.createContext("/", exchange -> { + String response = "[]"; // an empty json array + exchange.sendResponseHeaders(200, response.length()); + try (OutputStream os = exchange.getResponseBody()) { + os.write(response.getBytes(StandardCharsets.UTF_8)); + } + }); + + // register the file types for the download fixture + for (String databaseType : databaseTypes) { + createContextForEnterpriseDatabase(databaseType); + } + + server.start(); + } + + private void createContextForEnterpriseDatabase(String databaseType) { + this.server.createContext("/" + databaseType + "/download", exchange -> { + exchange.sendResponseHeaders(200, 0); + if (exchange.getRequestURI().toString().contains("sha256")) { + MessageDigest sha256 = MessageDigests.sha256(); + try (InputStream inputStream = GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/" + databaseType + ".tgz")) { + sha256.update(inputStream.readAllBytes()); + } + exchange.getResponseBody() + .write( + (MessageDigests.toHexString(sha256.digest()) + " " + databaseType + "_20240709.tar.gz").getBytes( + StandardCharsets.UTF_8 + ) + ); + } else { + try ( + OutputStream outputStream = exchange.getResponseBody(); + InputStream inputStream = GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/" + databaseType + ".tgz") + ) { + inputStream.transferTo(outputStream); + } + } + exchange.getResponseBody().close(); + }); + } + + @Override + protected void after() { + server.stop(0); + } + + private void copyFiles() throws Exception { + for (String databaseType : databaseTypes) { + Files.copy( + GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/GeoIP2-City.tgz"), + source.resolve(databaseType + ".tgz"), + StandardCopyOption.REPLACE_EXISTING + ); + } + } +} diff --git a/test/fixtures/geoip-fixture/src/main/resources/geoip-fixture/GeoIP2-City.tgz b/test/fixtures/geoip-fixture/src/main/resources/geoip-fixture/GeoIP2-City.tgz new file mode 100644 index 0000000000000..76dd40000f132 Binary files /dev/null and b/test/fixtures/geoip-fixture/src/main/resources/geoip-fixture/GeoIP2-City.tgz differ diff --git a/test/fixtures/krb5kdc-fixture/src/main/java/org/elasticsearch/test/fixtures/krb5kdc/Krb5kDcContainer.java b/test/fixtures/krb5kdc-fixture/src/main/java/org/elasticsearch/test/fixtures/krb5kdc/Krb5kDcContainer.java index fa75b57ea87a6..14357b6d47bbc 100644 --- a/test/fixtures/krb5kdc-fixture/src/main/java/org/elasticsearch/test/fixtures/krb5kdc/Krb5kDcContainer.java +++ b/test/fixtures/krb5kdc-fixture/src/main/java/org/elasticsearch/test/fixtures/krb5kdc/Krb5kDcContainer.java @@ -22,6 +22,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -74,6 +75,7 @@ public Krb5kDcContainer(ProvisioningId provisioningId) { this.provisioningId = provisioningId; withNetwork(Network.newNetwork()); addExposedPorts(88, 4444); + withStartupTimeout(Duration.ofMinutes(2)); withCreateContainerCmdModifier(cmd -> { // Add previously exposed ports and UDP port List exposedPorts = new ArrayList<>(); diff --git a/test/framework/build.gradle b/test/framework/build.gradle index 7906a52479b29..4d598a00de7b6 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -16,7 +16,6 @@ dependencies { api project(':libs:elasticsearch-ssl-config') api project(":server") api project(":libs:elasticsearch-cli") - api project(':libs:elasticsearch-preallocate') api "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" api "junit:junit:${versions.junit}" api "org.hamcrest:hamcrest:${versions.hamcrest}" @@ -64,7 +63,15 @@ tasks.named("thirdPartyAudit").configure { // mockito 'net.bytebuddy.agent.ByteBuddyAgent', 'org.mockito.internal.creation.bytebuddy.inject.MockMethodDispatcher', - 'org.opentest4j.AssertionFailedError' + 'org.opentest4j.AssertionFailedError', + + // not using JNA + 'com.sun.jna.FunctionMapper', + 'com.sun.jna.JNIEnv', + 'com.sun.jna.Library', + 'com.sun.jna.Native', + 'com.sun.jna.NativeLibrary', + 'com.sun.jna.Platform' ) ignoreViolations( diff --git a/test/framework/src/main/java/org/elasticsearch/action/support/ActionTestUtils.java b/test/framework/src/main/java/org/elasticsearch/action/support/ActionTestUtils.java index 023305101f4c4..6f48c5a9f8bc5 100644 --- a/test/framework/src/main/java/org/elasticsearch/action/support/ActionTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/action/support/ActionTestUtils.java @@ -21,7 +21,6 @@ import org.elasticsearch.transport.Transport; import java.util.Map; -import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import static org.elasticsearch.test.ESTestCase.fail; @@ -34,10 +33,8 @@ public static R TransportAction action, Request request ) { - return PlainActionFuture.get( - future -> action.execute(request.createTask(1L, "direct", action.actionName, TaskId.EMPTY_TASK_ID, Map.of()), request, future), - 10, - TimeUnit.SECONDS + return ESTestCase.safeAwait( + future -> action.execute(request.createTask(1L, "direct", action.actionName, TaskId.EMPTY_TASK_ID, Map.of()), request, future) ); } @@ -47,11 +44,7 @@ public static R TransportAction action, Request request ) { - return PlainActionFuture.get( - future -> taskManager.registerAndExecute("transport", action, request, localConnection, future), - 10, - TimeUnit.SECONDS - ); + return ESTestCase.safeAwait(future -> taskManager.registerAndExecute("transport", action, request, localConnection, future)); } /** diff --git a/test/framework/src/main/java/org/elasticsearch/action/support/TestPlainActionFuture.java b/test/framework/src/main/java/org/elasticsearch/action/support/TestPlainActionFuture.java new file mode 100644 index 0000000000000..0264920c9d017 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/action/support/TestPlainActionFuture.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.support; + +/** + * A {@link PlainActionFuture} which bypasses the deadlock-detection checks since we're only using this in tests. + */ +public class TestPlainActionFuture extends PlainActionFuture { + @Override + boolean allowedExecutors(Thread thread1, Thread thread2) { + return true; + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java index 8ef80c08517de..c231502f9692c 100644 --- a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java +++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java @@ -14,7 +14,6 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.filesystem.FileSystemNatives; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.network.IfConfig; import org.elasticsearch.common.settings.Settings; @@ -91,9 +90,6 @@ public class BootstrapForTesting { final boolean systemCallFilter = Booleans.parseBoolean(System.getProperty("tests.system_call_filter", "true")); Elasticsearch.initializeNatives(javaTmpDir, memoryLock, systemCallFilter, true); - // init filesystem natives - FileSystemNatives.init(); - // initialize probes Elasticsearch.initializeProbes(); @@ -222,7 +218,6 @@ static Map getCodebases() { addClassCodebase(codebases, "elasticsearch-rest-client", "org.elasticsearch.client.RestClient"); addClassCodebase(codebases, "elasticsearch-core", "org.elasticsearch.core.Booleans"); addClassCodebase(codebases, "elasticsearch-cli", "org.elasticsearch.cli.Command"); - addClassCodebase(codebases, "elasticsearch-preallocate", "org.elasticsearch.preallocate.Preallocate"); addClassCodebase(codebases, "elasticsearch-simdvec", "org.elasticsearch.simdvec.VectorScorerFactory"); addClassCodebase(codebases, "framework", "org.elasticsearch.test.ESTestCase"); return codebases; diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index 70738c510f62a..290b4ac6dd3e3 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -64,7 +64,6 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException; import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Nullable; @@ -127,6 +126,7 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiFunction; @@ -250,7 +250,7 @@ public static EngineConfig copy(EngineConfig config, LongSupplier globalCheckpoi config.getMergePolicy(), config.getAnalyzer(), config.getSimilarity(), - config.getCodecService(), + config.getCodecProvider(), config.getEventListener(), config.getQueryCache(), config.getQueryCachingPolicy(), @@ -282,7 +282,7 @@ public EngineConfig copy(EngineConfig config, Analyzer analyzer) { config.getMergePolicy(), analyzer, config.getSimilarity(), - config.getCodecService(), + config.getCodecProvider(), config.getEventListener(), config.getQueryCache(), config.getQueryCachingPolicy(), @@ -314,7 +314,7 @@ public EngineConfig copy(EngineConfig config, MergePolicy mergePolicy) { mergePolicy, config.getAnalyzer(), config.getSimilarity(), - config.getCodecService(), + config.getCodecProvider(), config.getEventListener(), config.getQueryCache(), config.getQueryCachingPolicy(), @@ -670,7 +670,7 @@ public static InternalEngine createInternalEngine( if (localCheckpointTrackerSupplier == null) { return new InternalTestEngine(config) { @Override - IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOException { + protected IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOException { return (indexWriterFactory != null) ? indexWriterFactory.createWriter(directory, iwc) : super.createWriter(directory, iwc); @@ -686,7 +686,7 @@ protected long doGenerateSeqNoForOperation(final Operation operation) { } else { return new InternalTestEngine(config, IndexWriter.MAX_DOCS, localCheckpointTrackerSupplier) { @Override - IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOException { + protected IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOException { return (indexWriterFactory != null) ? indexWriterFactory.createWriter(directory, iwc) : super.createWriter(directory, iwc); @@ -1628,22 +1628,22 @@ public static void recoverFromTranslog(Engine engine, Engine.TranslogRecoveryRun throws IOException { // This is an adapter between the older synchronous (blocking) code and the newer (async) API. Callers expect exceptions to be // thrown directly, so we must undo the layers of wrapping added by future#get and friends. + final var future = new PlainActionFuture(); + engine.recoverFromTranslog(translogRecoveryRunner, recoverUpToSeqNo, future); try { - PlainActionFuture.get( - future -> engine.recoverFromTranslog(translogRecoveryRunner, recoverUpToSeqNo, future), - 30, - TimeUnit.SECONDS - ); - } catch (UncategorizedExecutionException e) { - if (e.getCause() instanceof ExecutionException executionException - && executionException.getCause() instanceof IOException ioException) { + future.get(30, TimeUnit.SECONDS); + } catch (ExecutionException e) { + if (e.getCause() instanceof IOException ioException) { throw ioException; - } else { - fail(e); } - } catch (RuntimeException e) { - throw e; - } catch (Exception e) { + if (e.getCause() instanceof RuntimeException runtimeException) { + throw runtimeException; + } + fail(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + fail(e); + } catch (TimeoutException e) { fail(e); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java index 675b5959f35a3..d812e158a1675 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.query.ExistsQueryBuilder; @@ -265,6 +266,7 @@ protected static FieldDataContext mockFielddataContext() { SearchExecutionContext searchExecutionContext = mockContext(); return new FieldDataContext( "test", + null, searchExecutionContext::lookup, mockContext()::sourcePath, MappedFieldType.FielddataOperation.SCRIPT @@ -299,7 +301,7 @@ protected static SearchExecutionContext mockContext( when(context.allowExpensiveQueries()).thenReturn(allowExpensiveQueries); SearchLookup lookup = new SearchLookup( context::getFieldType, - (mft, lookupSupplier, fdo) -> mft.fielddataBuilder(new FieldDataContext("test", lookupSupplier, context::sourcePath, fdo)) + (mft, lookupSupplier, fdo) -> mft.fielddataBuilder(new FieldDataContext("test", null, lookupSupplier, context::sourcePath, fdo)) .build(null, null), sourceProvider ); @@ -307,7 +309,7 @@ protected static SearchExecutionContext mockContext( when(context.getForField(any(), any())).then(args -> { MappedFieldType ft = args.getArgument(0); MappedFieldType.FielddataOperation fdo = args.getArgument(1); - return ft.fielddataBuilder(new FieldDataContext("test", context::lookup, context::sourcePath, fdo)) + return ft.fielddataBuilder(new FieldDataContext("test", null, context::lookup, context::sourcePath, fdo)) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); }); when(context.getMatchingFieldNames(any())).thenReturn(Set.of("dummy_field")); @@ -452,6 +454,11 @@ public String indexName() { throw new UnsupportedOperationException(); } + @Override + public IndexSettings indexSettings() { + throw new UnsupportedOperationException(); + } + @Override public MappedFieldType.FieldExtractPreference fieldExtractPreference() { return MappedFieldType.FieldExtractPreference.NONE; diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index a6b737f162547..b5a42efd67088 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -36,6 +36,7 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; @@ -151,7 +152,7 @@ protected final DocumentMapper createTimeSeriesModeDocumentMapper(XContentBuilde } protected final DocumentMapper createLogsModeDocumentMapper(XContentBuilder mappings) throws IOException { - Settings settings = Settings.builder().put(IndexSettings.MODE.getKey(), "logs").build(); + Settings settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()).build(); return createMapperService(settings, mappings).documentMapper(); } @@ -225,6 +226,7 @@ protected class TestMapperServiceBuilder { private BooleanSupplier idFieldDataEnabled; private ScriptCompiler scriptCompiler; private MapperMetrics mapperMetrics; + private boolean applyDefaultMapping; public TestMapperServiceBuilder() { indexVersion = getVersion(); @@ -232,6 +234,7 @@ public TestMapperServiceBuilder() { idFieldDataEnabled = () -> true; scriptCompiler = MapperServiceTestCase.this::compileScript; mapperMetrics = MapperMetrics.NOOP; + applyDefaultMapping = true; } public TestMapperServiceBuilder indexVersion(IndexVersion indexVersion) { @@ -254,6 +257,11 @@ public TestMapperServiceBuilder mapperMetrics(MapperMetrics mapperMetrics) { return this; } + public TestMapperServiceBuilder applyDefaultMapping(boolean applyDefaultMapping) { + this.applyDefaultMapping = applyDefaultMapping; + return this; + } + public MapperService build() { IndexSettings indexSettings = createIndexSettings(indexVersion, settings); SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of()); @@ -269,7 +277,7 @@ public void onCache(ShardId shardId, Accountable accountable) {} public void onRemoval(ShardId shardId, Accountable accountable) {} }); - return new MapperService( + var mapperService = new MapperService( () -> TransportVersion.current(), indexSettings, createIndexAnalyzers(indexSettings), @@ -284,6 +292,12 @@ public void onRemoval(ShardId shardId, Accountable accountable) {} bitsetFilterCache::getBitSetProducer, mapperMetrics ); + + if (applyDefaultMapping && indexSettings.getMode().getDefaultMapping() != null) { + mapperService.merge(null, indexSettings.getMode().getDefaultMapping(), MapperService.MergeReason.MAPPING_UPDATE); + } + + return mapperService; } } @@ -764,7 +778,7 @@ protected TriFunction, MappedFieldType.F protected TriFunction, MappedFieldType.FielddataOperation, IndexFieldData> fieldDataLookup( Function> sourcePathsLookup ) { - return (mft, lookupSource, fdo) -> mft.fielddataBuilder(new FieldDataContext("test", lookupSource, sourcePathsLookup, fdo)) + return (mft, lookupSource, fdo) -> mft.fielddataBuilder(new FieldDataContext("test", null, lookupSource, sourcePathsLookup, fdo)) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index fd028ed32f975..9eaace8f93e58 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -578,7 +578,13 @@ protected static void assertScriptDocValues(MapperService mapperService, Object } : SourceProvider.fromStoredFields(); SearchLookup searchLookup = new SearchLookup(null, null, sourceProvider); IndexFieldData sfd = ft.fielddataBuilder( - new FieldDataContext("", () -> searchLookup, Set::of, MappedFieldType.FielddataOperation.SCRIPT) + new FieldDataContext( + "", + mapperService.getIndexSettings(), + () -> searchLookup, + Set::of, + MappedFieldType.FielddataOperation.SCRIPT + ) ).build(null, null); LeafFieldData lfd = sfd.load(getOnlyLeafReader(searcher.getIndexReader()).getContext()); DocValuesScriptFieldFactory sff = lfd.getScriptFieldFactory("field"); @@ -1325,6 +1331,11 @@ public String indexName() { throw new UnsupportedOperationException(); } + @Override + public IndexSettings indexSettings() { + throw new UnsupportedOperationException(); + } + @Override public MappedFieldType.FieldExtractPreference fieldExtractPreference() { return columnReader ? DOC_VALUES : NONE; diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index 0488614f04dfb..4d15bf9e3f943 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.UnsafePlainActionFuture; import org.elasticsearch.action.support.replication.TransportReplicationAction; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -945,19 +946,20 @@ protected void promoteReplica(IndexShard replica, Set inSyncIds, IndexSh } public static Releasable getOperationPermit(final IndexShard shard) { - return PlainActionFuture.get(future -> { - if (shard.routingEntry().primary()) { - shard.acquirePrimaryOperationPermit(future, null); - } else { - shard.acquireReplicaOperationPermit( - shard.getOperationPrimaryTerm(), - SequenceNumbers.NO_OPS_PERFORMED, - SequenceNumbers.NO_OPS_PERFORMED, - future, - null - ); - } - }, 0, TimeUnit.NANOSECONDS); + final var listener = new SubscribableListener(); + if (shard.routingEntry().primary()) { + shard.acquirePrimaryOperationPermit(listener, null); + } else { + shard.acquireReplicaOperationPermit( + shard.getOperationPrimaryTerm(), + SequenceNumbers.NO_OPS_PERFORMED, + SequenceNumbers.NO_OPS_PERFORMED, + listener, + null + ); + } + assertTrue(listener.isDone()); + return safeAwait(listener); } public static Set getShardDocUIDs(final IndexShard shard) throws IOException { @@ -1191,6 +1193,6 @@ public static Engine.Warmer createTestWarmer(IndexSettings indexSettings) { } public static long recoverLocallyUpToGlobalCheckpoint(IndexShard indexShard) { - return PlainActionFuture.get(indexShard::recoverLocallyUpToGlobalCheckpoint, 10, TimeUnit.SECONDS); + return safeAwait(indexShard::recoverLocallyUpToGlobalCheckpoint); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGenerator.java new file mode 100644 index 0000000000000..1aec45fa3f287 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGenerator.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.logsdb.datageneration; + +import org.elasticsearch.logsdb.datageneration.fields.ObjectFieldDataGenerator; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Entry point of data generation logic. + * Every instance of generator generates a random mapping and a document generation routine + * that produces randomly generated documents valid for this mapping. + */ +public class DataGenerator { + private final FieldDataGenerator topLevelGenerator; + + public DataGenerator(DataGeneratorSpecification specification) { + this.topLevelGenerator = new ObjectFieldDataGenerator(specification); + } + + public void writeMapping(XContentBuilder mapping) throws IOException { + mapping.startObject().field("_doc"); + topLevelGenerator.mappingWriter().accept(mapping); + mapping.endObject(); + } + + public void generateDocument(XContentBuilder document) throws IOException { + topLevelGenerator.fieldValueGenerator().accept(document); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSpecification.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSpecification.java new file mode 100644 index 0000000000000..4a0ed074b1411 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSpecification.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.logsdb.datageneration; + +import org.elasticsearch.logsdb.datageneration.arbitrary.Arbitrary; +import org.elasticsearch.logsdb.datageneration.arbitrary.RandomBasedArbitrary; + +/** + * Allows configuring behavior of {@link DataGenerator}. + * @param arbitrary provides arbitrary values used during generation + * @param maxFieldCountPerLevel maximum number of fields that an individual object in mapping has. + * Applies to subobjects. + * @param maxObjectDepth maximum depth of nested objects + * @param nestedFieldsLimit how many total nested fields can be present in a produced mapping + */ +public record DataGeneratorSpecification(Arbitrary arbitrary, int maxFieldCountPerLevel, int maxObjectDepth, int nestedFieldsLimit) { + + public static Builder builder() { + return new Builder(); + } + + public static DataGeneratorSpecification buildDefault() { + return builder().build(); + } + + public static class Builder { + private Arbitrary arbitrary; + private int maxFieldCountPerLevel; + private int maxObjectDepth; + private int nestedFieldsLimit; + + public Builder() { + // Simply sufficiently big numbers to get some permutations + maxFieldCountPerLevel = 50; + maxObjectDepth = 3; + // Default value of index.mapping.nested_fields.limit + nestedFieldsLimit = 50; + arbitrary = new RandomBasedArbitrary(); + } + + public Builder withArbitrary(Arbitrary arbitrary) { + this.arbitrary = arbitrary; + return this; + } + + public Builder withMaxFieldCountPerLevel(int maxFieldCountPerLevel) { + this.maxFieldCountPerLevel = maxFieldCountPerLevel; + return this; + } + + public Builder withMaxObjectDepth(int maxObjectDepth) { + this.maxObjectDepth = maxObjectDepth; + return this; + } + + public Builder withNestedFieldsLimit(int nestedFieldsLimit) { + this.nestedFieldsLimit = nestedFieldsLimit; + return this; + } + + public DataGeneratorSpecification build() { + return new DataGeneratorSpecification(arbitrary, maxFieldCountPerLevel, maxObjectDepth, nestedFieldsLimit); + } + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java new file mode 100644 index 0000000000000..b53794fdd1961 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.logsdb.datageneration; + +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Entity responsible for generating a valid randomized mapping for a field + * and a generator of field values valid for this mapping. + * + * Generator is expected to produce the same mapping per instance of generator. + * Function returned by {@link FieldDataGenerator#fieldValueGenerator() } is expected + * to produce a randomized value each time. + */ +public interface FieldDataGenerator { + CheckedConsumer mappingWriter(); + + CheckedConsumer fieldValueGenerator(); +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java new file mode 100644 index 0000000000000..0a675d85077e4 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.logsdb.datageneration; + +/** + * Lists all leaf field types that are supported for data generation. + */ +public enum FieldType { + KEYWORD, + LONG +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/arbitrary/Arbitrary.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/arbitrary/Arbitrary.java new file mode 100644 index 0000000000000..139994d530f77 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/arbitrary/Arbitrary.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.logsdb.datageneration.arbitrary; + +import org.elasticsearch.logsdb.datageneration.FieldType; + +/** + * Provides arbitrary values for different purposes. + */ +public interface Arbitrary { + boolean generateSubObject(); + + boolean generateNestedObject(); + + int childFieldCount(int lowerBound, int upperBound); + + String fieldName(int lengthLowerBound, int lengthUpperBound); + + FieldType fieldType(); + + long longValue(); + + String stringValue(int lengthLowerBound, int lengthUpperBound); +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/arbitrary/RandomBasedArbitrary.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/arbitrary/RandomBasedArbitrary.java new file mode 100644 index 0000000000000..71152191e27f9 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/arbitrary/RandomBasedArbitrary.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.logsdb.datageneration.arbitrary; + +import org.elasticsearch.logsdb.datageneration.FieldType; + +import static org.elasticsearch.test.ESTestCase.randomAlphaOfLengthBetween; +import static org.elasticsearch.test.ESTestCase.randomDouble; +import static org.elasticsearch.test.ESTestCase.randomFrom; +import static org.elasticsearch.test.ESTestCase.randomIntBetween; +import static org.elasticsearch.test.ESTestCase.randomLong; + +public class RandomBasedArbitrary implements Arbitrary { + @Override + public boolean generateSubObject() { + // Using a static 10% change, this is just a chosen value that can be tweaked. + return randomDouble() <= 0.1; + } + + @Override + public boolean generateNestedObject() { + // Using a static 10% change, this is just a chosen value that can be tweaked. + return randomDouble() <= 0.1; + } + + @Override + public int childFieldCount(int lowerBound, int upperBound) { + return randomIntBetween(lowerBound, upperBound); + } + + @Override + public String fieldName(int lengthLowerBound, int lengthUpperBound) { + return randomAlphaOfLengthBetween(lengthLowerBound, lengthUpperBound); + } + + @Override + public FieldType fieldType() { + return randomFrom(FieldType.values()); + } + + @Override + public long longValue() { + return randomLong(); + } + + @Override + public String stringValue(int lengthLowerBound, int lengthUpperBound) { + return randomAlphaOfLengthBetween(lengthLowerBound, lengthLowerBound); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java new file mode 100644 index 0000000000000..b78e1e2dda0d4 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.logsdb.datageneration.fields; + +import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; + +class Context { + private final DataGeneratorSpecification specification; + private final int objectDepth; + private final int nestedFieldsCount; + + Context(DataGeneratorSpecification specification) { + this(specification, 0, 0); + } + + private Context(DataGeneratorSpecification specification, int objectDepth, int nestedFieldsCount) { + this.specification = specification; + this.objectDepth = objectDepth; + this.nestedFieldsCount = nestedFieldsCount; + } + + public DataGeneratorSpecification specification() { + return specification; + } + + public Context subObject() { + return new Context(specification, objectDepth + 1, nestedFieldsCount); + } + + public Context nestedObject() { + return new Context(specification, objectDepth + 1, nestedFieldsCount + 1); + } + + public boolean shouldAddObjectField() { + return specification.arbitrary().generateSubObject() && objectDepth < specification.maxObjectDepth(); + } + + public boolean shouldAddNestedField() { + return specification.arbitrary().generateNestedObject() + && objectDepth < specification.maxObjectDepth() + && nestedFieldsCount < specification.nestedFieldsLimit(); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java new file mode 100644 index 0000000000000..cc1ae57b8996c --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.logsdb.datageneration.fields; + +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.FieldType; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Generic generator for any type of object field (e.g. "object", "nested"). + */ +public class GenericSubObjectFieldDataGenerator { + private final Context context; + + private final List childFields; + + public GenericSubObjectFieldDataGenerator(Context context) { + this.context = context; + + childFields = new ArrayList<>(); + generateChildFields(); + } + + public CheckedConsumer mappingWriter( + CheckedConsumer customMappingParameters + ) { + return b -> { + b.startObject(); + customMappingParameters.accept(b); + + b.startObject("properties"); + for (var childField : childFields) { + b.field(childField.fieldName); + childField.generator.mappingWriter().accept(b); + } + b.endObject(); + + b.endObject(); + }; + } + + public CheckedConsumer fieldValueGenerator() { + return b -> { + b.startObject(); + + for (var childField : childFields) { + b.field(childField.fieldName); + childField.generator.fieldValueGenerator().accept(b); + } + + b.endObject(); + }; + } + + private void generateChildFields() { + var existingFields = new HashSet(); + // no child fields is legal + var childFieldsCount = context.specification().arbitrary().childFieldCount(0, context.specification().maxFieldCountPerLevel()); + + for (int i = 0; i < childFieldsCount; i++) { + var fieldName = generateFieldName(existingFields); + + if (context.shouldAddObjectField()) { + childFields.add(new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject()))); + } else if (context.shouldAddNestedField()) { + childFields.add(new ChildField(fieldName, new NestedFieldDataGenerator(context.nestedObject()))); + } else { + var fieldType = context.specification().arbitrary().fieldType(); + addLeafField(fieldType, fieldName); + } + } + } + + private void addLeafField(FieldType type, String fieldName) { + var generator = switch (type) { + case LONG -> new LongFieldDataGenerator(context.specification().arbitrary()); + case KEYWORD -> new KeywordFieldDataGenerator(context.specification().arbitrary()); + }; + + childFields.add(new ChildField(fieldName, generator)); + } + + private String generateFieldName(Set existingFields) { + var fieldName = context.specification().arbitrary().fieldName(1, 10); + while (existingFields.contains(fieldName)) { + fieldName = context.specification().arbitrary().fieldName(1, 10); + } + existingFields.add(fieldName); + + return fieldName; + } + + private record ChildField(String fieldName, FieldDataGenerator generator) {} +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/KeywordFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/KeywordFieldDataGenerator.java new file mode 100644 index 0000000000000..31a1499ce2799 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/KeywordFieldDataGenerator.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.logsdb.datageneration.fields; + +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.arbitrary.Arbitrary; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; + +public class KeywordFieldDataGenerator implements FieldDataGenerator { + private final Arbitrary arbitrary; + + public KeywordFieldDataGenerator(Arbitrary arbitrary) { + this.arbitrary = arbitrary; + } + + @Override + public CheckedConsumer mappingWriter() { + return b -> b.startObject().field("type", "keyword").endObject(); + } + + @Override + public CheckedConsumer fieldValueGenerator() { + return b -> b.value(arbitrary.stringValue(0, 50)); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/LongFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/LongFieldDataGenerator.java new file mode 100644 index 0000000000000..f8753a7cdd1c6 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/LongFieldDataGenerator.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.logsdb.datageneration.fields; + +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.arbitrary.Arbitrary; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; + +public class LongFieldDataGenerator implements FieldDataGenerator { + private final Arbitrary arbitrary; + + public LongFieldDataGenerator(Arbitrary arbitrary) { + this.arbitrary = arbitrary; + } + + @Override + public CheckedConsumer mappingWriter() { + return b -> b.startObject().field("type", "long").endObject(); + } + + @Override + public CheckedConsumer fieldValueGenerator() { + return b -> b.value(arbitrary.longValue()); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java new file mode 100644 index 0000000000000..acceb3aebe421 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.logsdb.datageneration.fields; + +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; + +public class NestedFieldDataGenerator implements FieldDataGenerator { + private final GenericSubObjectFieldDataGenerator delegate; + + public NestedFieldDataGenerator(Context context) { + this.delegate = new GenericSubObjectFieldDataGenerator(context); + } + + @Override + public CheckedConsumer mappingWriter() { + return delegate.mappingWriter(b -> b.field("type", "nested")); + } + + @Override + public CheckedConsumer fieldValueGenerator() { + return delegate.fieldValueGenerator(); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java new file mode 100644 index 0000000000000..8cbedefe14ae5 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.logsdb.datageneration.fields; + +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; +import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; + +public class ObjectFieldDataGenerator implements FieldDataGenerator { + private final GenericSubObjectFieldDataGenerator delegate; + + public ObjectFieldDataGenerator(DataGeneratorSpecification specification) { + this(new Context(specification)); + } + + ObjectFieldDataGenerator(Context context) { + this.delegate = new GenericSubObjectFieldDataGenerator(context); + } + + @Override + public CheckedConsumer mappingWriter() { + return delegate.mappingWriter(b -> {}); + } + + @Override + public CheckedConsumer fieldValueGenerator() { + return delegate.fieldValueGenerator(); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java index faada33eade83..f0182a4e69898 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java @@ -20,32 +20,41 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.SecureSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException; import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.core.Streams; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; +import org.elasticsearch.repositories.blobstore.RequestedRangeNotSatisfiedException; import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.threadpool.ThreadPool; import java.io.ByteArrayInputStream; +import java.io.EOFException; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; +import java.util.function.Predicate; import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomNonDataPurpose; import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; public abstract class AbstractThirdPartyRepositoryTestCase extends ESSingleNodeTestCase { @@ -309,6 +318,66 @@ public void testReadFromPositionWithLength() { } } + public void testSkipBeyondBlobLengthShouldThrowEOFException() throws IOException { + final var blobName = randomIdentifier(); + final int blobLength = randomIntBetween(100, 2_000); + final var blobBytes = randomBytesReference(blobLength); + + final var repository = getRepository(); + executeOnBlobStore(repository, blobStore -> { + blobStore.writeBlob(randomPurpose(), blobName, blobBytes, true); + return null; + }); + + var blobContainer = repository.blobStore().blobContainer(repository.basePath()); + try (var input = blobContainer.readBlob(randomPurpose(), blobName, 0, blobLength); var output = new BytesStreamOutput()) { + Streams.copy(input, output, false); + expectThrows(EOFException.class, () -> input.skipNBytes(randomLongBetween(1, 1000))); + } + + try (var input = blobContainer.readBlob(randomPurpose(), blobName, 0, blobLength); var output = new BytesStreamOutput()) { + final int capacity = between(1, blobLength); + final ByteBuffer byteBuffer = randomBoolean() ? ByteBuffer.allocate(capacity) : ByteBuffer.allocateDirect(capacity); + Streams.read(input, byteBuffer, capacity); + expectThrows(EOFException.class, () -> input.skipNBytes((blobLength - capacity) + randomLongBetween(1, 1000))); + } + } + + protected void testReadFromPositionLargerThanBlobLength(Predicate responseCodeChecker) { + final var blobName = randomIdentifier(); + final var blobBytes = randomBytesReference(randomIntBetween(100, 2_000)); + + final var repository = getRepository(); + executeOnBlobStore(repository, blobStore -> { + blobStore.writeBlob(randomPurpose(), blobName, blobBytes, true); + return null; + }); + + long position = randomLongBetween(blobBytes.length(), Long.MAX_VALUE - 1L); + long length = randomLongBetween(1L, Long.MAX_VALUE - position); + + var exception = expectThrows(UncategorizedExecutionException.class, () -> readBlob(repository, blobName, position, length)); + assertThat(exception.getCause(), instanceOf(ExecutionException.class)); + assertThat(exception.getCause().getCause(), instanceOf(RequestedRangeNotSatisfiedException.class)); + assertThat( + exception.getCause().getCause().getMessage(), + containsString( + "Requested range [position=" + + position + + ", length=" + + length + + "] cannot be satisfied for [" + + repository.basePath().buildAsString() + + blobName + + ']' + ) + ); + var rangeNotSatisfiedException = (RequestedRangeNotSatisfiedException) exception.getCause().getCause(); + assertThat(rangeNotSatisfiedException.getPosition(), equalTo(position)); + assertThat(rangeNotSatisfiedException.getLength(), equalTo(length)); + assertThat(responseCodeChecker.test(rangeNotSatisfiedException), is(true)); + } + protected static T executeOnBlobStore(BlobStoreRepository repository, CheckedFunction fn) { final var future = new PlainActionFuture(); repository.threadPool().generic().execute(ActionRunnable.supply(future, () -> { diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java index 6951e1941686d..c53d85a043128 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java @@ -11,7 +11,6 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.SetOnce; -import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequestBuilder; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequestBuilder; @@ -292,12 +291,10 @@ protected BlobStore newBlobStore() { } protected BlobStore newBlobStore(String repository) { - final BlobStoreRepository blobStoreRepository = (BlobStoreRepository) internalCluster().getAnyMasterNodeInstance( - RepositoriesService.class - ).repository(repository); - return PlainActionFuture.get( - f -> blobStoreRepository.threadPool().generic().execute(ActionRunnable.supply(f, blobStoreRepository::blobStore)) - ); + return asInstanceOf( + BlobStoreRepository.class, + internalCluster().getAnyMasterNodeInstance(RepositoriesService.class).repository(repository) + ).blobStore(); } public void testSnapshotAndRestore() throws Exception { @@ -610,7 +607,7 @@ public void testDanglingShardLevelBlobCleanup() throws Exception { // Prepare to compute the expected blobs final var shardGeneration = Objects.requireNonNull(getRepositoryData(repo).shardGenerations().getShardGen(indexId, 0)); final var snapBlob = Strings.format(SNAPSHOT_NAME_FORMAT, snapshot2Info.snapshotId().getUUID()); - final var indexBlob = Strings.format(SNAPSHOT_INDEX_NAME_FORMAT, shardGeneration.toBlobNamePart()); + final var indexBlob = Strings.format(SNAPSHOT_INDEX_NAME_FORMAT, shardGeneration.getGenerationUUID()); for (var fileInfos : List.of( // The expected blobs according to the BlobStoreIndexShardSnapshot (snap-UUID.dat) blob diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 3bed333b135fb..f3fc4479a21a4 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -354,6 +354,7 @@ private AggregationContext createAggregationContext( .fielddataBuilder( new FieldDataContext( indexSettings.getIndex().getName(), + indexSettings, context.lookupSupplier(), context.sourcePathsLookup(), context.fielddataOperation() diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index 23ea4cc95fa35..1b49209b49c7f 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -11,7 +11,6 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.get.SnapshotSortKey; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -177,11 +176,7 @@ protected RepositoryData getRepositoryData(String repository) { } public static RepositoryData getRepositoryData(Repository repository) { - return PlainActionFuture.get( - listener -> repository.getRepositoryData(EsExecutors.DIRECT_EXECUTOR_SERVICE, listener), - 10, - TimeUnit.SECONDS - ); + return safeAwait(listener -> repository.getRepositoryData(EsExecutors.DIRECT_EXECUTOR_SERVICE, listener)); } public static long getFailureCount(String repository) { @@ -429,20 +424,11 @@ protected String initWithSnapshotVersion(String repoName, Path repoPath, IndexVe downgradedSnapshotInfo = SnapshotInfo.fromXContentInternal(repoName, parser); } final BlobStoreRepository blobStoreRepository = getRepositoryOnMaster(repoName); - PlainActionFuture.get( - f -> blobStoreRepository.threadPool() - .generic() - .execute( - ActionRunnable.run( - f, - () -> BlobStoreRepository.SNAPSHOT_FORMAT.write( - downgradedSnapshotInfo, - blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath()), - snapshotInfo.snapshotId().getUUID(), - randomBoolean() - ) - ) - ) + BlobStoreRepository.SNAPSHOT_FORMAT.write( + downgradedSnapshotInfo, + blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath()), + snapshotInfo.snapshotId().getUUID(), + randomBoolean() ); final RepositoryMetadata repoMetadata = blobStoreRepository.getMetadata(); @@ -554,15 +540,15 @@ protected void addBwCFailedSnapshot(String repoName, String snapshotName, Mapget( - f -> repo.finalizeSnapshot( + safeAwait( + (ActionListener listener) -> repo.finalizeSnapshot( new FinalizeSnapshotContext( ShardGenerations.EMPTY, getRepositoryData(repoName).getGenId(), state.metadata(), snapshotInfo, SnapshotsService.OLD_SNAPSHOT_FORMAT, - f, + listener, info -> {} ) ) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java index c1491bd673e40..fc4a39acf853b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java @@ -226,8 +226,8 @@ public void onTimeout(TimeValue timeout) { } public static void awaitNoPendingTasks(ClusterService clusterService) { - PlainActionFuture.get( - fut -> clusterService.submitUnbatchedStateUpdateTask( + ESTestCase.safeAwait( + listener -> clusterService.submitUnbatchedStateUpdateTask( "await-queue-empty", new ClusterStateUpdateTask(Priority.LANGUID, TimeValue.timeValueSeconds(10)) { @Override @@ -237,17 +237,15 @@ public ClusterState execute(ClusterState currentState) { @Override public void onFailure(Exception e) { - fut.onFailure(e); + listener.onFailure(e); } @Override public void clusterStateProcessed(ClusterState initialState, ClusterState newState) { - fut.onResponse(null); + listener.onResponse(null); } } - ), - 10, - TimeUnit.SECONDS + ) ); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 7fdc5765a90e8..a538c39704a73 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -133,9 +133,7 @@ public void tearDown() throws Exception { ensureNoInitializingShards(); ensureAllFreeContextActionsAreConsumed(); - SearchService searchService = getInstanceFromNode(SearchService.class); - assertThat(searchService.getActiveContexts(), equalTo(0)); - assertThat(searchService.getOpenScrollContexts(), equalTo(0)); + ensureAllContextsReleased(getInstanceFromNode(SearchService.class)); super.tearDown(); var deleteDataStreamsRequest = new DeleteDataStreamAction.Request("*"); deleteDataStreamsRequest.indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 7295dce7a257a..b5c03d118a43b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -41,10 +41,11 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.RequestBuilder; import org.elasticsearch.action.support.ActionTestUtils; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.SubscribableListener; +import org.elasticsearch.action.support.TestPlainActionFuture; import org.elasticsearch.bootstrap.BootstrapForTesting; import org.elasticsearch.client.internal.Requests; import org.elasticsearch.cluster.ClusterModule; @@ -83,6 +84,7 @@ import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Booleans; +import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.PathUtils; import org.elasticsearch.core.PathUtilsForTesting; @@ -113,6 +115,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.MockSearchService; +import org.elasticsearch.search.SearchService; import org.elasticsearch.test.junit.listeners.LoggingListener; import org.elasticsearch.test.junit.listeners.ReproduceInfoPrinter; import org.elasticsearch.threadpool.ExecutorBuilder; @@ -152,6 +155,7 @@ import java.lang.invoke.MethodHandles; import java.math.BigInteger; import java.net.InetAddress; +import java.net.URI; import java.net.UnknownHostException; import java.nio.file.Path; import java.security.NoSuchAlgorithmException; @@ -1474,10 +1478,24 @@ public Path getDataPath(String relativePath) { // we override LTC behavior here: wrap even resources with mockfilesystems, // because some code is buggy when it comes to multiple nio.2 filesystems // (e.g. FileSystemUtils, and likely some tests) + return getResourceDataPath(getClass(), relativePath); + } + + public static Path getResourceDataPath(Class clazz, String relativePath) { + final var resource = Objects.requireNonNullElseGet( + clazz.getResource(relativePath), + () -> fail(null, "resource not found: [%s][%s]", clazz.getCanonicalName(), relativePath) + ); + final URI uri; try { - return PathUtils.get(getClass().getResource(relativePath).toURI()).toAbsolutePath().normalize(); + uri = resource.toURI(); } catch (Exception e) { - throw new RuntimeException("resource not found: " + relativePath, e); + return fail(null, "resource URI not found: [%s][%s]", clazz.getCanonicalName(), relativePath); + } + try { + return PathUtils.get(uri).toAbsolutePath().normalize(); + } catch (Exception e) { + return fail(e, "resource path not found: %s", uri); } } @@ -2285,11 +2303,22 @@ public static void safeAcquire(int permits, Semaphore semaphore) { * @return The value with which the {@code listener} was completed. */ public static T safeAwait(SubscribableListener listener) { - final var future = new PlainActionFuture(); + final var future = new TestPlainActionFuture(); listener.addListener(future); return safeGet(future); } + /** + * Call an async action (a {@link Consumer} of an {@link ActionListener}), wait for it to complete the listener, and then return the + * result. Preserves the thread's interrupt status flag and converts all exceptions into an {@link AssertionError} to trigger a test + * failure. + * + * @return The value with which the consumed listener was completed. + */ + public static T safeAwait(CheckedConsumer, ?> consumer) { + return safeAwait(SubscribableListener.newForked(consumer)); + } + /** * Wait for the successful completion of the given {@link Future}, with a timeout of {@link #SAFE_AWAIT_TIMEOUT}, preserving the * thread's interrupt status flag and converting all exceptions into an {@link AssertionError} to trigger a test failure. @@ -2331,11 +2360,31 @@ public static T safeGet(CheckedSupplier supplier) { * @return The exception with which the {@code listener} was completed exceptionally. */ public static Exception safeAwaitFailure(SubscribableListener listener) { - return safeAwait( - SubscribableListener.newForked( - exceptionListener -> listener.addListener(ActionTestUtils.assertNoSuccessListener(exceptionListener::onResponse)) - ) - ); + return safeAwait(exceptionListener -> listener.addListener(ActionTestUtils.assertNoSuccessListener(exceptionListener::onResponse))); + } + + /** + * Wait for the exceptional completion of the given async action, with a timeout of {@link #SAFE_AWAIT_TIMEOUT}, + * preserving the thread's interrupt status flag and converting a successful completion, interrupt or timeout into an {@link + * AssertionError} to trigger a test failure. + * + * @return The exception with which the {@code listener} was completed exceptionally. + */ + public static Exception safeAwaitFailure(Consumer> consumer) { + return safeAwait(exceptionListener -> consumer.accept(ActionTestUtils.assertNoSuccessListener(exceptionListener::onResponse))); + } + + /** + * Wait for the exceptional completion of the given async action, with a timeout of {@link #SAFE_AWAIT_TIMEOUT}, + * preserving the thread's interrupt status flag and converting a successful completion, interrupt or timeout into an {@link + * AssertionError} to trigger a test failure. + * + * @param responseType Class of listener response type, to aid type inference but otherwise ignored. + * + * @return The exception with which the {@code listener} was completed exceptionally. + */ + public static Exception safeAwaitFailure(@SuppressWarnings("unused") Class responseType, Consumer> consumer) { + return safeAwaitFailure(consumer); } /** @@ -2482,4 +2531,15 @@ public static void runInParallel(int numberOfTasks, IntConsumer taskFactory) thr throw new AssertionError(e); } } + + public static void ensureAllContextsReleased(SearchService searchService) { + try { + assertBusy(() -> { + assertThat(searchService.getActiveContexts(), equalTo(0)); + assertThat(searchService.getOpenScrollContexts(), equalTo(0)); + }); + } catch (Exception e) { + throw new AssertionError("Failed to verify search contexts", e); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/EnumSerializationTestUtils.java b/test/framework/src/main/java/org/elasticsearch/test/EnumSerializationTestUtils.java new file mode 100644 index 0000000000000..69d08c69f5e71 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/EnumSerializationTestUtils.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.test; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +import static org.elasticsearch.test.ESTestCase.assertEquals; + +/** + * Enum serialization via {@link StreamOutput#writeEnum} and {@link StreamInput#readEnum(Class)} uses the enum value's ordinal on the wire. + * Reordering the values in an enum, or adding a new value, will change the ordinals and is therefore a wire protocol change, but it's easy + * to miss this fact in the context of a larger commit. To protect against this trap, any enums that we send over the wire should have a + * test that uses {@link #assertEnumSerialization} to assert a fixed mapping between ordinals and values. That way, a change to the ordinals + * will require a test change, and thus some thought about BwC. + */ +public class EnumSerializationTestUtils { + private EnumSerializationTestUtils() {/* no instances */} + + /** + * Assert that the enum constants of the given class are exactly the ones passed in explicitly as arguments, which fixes its wire + * protocol when using {@link StreamOutput#writeEnum} and {@link StreamInput#readEnum(Class)}. + * + * @param value0 The enum constant with ordinal {@code 0}, passed as a separate argument to avoid prevent callers just lazily using + * {@code EnumClass.values()} to pass the values of the enum, which would negate the point of this test. + * @param values The remaining enum constants, in ordinal order. + */ + @SafeVarargs + public static > void assertEnumSerialization(Class clazz, E value0, E... values) { + final var enumConstants = clazz.getEnumConstants(); + assertEquals(clazz.getCanonicalName(), enumConstants.length, values.length + 1); + for (var ordinal = 0; ordinal < values.length + 1; ordinal++) { + final var enumValue = ordinal == 0 ? value0 : values[ordinal - 1]; + final var description = clazz.getCanonicalName() + "[" + ordinal + "]"; + assertEquals(description, enumConstants[ordinal], enumValue); + try (var out = new BytesStreamOutput(1)) { + out.writeEnum(enumValue); + try (var in = out.bytes().streamInput()) { + assertEquals(description, ordinal, in.readVInt()); + } + } catch (IOException e) { + ESTestCase.fail(e); + } + try (var out = new BytesStreamOutput(1)) { + out.writeVInt(ordinal); + try (var in = out.bytes().streamInput()) { + assertEquals(description, enumValue, in.readEnum(clazz)); + } + } catch (IOException e) { + ESTestCase.fail(e); + } + } + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index af37fb6feefbd..0b69245177c7a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -2545,15 +2545,7 @@ public void assertRequestsFinished() { private void assertSearchContextsReleased() { for (NodeAndClient nodeAndClient : nodes.values()) { - SearchService searchService = getInstance(SearchService.class, nodeAndClient.name); - try { - assertBusy(() -> { - assertThat(searchService.getActiveContexts(), equalTo(0)); - assertThat(searchService.getOpenScrollContexts(), equalTo(0)); - }); - } catch (Exception e) { - throw new AssertionError("Failed to verify search contexts", e); - } + ESTestCase.ensureAllContextsReleased(getInstance(SearchService.class, nodeAndClient.name)); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/TaskAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/TaskAssertions.java index b4ecc36fc5b97..d0862c91537cb 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TaskAssertions.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TaskAssertions.java @@ -10,16 +10,19 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.core.Nullable; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskInfo; -import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.transport.TransportService; +import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static junit.framework.TestCase.assertFalse; import static junit.framework.TestCase.assertTrue; import static junit.framework.TestCase.fail; import static org.elasticsearch.test.ESIntegTestCase.client; @@ -59,30 +62,28 @@ private static void awaitTaskWithPrefix(String actionPrefix, Iterable checking that all tasks with prefix {} are marked as cancelled", actionPrefix); assertBusy(() -> { - boolean foundTask = false; + var tasks = new ArrayList(); for (TransportService transportService : internalCluster().getInstances(TransportService.class)) { - final TaskManager taskManager = transportService.getTaskManager(); + var taskManager = transportService.getTaskManager(); assertTrue(taskManager.assertCancellableTaskConsistency()); - for (CancellableTask cancellableTask : taskManager.getCancellableTasks().values()) { - if (cancellableTask.getAction().startsWith(actionPrefix)) { - logger.trace("--> found task with prefix [{}]: [{}]", actionPrefix, cancellableTask); - foundTask = true; - assertTrue( - "task " + cancellableTask.getId() + "/" + cancellableTask.getAction() + " not cancelled", - cancellableTask.isCancelled() - ); - logger.trace("--> Task with prefix [{}] is marked as cancelled: [{}]", actionPrefix, cancellableTask); - } - } + taskManager.getCancellableTasks().values().stream().filter(t -> t.getAction().startsWith(actionPrefix)).forEach(tasks::add); } - assertTrue("found no cancellable tasks", foundTask); + assertFalse("no tasks found for action: " + actionPrefix, tasks.isEmpty()); + assertTrue( + tasks.toString(), + tasks.stream().allMatch(t -> t.isCancelled() && Objects.equals(t.getHeader(Task.X_OPAQUE_ID_HTTP_HEADER), opaqueId)) + ); }, 30, TimeUnit.SECONDS); } + public static void assertAllCancellableTasksAreCancelled(String actionPrefix) throws Exception { + assertAllCancellableTasksAreCancelled(actionPrefix, null); + } + public static void assertAllTasksHaveFinished(String actionPrefix) throws Exception { logger.info("--> checking that all tasks with prefix {} have finished", actionPrefix); assertBusy(() -> { diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestTrustStore.java b/test/framework/src/main/java/org/elasticsearch/test/TestTrustStore.java new file mode 100644 index 0000000000000..23d178ea0c672 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/TestTrustStore.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.test; + +import org.elasticsearch.common.CheckedSupplier; +import org.elasticsearch.common.ssl.KeyStoreUtil; +import org.junit.rules.ExternalResource; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.cert.Certificate; +import java.security.cert.CertificateFactory; +import java.util.List; +import java.util.Objects; + +import static org.apache.lucene.tests.util.LuceneTestCase.createTempDir; + +public class TestTrustStore extends ExternalResource { + + private final CheckedSupplier pemStreamSupplier; + + public TestTrustStore(CheckedSupplier pemStreamSupplier) { + this.pemStreamSupplier = pemStreamSupplier; + } + + private Path trustStorePath; + + public Path getTrustStorePath() { + return Objects.requireNonNullElseGet(trustStorePath, () -> ESTestCase.fail(null, "trust store not created")); + } + + @Override + protected void before() { + final var tmpDir = createTempDir(); + final var tmpTrustStorePath = tmpDir.resolve("trust-store.jks"); + try (var pemStream = pemStreamSupplier.get(); var jksStream = Files.newOutputStream(tmpTrustStorePath)) { + final List certificates = CertificateFactory.getInstance("X.509") + .generateCertificates(pemStream) + .stream() + .map(i -> (Certificate) i) + .toList(); + final var trustStore = KeyStoreUtil.buildTrustStore(certificates); + trustStore.store(jksStream, null); + trustStorePath = tmpTrustStorePath; + } catch (Exception e) { + throw new AssertionError("unexpected", e); + } + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java index 5844dcbd66471..a591fc86979c3 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java +++ b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java @@ -67,7 +67,7 @@ public void testFailure(Failure failure) throws Exception { boolean isBwcTest = Boolean.parseBoolean(System.getProperty("tests.bwc", "false")); // append Gradle test runner test filter string - b.append("'" + task + "'"); + b.append("\"" + task + "\""); if (isBwcTest) { // Use "legacy" method for bwc tests so that it applies globally to all upstream bwc test tasks b.append(" -Dtests.class=\""); diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java index c0ce4061f2459..e264e25641795 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java @@ -1868,8 +1868,8 @@ public void handleException(TransportException exp) { assertBusy(() -> assertFalse(serviceB.nodeConnected(nodeA))); // now try to connect again and see that it fails - expectThrows(ConnectTransportException.class, () -> connectToNode(serviceB, nodeA)); - expectThrows(ConnectTransportException.class, () -> openConnection(serviceB, nodeA, TestProfiles.LIGHT_PROFILE)); + assertNotNull(connectToNodeExpectFailure(serviceB, nodeA, null)); + assertNotNull(openConnectionExpectFailure(serviceB, nodeA, TestProfiles.LIGHT_PROFILE)); } public void testMockUnresponsiveRule() throws InterruptedException { @@ -1916,11 +1916,9 @@ public void handleException(TransportException exp) { ); assertThat(expectThrows(ExecutionException.class, res::get).getCause(), instanceOf(ReceiveTimeoutTransportException.class)); - expectThrows(ConnectTransportException.class, () -> { - serviceB.disconnectFromNode(nodeA); - connectToNode(serviceB, nodeA); - }); - expectThrows(ConnectTransportException.class, () -> openConnection(serviceB, nodeA, TestProfiles.LIGHT_PROFILE)); + serviceB.disconnectFromNode(nodeA); + assertNotNull(connectToNodeExpectFailure(serviceB, nodeA, null)); + assertNotNull(openConnectionExpectFailure(serviceB, nodeA, TestProfiles.LIGHT_PROFILE)); } public void testHostOnMessages() throws InterruptedException { @@ -2342,7 +2340,7 @@ public void testHandshakeWithIncompatVersion() { TransportRequestOptions.Type.REG, TransportRequestOptions.Type.STATE ); - expectThrows(ConnectTransportException.class, () -> openConnection(serviceA, node, builder.build())); + assertNotNull(openConnectionExpectFailure(serviceA, node, builder.build())); } } @@ -2448,10 +2446,7 @@ public void testTcpHandshakeTimeout() throws IOException { TransportRequestOptions.Type.STATE ); builder.setHandshakeTimeout(TimeValue.timeValueMillis(1)); - ConnectTransportException ex = expectThrows( - ConnectTransportException.class, - () -> connectToNode(serviceA, dummy, builder.build()) - ); + ConnectTransportException ex = connectToNodeExpectFailure(serviceA, dummy, builder.build()); assertEquals("[][" + dummy.getAddress() + "] handshake_timeout[1ms]", ex.getMessage()); } } @@ -2488,10 +2483,7 @@ public void run() { TransportRequestOptions.Type.STATE ); builder.setHandshakeTimeout(TimeValue.timeValueHours(1)); - ConnectTransportException ex = expectThrows( - ConnectTransportException.class, - () -> connectToNode(serviceA, dummy, builder.build()) - ); + ConnectTransportException ex = connectToNodeExpectFailure(serviceA, dummy, builder.build()); assertEquals("[][" + dummy.getAddress() + "] general node connection failure", ex.getMessage()); assertThat(ex.getCause().getMessage(), startsWith("handshake failed")); t.join(); @@ -3160,10 +3152,7 @@ public void onConnectionClosed(Transport.Connection connection) { TransportRequestOptions.Type.REG, TransportRequestOptions.Type.STATE ); - final ConnectTransportException e = expectThrows( - ConnectTransportException.class, - () -> openConnection(service, nodeA, builder.build()) - ); + final ConnectTransportException e = openConnectionExpectFailure(service, nodeA, builder.build()); assertThat(e, hasToString(containsString(("a channel closed while connecting")))); assertTrue(connectionClosedListenerCalled.get()); } @@ -3211,36 +3200,32 @@ public void testChannelToString() { channel.sendResponse(TransportResponse.Empty.INSTANCE); }); - PlainActionFuture.get( - f -> submitRequest( + safeAwait( + listener -> submitRequest( serviceA, serviceA.getLocalNode(), ACTION, new EmptyRequest(), new ActionListenerResponseHandler<>( - f, + listener, ignored -> TransportResponse.Empty.INSTANCE, TransportResponseHandler.TRANSPORT_WORKER ) - ), - 10, - TimeUnit.SECONDS + ) ); - PlainActionFuture.get( - f -> submitRequest( + safeAwait( + listener -> submitRequest( serviceA, serviceB.getLocalNode(), ACTION, new EmptyRequest(), new ActionListenerResponseHandler<>( - f, + listener, ignored -> TransportResponse.Empty.INSTANCE, TransportResponseHandler.TRANSPORT_WORKER ) - ), - 10, - TimeUnit.SECONDS + ) ); } @@ -3309,16 +3294,14 @@ public void writeTo(StreamOutput out) throws IOException { for (int iteration = 1; iteration <= 5; iteration++) { assertEquals( responseSize, - PlainActionFuture.get( - f -> submitRequest( + safeAwait( + (ActionListener listener) -> submitRequest( serviceA, serviceB.getLocalNode(), ACTION, new Request(requestSize), - new ActionListenerResponseHandler<>(f, Response::new, TransportResponseHandler.TRANSPORT_WORKER) - ), - 10, - TimeUnit.SECONDS + new ActionListenerResponseHandler<>(listener, Response::new, TransportResponseHandler.TRANSPORT_WORKER) + ) ).refSize ); @@ -3512,7 +3495,21 @@ public static void connectToNode(TransportService service, DiscoveryNode node) t * @param connectionProfile the connection profile to use when connecting to this node */ public static void connectToNode(TransportService service, DiscoveryNode node, ConnectionProfile connectionProfile) { - UnsafePlainActionFuture.get(fut -> service.connectToNode(node, connectionProfile, fut.map(x -> null)), ThreadPool.Names.GENERIC); + safeAwait(listener -> service.connectToNode(node, connectionProfile, listener.map(ignored -> null))); + } + + /** + * Attempt to connect to the specified node, but assert that this fails and return the resulting exception. + */ + public static ConnectTransportException connectToNodeExpectFailure( + TransportService service, + DiscoveryNode node, + ConnectionProfile connectionProfile + ) { + return asInstanceOf( + ConnectTransportException.class, + safeAwaitFailure(Releasable.class, listener -> service.connectToNode(node, connectionProfile, listener)) + ); } /** @@ -3523,7 +3520,21 @@ public static void connectToNode(TransportService service, DiscoveryNode node, C * @param connectionProfile the connection profile to use */ public static Transport.Connection openConnection(TransportService service, DiscoveryNode node, ConnectionProfile connectionProfile) { - return PlainActionFuture.get(fut -> service.openConnection(node, connectionProfile, fut)); + return safeAwait(listener -> service.openConnection(node, connectionProfile, listener)); + } + + /** + * Attempt to connect to the specified node, but assert that this fails and return the resulting exception. + */ + public static ConnectTransportException openConnectionExpectFailure( + TransportService service, + DiscoveryNode node, + ConnectionProfile connectionProfile + ) { + return asInstanceOf( + ConnectTransportException.class, + safeAwaitFailure(Transport.Connection.class, listener -> service.openConnection(node, connectionProfile, listener)) + ); } public static Future submitRequest( diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSnapshotTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSnapshotTests.java new file mode 100644 index 0000000000000..868c8c749ea11 --- /dev/null +++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSnapshotTests.java @@ -0,0 +1,157 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.logsdb.datageneration; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.logsdb.datageneration.arbitrary.Arbitrary; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; + +public class DataGeneratorSnapshotTests extends ESTestCase { + public void testSnapshot() throws Exception { + var dataGenerator = new DataGenerator( + DataGeneratorSpecification.builder() + .withArbitrary(new TestArbitrary()) + .withMaxFieldCountPerLevel(5) + .withMaxObjectDepth(2) + .build() + ); + + var mapping = XContentBuilder.builder(XContentType.JSON.xContent()).prettyPrint(); + dataGenerator.writeMapping(mapping); + + var document = XContentBuilder.builder(XContentType.JSON.xContent()).prettyPrint(); + dataGenerator.generateDocument(document); + + var expectedMapping = """ + { + "_doc" : { + "properties" : { + "f1" : { + "properties" : { + "f2" : { + "properties" : { + "f3" : { + "type" : "keyword" + }, + "f4" : { + "type" : "long" + } + } + }, + "f5" : { + "properties" : { + "f6" : { + "type" : "keyword" + }, + "f7" : { + "type" : "long" + } + } + } + } + }, + "f8" : { + "type" : "nested", + "properties" : { + "f9" : { + "type" : "nested", + "properties" : { + "f10" : { + "type" : "keyword" + }, + "f11" : { + "type" : "long" + } + } + }, + "f12" : { + "type" : "keyword" + } + } + } + } + } + }"""; + + var expectedDocument = """ + { + "f1" : { + "f2" : { + "f3" : "string1", + "f4" : 0 + }, + "f5" : { + "f6" : "string2", + "f7" : 1 + } + }, + "f8" : { + "f9" : { + "f10" : "string3", + "f11" : 2 + }, + "f12" : "string4" + } + }"""; + + assertEquals(expectedMapping, Strings.toString(mapping)); + assertEquals(expectedDocument, Strings.toString(document)); + } + + private class TestArbitrary implements Arbitrary { + private int generatedFields = 0; + private FieldType fieldType = FieldType.KEYWORD; + private long longValue = 0; + private long generatedStringValues = 0; + + @Override + public boolean generateSubObject() { + return generatedFields < 6; + } + + @Override + public boolean generateNestedObject() { + return generatedFields > 6 && generatedFields < 12; + } + + @Override + public int childFieldCount(int lowerBound, int upperBound) { + assert lowerBound < 2 && upperBound > 2; + return 2; + } + + @Override + public String fieldName(int lengthLowerBound, int lengthUpperBound) { + return "f" + (generatedFields++ + 1); + } + + @Override + public FieldType fieldType() { + if (fieldType == FieldType.KEYWORD) { + fieldType = FieldType.LONG; + return FieldType.KEYWORD; + } + + fieldType = FieldType.KEYWORD; + return FieldType.LONG; + } + + @Override + public long longValue() { + return longValue++; + } + + @Override + public String stringValue(int lengthLowerBound, int lengthUpperBound) { + return "string" + (generatedStringValues++ + 1); + } + }; +} diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorTests.java new file mode 100644 index 0000000000000..cd8b2424ac5ae --- /dev/null +++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorTests.java @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.logsdb.datageneration; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.logsdb.datageneration.arbitrary.Arbitrary; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; + +public class DataGeneratorTests extends ESTestCase { + public void testDataGeneratorSanity() throws IOException { + var dataGenerator = new DataGenerator(DataGeneratorSpecification.buildDefault()); + + var mapping = XContentBuilder.builder(XContentType.JSON.xContent()); + dataGenerator.writeMapping(mapping); + + for (int i = 0; i < 1000; i++) { + var document = XContentBuilder.builder(XContentType.JSON.xContent()); + dataGenerator.generateDocument(document); + } + } + + public void testDataGeneratorProducesValidMappingAndDocument() throws IOException { + // Make sure objects, nested objects and all field types are covered. + var testArbitrary = new Arbitrary() { + private boolean subObjectCovered = false; + private boolean nestedCovered = false; + private int generatedFields = 0; + + @Override + public boolean generateSubObject() { + if (subObjectCovered == false) { + subObjectCovered = true; + return true; + } + + return false; + } + + @Override + public boolean generateNestedObject() { + if (nestedCovered == false) { + nestedCovered = true; + return true; + } + + return false; + } + + @Override + public int childFieldCount(int lowerBound, int upperBound) { + // Make sure to generate enough fields to go through all field types. + return 20; + } + + @Override + public String fieldName(int lengthLowerBound, int lengthUpperBound) { + return "f" + generatedFields++; + } + + @Override + public FieldType fieldType() { + return FieldType.values()[generatedFields % FieldType.values().length]; + } + + @Override + public long longValue() { + return randomLong(); + } + + @Override + public String stringValue(int lengthLowerBound, int lengthUpperBound) { + return randomAlphaOfLengthBetween(lengthLowerBound, lengthUpperBound); + } + }; + + var dataGenerator = new DataGenerator(DataGeneratorSpecification.builder().withArbitrary(testArbitrary).build()); + + var mapping = XContentBuilder.builder(XContentType.JSON.xContent()); + dataGenerator.writeMapping(mapping); + + var mappingService = new MapperServiceTestCase() { + }.createMapperService(mapping); + + var document = XContentBuilder.builder(XContentType.JSON.xContent()); + dataGenerator.generateDocument(document); + + mappingService.documentMapper().parse(new SourceToParse("1", BytesReference.bytes(document), XContentType.JSON)); + } + + public void testDataGeneratorStressTest() throws IOException { + // Let's generate 1000000 fields to test an extreme case (2 levels of objects + 1 leaf level with 100 fields per object). + var arbitrary = new Arbitrary() { + private int generatedFields = 0; + + @Override + public boolean generateSubObject() { + return true; + } + + @Override + public boolean generateNestedObject() { + return false; + } + + @Override + public int childFieldCount(int lowerBound, int upperBound) { + return upperBound; + } + + @Override + public String fieldName(int lengthLowerBound, int lengthUpperBound) { + return "f" + generatedFields++; + } + + @Override + public FieldType fieldType() { + return FieldType.LONG; + } + + @Override + public long longValue() { + return 0; + } + + @Override + public String stringValue(int lengthLowerBound, int lengthUpperBound) { + return ""; + } + }; + var dataGenerator = new DataGenerator( + DataGeneratorSpecification.builder().withArbitrary(arbitrary).withMaxFieldCountPerLevel(100).withMaxObjectDepth(2).build() + ); + + var mapping = XContentBuilder.builder(XContentType.JSON.xContent()); + dataGenerator.writeMapping(mapping); + + var document = XContentBuilder.builder(XContentType.JSON.xContent()); + dataGenerator.generateDocument(document); + } +} diff --git a/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java b/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java index 9582d28327122..f79dbbd1a9b35 100644 --- a/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java +++ b/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java @@ -593,32 +593,28 @@ public void testBrokenLinkFailsToConnect() { disconnectedLinks.add(Tuple.tuple(node1, node2)); assertThat( - expectThrows(ConnectTransportException.class, () -> AbstractSimpleTransportTestCase.connectToNode(service1, node2)) - .getMessage(), + AbstractSimpleTransportTestCase.connectToNodeExpectFailure(service1, node2, null).getMessage(), endsWith("is [DISCONNECTED] not [CONNECTED]") ); disconnectedLinks.clear(); blackholedLinks.add(Tuple.tuple(node1, node2)); assertThat( - expectThrows(ConnectTransportException.class, () -> AbstractSimpleTransportTestCase.connectToNode(service1, node2)) - .getMessage(), + AbstractSimpleTransportTestCase.connectToNodeExpectFailure(service1, node2, null).getMessage(), endsWith("is [BLACK_HOLE] not [CONNECTED]") ); blackholedLinks.clear(); blackholedRequestLinks.add(Tuple.tuple(node1, node2)); assertThat( - expectThrows(ConnectTransportException.class, () -> AbstractSimpleTransportTestCase.connectToNode(service1, node2)) - .getMessage(), + AbstractSimpleTransportTestCase.connectToNodeExpectFailure(service1, node2, null).getMessage(), endsWith("is [BLACK_HOLE_REQUESTS_ONLY] not [CONNECTED]") ); blackholedRequestLinks.clear(); final DiscoveryNode node3 = DiscoveryNodeUtils.create("node3"); assertThat( - expectThrows(ConnectTransportException.class, () -> AbstractSimpleTransportTestCase.connectToNode(service1, node3)) - .getMessage(), + AbstractSimpleTransportTestCase.connectToNodeExpectFailure(service1, node3, null).getMessage(), endsWith("does not exist") ); } diff --git a/test/immutable-collections-patch/build.gradle b/test/immutable-collections-patch/build.gradle index 2d42215b3e02c..c3354e189847d 100644 --- a/test/immutable-collections-patch/build.gradle +++ b/test/immutable-collections-patch/build.gradle @@ -35,7 +35,7 @@ generatePatch.configure { executable = "${BuildParams.runtimeJavaHome}/bin/java" + (OS.current() == OS.WINDOWS ? '.exe' : '') } else { javaLauncher = javaToolchains.launcherFor { - languageVersion = JavaLanguageVersion.of(BuildParams.runtimeJavaVersion.majorVersion) + languageVersion = JavaLanguageVersion.of(VersionProperties.bundledJdkMajorVersion) vendor = VersionProperties.bundledJdkVendor == "openjdk" ? JvmVendorSpec.ORACLE : JvmVendorSpec.matching(VersionProperties.bundledJdkVendor) diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoService.java index a14c49b4e5e21..14364cf189072 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoService.java @@ -125,7 +125,7 @@ private void sendToMissingNodes(Function nodeLookup, Set< }; final NodesStatsRequest nodesStatsRequest = new NodesStatsRequest( missingNodes.stream().map(DiscoveryNode::getId).toArray(String[]::new) - ).clear().addMetric(NodesStatsRequestParameters.Metric.OS.metricName()).timeout(fetchTimeout); + ).clear().addMetric(NodesStatsRequestParameters.Metric.OS).timeout(fetchTimeout); nodesStatsRequest.setIncludeShardsStats(false); client.admin() .cluster() diff --git a/x-pack/plugin/blob-cache/build.gradle b/x-pack/plugin/blob-cache/build.gradle index b938e38a152ae..da9c48438af28 100644 --- a/x-pack/plugin/blob-cache/build.gradle +++ b/x-pack/plugin/blob-cache/build.gradle @@ -16,5 +16,5 @@ esplugin { } dependencies { - compileOnly project(path: ':libs:elasticsearch-preallocate') + compileOnly project(path: ':libs:elasticsearch-native') } diff --git a/x-pack/plugin/blob-cache/src/main/java/module-info.java b/x-pack/plugin/blob-cache/src/main/java/module-info.java index 5d895401c273d..23c389b8cb353 100644 --- a/x-pack/plugin/blob-cache/src/main/java/module-info.java +++ b/x-pack/plugin/blob-cache/src/main/java/module-info.java @@ -8,7 +8,7 @@ module org.elasticsearch.blobcache { requires org.elasticsearch.base; requires org.elasticsearch.server; - requires org.elasticsearch.preallocate; + requires org.elasticsearch.nativeaccess; requires org.apache.logging.log4j; requires org.apache.lucene.core; diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java index ac22d22d5affb..9cb83e35b63d6 100644 --- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java +++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java @@ -31,7 +31,9 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.Assertions; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; @@ -41,6 +43,7 @@ import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; +import java.io.InputStream; import java.io.UncheckedIOException; import java.lang.invoke.MethodHandles; import java.lang.invoke.VarHandle; @@ -643,9 +646,10 @@ private RangeMissingHandler writerWithOffset(RangeMissingHandler writer, int wri // no need to allocate a new capturing lambda if the offset isn't adjusted return writer; } - return (channel, channelPos, relativePos, len, progressUpdater) -> writer.fillCacheRange( + return (channel, channelPos, streamFactory, relativePos, len, progressUpdater) -> writer.fillCacheRange( channel, channelPos, + streamFactory, relativePos - writeOffset, len, progressUpdater @@ -923,9 +927,10 @@ void populate( return; } try (var gapsListener = new RefCountingListener(listener.map(unused -> true))) { + assert writer.sharedInputStreamFactory(gaps) == null; for (SparseFileTracker.Gap gap : gaps) { executor.execute( - fillGapRunnable(gap, writer, ActionListener.releaseAfter(gapsListener.acquire(), refs.acquire())) + fillGapRunnable(gap, writer, null, ActionListener.releaseAfter(gapsListener.acquire(), refs.acquire())) ); } } @@ -968,8 +973,30 @@ void populateAndRead( ); if (gaps.isEmpty() == false) { - for (SparseFileTracker.Gap gap : gaps) { - executor.execute(fillGapRunnable(gap, writer, refs.acquireListener())); + final SourceInputStreamFactory streamFactory = writer.sharedInputStreamFactory(gaps); + logger.trace( + () -> Strings.format( + "fill gaps %s %s shared input stream factory", + gaps, + (streamFactory == null ? "without" : "with"), + (streamFactory == null ? "" : " " + streamFactory) + ) + ); + if (streamFactory == null) { + for (SparseFileTracker.Gap gap : gaps) { + executor.execute(fillGapRunnable(gap, writer, null, refs.acquireListener())); + } + } else { + final List gapFillingTasks = gaps.stream() + .map(gap -> fillGapRunnable(gap, writer, streamFactory, refs.acquireListener())) + .toList(); + executor.execute(() -> { + try (streamFactory) { + // Fill the gaps in order. If a gap fails to fill for whatever reason, the task for filling the next + // gap will still be executed. + gapFillingTasks.forEach(Runnable::run); + } + }); } } } @@ -978,7 +1005,12 @@ void populateAndRead( } } - private AbstractRunnable fillGapRunnable(SparseFileTracker.Gap gap, RangeMissingHandler writer, ActionListener listener) { + private AbstractRunnable fillGapRunnable( + SparseFileTracker.Gap gap, + RangeMissingHandler writer, + @Nullable SourceInputStreamFactory streamFactory, + ActionListener listener + ) { return ActionRunnable.run(listener.delegateResponse((l, e) -> failGapAndListener(gap, l, e)), () -> { var ioRef = io; assert regionOwners.get(ioRef) == CacheFileRegion.this; @@ -987,6 +1019,7 @@ private AbstractRunnable fillGapRunnable(SparseFileTracker.Gap gap, RangeMissing writer.fillCacheRange( ioRef, start, + streamFactory, start, Math.toIntExact(gap.end() - start), progress -> gap.onProgress(start + progress) @@ -1072,16 +1105,21 @@ public int populateAndRead( // We are interested in the total time that the system spends when fetching a result (including time spent queuing), so we start // our measurement here. final long startTime = relativeTimeInNanosSupplier.getAsLong(); - RangeMissingHandler writerInstrumentationDecorator = ( - SharedBytes.IO channel, - int channelPos, - int relativePos, - int length, - IntConsumer progressUpdater) -> { - writer.fillCacheRange(channel, channelPos, relativePos, length, progressUpdater); - var elapsedTime = TimeUnit.NANOSECONDS.toMicros(relativeTimeInNanosSupplier.getAsLong() - startTime); - SharedBlobCacheService.this.blobCacheMetrics.getCacheMissLoadTimes().record(elapsedTime); - SharedBlobCacheService.this.blobCacheMetrics.getCacheMissCounter().increment(); + RangeMissingHandler writerInstrumentationDecorator = new DelegatingRangeMissingHandler(writer) { + @Override + public void fillCacheRange( + SharedBytes.IO channel, + int channelPos, + SourceInputStreamFactory streamFactory, + int relativePos, + int length, + IntConsumer progressUpdater + ) throws IOException { + writer.fillCacheRange(channel, channelPos, streamFactory, relativePos, length, progressUpdater); + var elapsedTime = TimeUnit.NANOSECONDS.toMicros(relativeTimeInNanosSupplier.getAsLong() - startTime); + SharedBlobCacheService.this.blobCacheMetrics.getCacheMissLoadTimes().record(elapsedTime); + SharedBlobCacheService.this.blobCacheMetrics.getCacheMissCounter().increment(); + } }; if (rangeToRead.isEmpty()) { // nothing to read, skip @@ -1165,20 +1203,36 @@ private RangeMissingHandler writerWithOffset(RangeMissingHandler writer, CacheFi // no need to allocate a new capturing lambda if the offset isn't adjusted adjustedWriter = writer; } else { - adjustedWriter = (channel, channelPos, relativePos, len, progressUpdater) -> writer.fillCacheRange( - channel, - channelPos, - relativePos - writeOffset, - len, - progressUpdater - ); + adjustedWriter = new DelegatingRangeMissingHandler(writer) { + @Override + public void fillCacheRange( + SharedBytes.IO channel, + int channelPos, + SourceInputStreamFactory streamFactory, + int relativePos, + int len, + IntConsumer progressUpdater + ) throws IOException { + delegate.fillCacheRange(channel, channelPos, streamFactory, relativePos - writeOffset, len, progressUpdater); + } + }; } if (Assertions.ENABLED) { - return (channel, channelPos, relativePos, len, progressUpdater) -> { - assert assertValidRegionAndLength(fileRegion, channelPos, len); - adjustedWriter.fillCacheRange(channel, channelPos, relativePos, len, progressUpdater); - assert regionOwners.get(fileRegion.io) == fileRegion - : "File chunk [" + fileRegion.regionKey + "] no longer owns IO [" + fileRegion.io + "]"; + return new DelegatingRangeMissingHandler(adjustedWriter) { + @Override + public void fillCacheRange( + SharedBytes.IO channel, + int channelPos, + SourceInputStreamFactory streamFactory, + int relativePos, + int len, + IntConsumer progressUpdater + ) throws IOException { + assert assertValidRegionAndLength(fileRegion, channelPos, len); + delegate.fillCacheRange(channel, channelPos, streamFactory, relativePos, len, progressUpdater); + assert regionOwners.get(fileRegion.io) == fileRegion + : "File chunk [" + fileRegion.regionKey + "] no longer owns IO [" + fileRegion.io + "]"; + } }; } return adjustedWriter; @@ -1240,18 +1294,79 @@ public interface RangeAvailableHandler { @FunctionalInterface public interface RangeMissingHandler { + /** + * Attempt to get a shared {@link SourceInputStreamFactory} for the given list of Gaps so that all of them + * can be filled from the input stream created from the factory. If a factory is returned, the gaps must be + * filled sequentially by calling {@link #fillCacheRange} in order with the factory. If {@code null} is returned, + * each invocation of {@link #fillCacheRange} creates its own input stream and can therefore be executed in parallel. + * @param gaps The list of gaps to be filled by fetching from source storage and writing into the cache. + * @return A factory object to be shared by all gaps filling process, or {@code null} if each gap filling should create + * its own input stream. + */ + @Nullable + default SourceInputStreamFactory sharedInputStreamFactory(List gaps) { + return null; + } + /** * Callback method used to fetch data (usually from a remote storage) and write it in the cache. * * @param channel is the cache region to write to * @param channelPos a position in the channel (cache file) to write to + * @param streamFactory factory to get the input stream positioned at the given value for the remote storage. + * This is useful for sharing the same stream across multiple calls to this method. + * If it is {@code null}, the method should open input stream on its own. * @param relativePos the relative position in the remote storage to read from * @param length of data to fetch * @param progressUpdater consumer to invoke with the number of copied bytes as they are written in cache. * This is used to notify waiting readers that data become available in cache. */ - void fillCacheRange(SharedBytes.IO channel, int channelPos, int relativePos, int length, IntConsumer progressUpdater) - throws IOException; + void fillCacheRange( + SharedBytes.IO channel, + int channelPos, + @Nullable SourceInputStreamFactory streamFactory, + int relativePos, + int length, + IntConsumer progressUpdater + ) throws IOException; + } + + /** + * Factory to create the input stream for reading data from the remote storage as the source for filling local cache regions. + */ + public interface SourceInputStreamFactory extends Releasable { + + /** + * Create the input stream at the specified position. + * @param relativePos the relative position in the remote storage to read from. + * @return the input stream ready to be read from. + */ + InputStream create(int relativePos) throws IOException; + } + + private abstract static class DelegatingRangeMissingHandler implements RangeMissingHandler { + protected final RangeMissingHandler delegate; + + protected DelegatingRangeMissingHandler(RangeMissingHandler delegate) { + this.delegate = delegate; + } + + @Override + public SourceInputStreamFactory sharedInputStreamFactory(List gaps) { + return delegate.sharedInputStreamFactory(gaps); + } + + @Override + public void fillCacheRange( + SharedBytes.IO channel, + int channelPos, + SourceInputStreamFactory streamFactory, + int relativePos, + int length, + IntConsumer progressUpdater + ) throws IOException { + delegate.fillCacheRange(channel, channelPos, streamFactory, relativePos, length, progressUpdater); + } } public record Stats( diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBytes.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBytes.java index 051dfab1cdaa0..ad0d99104e8a4 100644 --- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBytes.java +++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBytes.java @@ -18,10 +18,11 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.preallocate.Preallocate; +import org.elasticsearch.nativeaccess.NativeAccess; import java.io.IOException; import java.io.InputStream; +import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; @@ -78,7 +79,7 @@ public class SharedBytes extends AbstractRefCounted { Path cacheFile = null; if (fileSize > 0) { cacheFile = findCacheSnapshotCacheFilePath(environment, fileSize); - Preallocate.preallocate(cacheFile, fileSize); + preallocate(cacheFile, fileSize); this.fileChannel = FileChannel.open(cacheFile, OPEN_OPTIONS); assert this.fileChannel.size() == fileSize : "expected file size " + fileSize + " but was " + fileChannel.size(); } else { @@ -141,6 +142,24 @@ public static Path findCacheSnapshotCacheFilePath(NodeEnvironment environment, l } } + @SuppressForbidden(reason = "random access file needed to set file size") + static void preallocate(Path cacheFile, long fileSize) throws IOException { + // first try using native methods to preallocate space in the file + NativeAccess.instance().tryPreallocate(cacheFile, fileSize); + // even if allocation was successful above, verify again here + try (RandomAccessFile raf = new RandomAccessFile(cacheFile.toFile(), "rw")) { + if (raf.length() != fileSize) { + logger.info("pre-allocating cache file [{}] ({} bytes) using setLength method", cacheFile, fileSize); + raf.setLength(fileSize); + logger.debug("pre-allocated cache file [{}] using setLength method", cacheFile); + } + } catch (final Exception e) { + logger.warn(() -> "failed to pre-allocate cache file [" + cacheFile + "] using setLength method", e); + // if anything goes wrong, delete the potentially created file to not waste disk space + Files.deleteIfExists(cacheFile); + } + } + /** * Copy {@code length} bytes from {@code input} to {@code fc}, only doing writes aligned along {@link #PAGE_SIZE}. * diff --git a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java index edeed9a16034a..e477673c90d6d 100644 --- a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java +++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java @@ -14,6 +14,9 @@ import org.elasticsearch.blobcache.BlobCacheMetrics; import org.elasticsearch.blobcache.BlobCacheUtils; import org.elasticsearch.blobcache.common.ByteRange; +import org.elasticsearch.blobcache.common.SparseFileTracker; +import org.elasticsearch.blobcache.shared.SharedBlobCacheService.RangeMissingHandler; +import org.elasticsearch.blobcache.shared.SharedBlobCacheService.SourceInputStreamFactory; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -23,6 +26,7 @@ import org.elasticsearch.common.unit.RatioValue; import org.elasticsearch.common.unit.RelativeByteSizeValue; import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.StoppableExecutorServiceWrapper; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.env.Environment; @@ -34,6 +38,7 @@ import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; +import java.io.InputStream; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -43,8 +48,11 @@ import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.IntConsumer; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -53,7 +61,10 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; public class SharedBlobCacheServiceTests extends ESTestCase { @@ -104,7 +115,7 @@ public void testBasicEviction() throws IOException { ByteRange.of(0L, 1L), ByteRange.of(0L, 1L), (channel, channelPos, relativePos, length) -> 1, - (channel, channelPos, relativePos, length, progressUpdater) -> progressUpdater.accept(length), + (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> progressUpdater.accept(length), taskQueue.getThreadPool().generic(), bytesReadFuture ); @@ -538,10 +549,17 @@ public void execute(Runnable command) { final long size = size(250); AtomicLong bytesRead = new AtomicLong(size); final PlainActionFuture future = new PlainActionFuture<>(); - cacheService.maybeFetchFullEntry(cacheKey, size, (channel, channelPos, relativePos, length, progressUpdater) -> { - bytesRead.addAndGet(-length); - progressUpdater.accept(length); - }, bulkExecutor, future); + cacheService.maybeFetchFullEntry( + cacheKey, + size, + (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> { + assert streamFactory == null : streamFactory; + bytesRead.addAndGet(-length); + progressUpdater.accept(length); + }, + bulkExecutor, + future + ); future.get(10, TimeUnit.SECONDS); assertEquals(0L, bytesRead.get()); @@ -552,7 +570,7 @@ public void execute(Runnable command) { // a download that would use up all regions should not run final var cacheKey = generateCacheKey(); assertEquals(2, cacheService.freeRegionCount()); - var configured = cacheService.maybeFetchFullEntry(cacheKey, size(500), (ch, chPos, relPos, len, update) -> { + var configured = cacheService.maybeFetchFullEntry(cacheKey, size(500), (ch, chPos, streamFactory, relPos, len, update) -> { throw new AssertionError("Should never reach here"); }, bulkExecutor, ActionListener.noop()); assertFalse(configured); @@ -591,19 +609,17 @@ public void testFetchFullCacheEntryConcurrently() throws Exception { threads[i] = new Thread(() -> { for (int j = 0; j < 1000; j++) { final var cacheKey = generateCacheKey(); - try { - PlainActionFuture.get( - f -> cacheService.maybeFetchFullEntry( - cacheKey, - size, - (channel, channelPos, relativePos, length, progressUpdater) -> progressUpdater.accept(length), - bulkExecutor, - f - ) - ); - } catch (Exception e) { - throw new AssertionError(e); - } + safeAwait( + (ActionListener listener) -> cacheService.maybeFetchFullEntry( + cacheKey, + size, + (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> progressUpdater.accept( + length + ), + bulkExecutor, + listener + ) + ); } }); } @@ -843,7 +859,7 @@ public void testMaybeEvictLeastUsed() throws Exception { var entry = cacheService.get(cacheKey, regionSize, 0); entry.populate( ByteRange.of(0L, regionSize), - (channel, channelPos, relativePos, length, progressUpdater) -> progressUpdater.accept(length), + (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> progressUpdater.accept(length), taskQueue.getThreadPool().generic(), ActionListener.noop() ); @@ -934,10 +950,18 @@ public void execute(Runnable command) { final long blobLength = size(250); // 3 regions AtomicLong bytesRead = new AtomicLong(0L); final PlainActionFuture future = new PlainActionFuture<>(); - cacheService.maybeFetchRegion(cacheKey, 0, blobLength, (channel, channelPos, relativePos, length, progressUpdater) -> { - bytesRead.addAndGet(length); - progressUpdater.accept(length); - }, bulkExecutor, future); + cacheService.maybeFetchRegion( + cacheKey, + 0, + blobLength, + (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> { + assert streamFactory == null : streamFactory; + bytesRead.addAndGet(length); + progressUpdater.accept(length); + }, + bulkExecutor, + future + ); var fetched = future.get(10, TimeUnit.SECONDS); assertThat("Region has been fetched", fetched, is(true)); @@ -961,7 +985,8 @@ public void execute(Runnable command) { cacheKey, region, blobLength, - (channel, channelPos, relativePos, length, progressUpdater) -> { + (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> { + assert streamFactory == null : streamFactory; bytesRead.addAndGet(length); progressUpdater.accept(length); }, @@ -985,7 +1010,7 @@ public void execute(Runnable command) { cacheKey, randomIntBetween(0, 10), randomLongBetween(1L, regionSize), - (channel, channelPos, relativePos, length, progressUpdater) -> { + (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> { throw new AssertionError("should not be executed"); }, bulkExecutor, @@ -1003,10 +1028,18 @@ public void execute(Runnable command) { long blobLength = randomLongBetween(1L, regionSize); AtomicLong bytesRead = new AtomicLong(0L); final PlainActionFuture future = new PlainActionFuture<>(); - cacheService.maybeFetchRegion(cacheKey, 0, blobLength, (channel, channelPos, relativePos, length, progressUpdater) -> { - bytesRead.addAndGet(length); - progressUpdater.accept(length); - }, bulkExecutor, future); + cacheService.maybeFetchRegion( + cacheKey, + 0, + blobLength, + (channel, channelPos, ignore, relativePos, length, progressUpdater) -> { + assert ignore == null : ignore; + bytesRead.addAndGet(length); + progressUpdater.accept(length); + }, + bulkExecutor, + future + ); var fetched = future.get(10, TimeUnit.SECONDS); assertThat("Region has been fetched", fetched, is(true)); @@ -1077,7 +1110,7 @@ public void execute(Runnable command) { region, range, blobLength, - (channel, channelPos, relativePos, length, progressUpdater) -> { + (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> { assertThat(range.start() + relativePos, equalTo(cacheService.getRegionStart(region) + regionRange.start())); assertThat(channelPos, equalTo(Math.toIntExact(regionRange.start()))); assertThat(length, equalTo(Math.toIntExact(regionRange.length()))); @@ -1117,7 +1150,7 @@ public void execute(Runnable command) { region, ByteRange.of(0L, blobLength), blobLength, - (channel, channelPos, relativePos, length, progressUpdater) -> bytesCopied.addAndGet(length), + (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> bytesCopied.addAndGet(length), bulkExecutor, listener ); @@ -1140,7 +1173,7 @@ public void execute(Runnable command) { randomIntBetween(0, 10), ByteRange.of(0L, blobLength), blobLength, - (channel, channelPos, relativePos, length, progressUpdater) -> { + (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> { throw new AssertionError("should not be executed"); }, bulkExecutor, @@ -1163,7 +1196,7 @@ public void execute(Runnable command) { 0, ByteRange.of(0L, blobLength), blobLength, - (channel, channelPos, relativePos, length, progressUpdater) -> bytesCopied.addAndGet(length), + (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> bytesCopied.addAndGet(length), bulkExecutor, future ); @@ -1204,7 +1237,7 @@ public void testPopulate() throws Exception { var entry = cacheService.get(cacheKey, blobLength, 0); AtomicLong bytesWritten = new AtomicLong(0L); final PlainActionFuture future1 = new PlainActionFuture<>(); - entry.populate(ByteRange.of(0, regionSize - 1), (channel, channelPos, relativePos, length, progressUpdater) -> { + entry.populate(ByteRange.of(0, regionSize - 1), (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> { bytesWritten.addAndGet(length); progressUpdater.accept(length); }, taskQueue.getThreadPool().generic(), future1); @@ -1215,7 +1248,7 @@ public void testPopulate() throws Exception { // start populating the second region entry = cacheService.get(cacheKey, blobLength, 1); final PlainActionFuture future2 = new PlainActionFuture<>(); - entry.populate(ByteRange.of(0, regionSize - 1), (channel, channelPos, relativePos, length, progressUpdater) -> { + entry.populate(ByteRange.of(0, regionSize - 1), (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> { bytesWritten.addAndGet(length); progressUpdater.accept(length); }, taskQueue.getThreadPool().generic(), future2); @@ -1223,7 +1256,7 @@ public void testPopulate() throws Exception { // start populating again the first region, listener should be called immediately entry = cacheService.get(cacheKey, blobLength, 0); final PlainActionFuture future3 = new PlainActionFuture<>(); - entry.populate(ByteRange.of(0, regionSize - 1), (channel, channelPos, relativePos, length, progressUpdater) -> { + entry.populate(ByteRange.of(0, regionSize - 1), (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> { bytesWritten.addAndGet(length); progressUpdater.accept(length); }, taskQueue.getThreadPool().generic(), future3); @@ -1310,4 +1343,113 @@ protected int computeCacheFileRegionSize(long fileLength, int region) { } } } + + public void testSharedSourceInputStreamFactory() throws Exception { + final long regionSizeInBytes = size(100); + final Settings settings = Settings.builder() + .put(NODE_NAME_SETTING.getKey(), "node") + .put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(200)).getStringRep()) + .put(SharedBlobCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(regionSizeInBytes).getStringRep()) + .put("path.home", createTempDir()) + .build(); + final ThreadPool threadPool = new TestThreadPool("test"); + try ( + NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings)); + var cacheService = new SharedBlobCacheService<>( + environment, + settings, + threadPool, + ThreadPool.Names.GENERIC, + BlobCacheMetrics.NOOP + ) + ) { + final var cacheKey = generateCacheKey(); + assertEquals(2, cacheService.freeRegionCount()); + final var region = cacheService.get(cacheKey, size(250), 0); + assertEquals(regionSizeInBytes, region.tracker.getLength()); + + // Read disjoint ranges to create holes in the region + final long interval = regionSizeInBytes / between(5, 20); + for (var start = interval; start < regionSizeInBytes - 2 * SharedBytes.PAGE_SIZE; start += interval) { + final var range = ByteRange.of(start, start + SharedBytes.PAGE_SIZE); + final PlainActionFuture future = new PlainActionFuture<>(); + region.populateAndRead( + range, + range, + (channel, channelPos, relativePos, length) -> length, + (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> progressUpdater.accept(length), + EsExecutors.DIRECT_EXECUTOR_SERVICE, + future + ); + safeGet(future); + } + + // Read the entire region with a shared source input stream and we want to ensure the following behaviours + // 1. fillCacheRange is invoked as many times as the number of holes/gaps + // 2. fillCacheRange is invoked single threaded with the gap order + // 3. The shared streamFactory is passed to each invocation + // 4. The factory is closed at the end + final int numberGaps = region.tracker.getCompletedRanges().size() + 1; + final var invocationCounter = new AtomicInteger(); + final var factoryClosed = new AtomicBoolean(false); + final var dummyStreamFactory = new SourceInputStreamFactory() { + @Override + public InputStream create(int relativePos) { + return null; + } + + @Override + public void close() { + factoryClosed.set(true); + } + }; + + final var rangeMissingHandler = new RangeMissingHandler() { + final AtomicReference invocationThread = new AtomicReference<>(); + final AtomicInteger position = new AtomicInteger(-1); + + @Override + public SourceInputStreamFactory sharedInputStreamFactory(List gaps) { + return dummyStreamFactory; + } + + @Override + public void fillCacheRange( + SharedBytes.IO channel, + int channelPos, + SourceInputStreamFactory streamFactory, + int relativePos, + int length, + IntConsumer progressUpdater + ) throws IOException { + if (invocationCounter.incrementAndGet() == 1) { + final Thread witness = invocationThread.compareAndExchange(null, Thread.currentThread()); + assertThat(witness, nullValue()); + } else { + assertThat(invocationThread.get(), sameInstance(Thread.currentThread())); + } + assertThat(streamFactory, sameInstance(dummyStreamFactory)); + assertThat(position.getAndSet(relativePos), lessThan(relativePos)); + progressUpdater.accept(length); + } + }; + + final var range = ByteRange.of(0, regionSizeInBytes); + final PlainActionFuture future = new PlainActionFuture<>(); + region.populateAndRead( + range, + range, + (channel, channelPos, relativePos, length) -> length, + rangeMissingHandler, + threadPool.generic(), + future + ); + safeGet(future); + assertThat(invocationCounter.get(), equalTo(numberGaps)); + assertThat(region.tracker.checkAvailable(regionSizeInBytes), is(true)); + assertBusy(() -> assertThat(factoryClosed.get(), is(true))); + } finally { + threadPool.shutdown(); + } + } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java index d9592c3df4950..2d0c43315f746 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java @@ -67,7 +67,7 @@ public TransportBulkShardOperationsAction( BulkShardOperationsRequest::new, BulkShardOperationsRequest::new, ExecutorSelector.getWriteExecutorForShard(threadPool), - false, + PrimaryActionExecution.RejectOnOverload, indexingPressure, systemIndices ); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/DeleteInternalCcrRepositoryAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/DeleteInternalCcrRepositoryAction.java index f8e4cda1501b6..7f6a04bfee746 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/DeleteInternalCcrRepositoryAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/DeleteInternalCcrRepositoryAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; @@ -38,7 +39,7 @@ public TransportDeleteInternalRepositoryAction( ActionFilters actionFilters, TransportService transportService ) { - super(NAME, actionFilters, transportService.getTaskManager()); + super(NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.repositoriesService = repositoriesService; } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutInternalCcrRepositoryAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutInternalCcrRepositoryAction.java index 0de1715e17a14..497339930d551 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutInternalCcrRepositoryAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutInternalCcrRepositoryAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; @@ -38,7 +39,7 @@ public TransportPutInternalRepositoryAction( ActionFilters actionFilters, TransportService transportService ) { - super(NAME, actionFilters, transportService.getTaskManager()); + super(NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.repositoriesService = repositoriesService; } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 67c4c769d21d1..d5a6e3c7e65c8 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -14,6 +14,9 @@ import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.RemoteClusterActionType; import org.elasticsearch.action.SingleResultDeduplicator; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; @@ -252,15 +255,30 @@ public void getSnapshotInfo( } } + private Response executeRecoveryAction( + RemoteClusterClient client, + RemoteClusterActionType action, + Request request + ) { + final var future = new PlainActionFuture(); + client.execute(action, request, future); + // TODO stop doing this as a blocking activity + // TODO on timeout, cancel the remote request, don't just carry on + // TODO handle exceptions better, don't just unwrap/rewrap them with actionGet + return future.actionGet(ccrSettings.getRecoveryActionTimeout().millis(), TimeUnit.MILLISECONDS); + } + @Override public Metadata getSnapshotGlobalMetadata(SnapshotId snapshotId) { assert SNAPSHOT_ID.equals(snapshotId) : "RemoteClusterRepository only supports " + SNAPSHOT_ID + " as the SnapshotId"; var remoteClient = getRemoteClusterClient(); - // We set a single dummy index name to avoid fetching all the index data - ClusterStateResponse clusterState = PlainActionFuture.get( - f -> remoteClient.execute(ClusterStateAction.REMOTE_TYPE, CcrRequests.metadataRequest("dummy_index_name"), f), - ccrSettings.getRecoveryActionTimeout().millis(), - TimeUnit.MILLISECONDS + ClusterStateResponse clusterState = executeRecoveryAction( + remoteClient, + ClusterStateAction.REMOTE_TYPE, + CcrRequests.metadataRequest( + // We set a single dummy index name to avoid fetching all the index data + "dummy_index_name" + ) ); return clusterState.getState().metadata(); } @@ -271,10 +289,10 @@ public IndexMetadata getSnapshotIndexMetaData(RepositoryData repositoryData, Sna String leaderIndex = index.getName(); var remoteClient = getRemoteClusterClient(); - ClusterStateResponse clusterState = PlainActionFuture.get( - f -> remoteClient.execute(ClusterStateAction.REMOTE_TYPE, CcrRequests.metadataRequest(leaderIndex), f), - ccrSettings.getRecoveryActionTimeout().millis(), - TimeUnit.MILLISECONDS + ClusterStateResponse clusterState = executeRecoveryAction( + remoteClient, + ClusterStateAction.REMOTE_TYPE, + CcrRequests.metadataRequest(leaderIndex) ); // Validates whether the leader cluster has been configured properly: @@ -556,14 +574,10 @@ void acquireRetentionLeaseOnLeader( public IndexShardSnapshotStatus.Copy getShardSnapshotStatus(SnapshotId snapshotId, IndexId index, ShardId shardId) { assert SNAPSHOT_ID.equals(snapshotId) : "RemoteClusterRepository only supports " + SNAPSHOT_ID + " as the SnapshotId"; final String leaderIndex = index.getName(); - final IndicesStatsResponse response = PlainActionFuture.get( - f -> getRemoteClusterClient().execute( - IndicesStatsAction.REMOTE_TYPE, - new IndicesStatsRequest().indices(leaderIndex).clear().store(true), - f - ), - ccrSettings.getRecoveryActionTimeout().millis(), - TimeUnit.MILLISECONDS + final IndicesStatsResponse response = executeRecoveryAction( + getRemoteClusterClient(), + IndicesStatsAction.REMOTE_TYPE, + new IndicesStatsRequest().indices(leaderIndex).clear().store(true) ); for (ShardStats shardStats : response.getIndex(leaderIndex).getShards()) { final ShardRouting shardRouting = shardStats.getShardRouting(); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java index 04a97ad9e7f95..5cd9f8bc5b78c 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java @@ -834,12 +834,12 @@ protected void adaptResponse(BulkShardOperationsResponse response, IndexShard in @Override protected void performOnReplica(BulkShardOperationsRequest request, IndexShard replica) throws Exception { try ( - Releasable ignored = PlainActionFuture.get( - f -> replica.acquireReplicaOperationPermit( + Releasable ignored = safeAwait( + listener -> replica.acquireReplicaOperationPermit( getPrimaryShard().getPendingPrimaryTerm(), getPrimaryShard().getLastKnownGlobalCheckpoint(), getPrimaryShard().getMaxSeqNoOfUpdatesOrDeletes(), - f, + ActionListener.assertOnce(listener), EsExecutors.DIRECT_EXECUTOR_SERVICE ) ) diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java index fbddfc7683d2f..478a0d08d6612 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java @@ -697,8 +697,8 @@ public void testProcessOnceOnPrimary() throws Exception { case TIME_SERIES: settingsBuilder.put("index.mode", "time_series").put("index.routing_path", "foo"); break; - case LOGS: - settingsBuilder.put("index.mode", "logs"); + case LOGSDB: + settingsBuilder.put("index.mode", IndexMode.LOGSDB.getName()); break; default: throw new UnsupportedOperationException("Unknown index mode [" + indexMode + "]"); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index 4ed2e2a8e056c..388868188b675 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -87,6 +87,7 @@ public final class XPackField { /** Name constant for the redact processor feature. */ public static final String REDACT_PROCESSOR = "redact_processor"; + public static final String ENTERPRISE_GEOIP_DOWNLOADER = "enterprise_geoip_downloader"; /** Name for Universal Profiling. */ public static final String UNIVERSAL_PROFILING = "universal_profiling"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java index a0ecc5dd566c2..1ac164d600115 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java @@ -40,7 +40,7 @@ public static class Request extends MasterNodeRequest { public Request(StreamInput in) throws IOException { super(in); - if (in.getTransportVersion().onOrAfter(TransportVersions.CCR_STATS_API_TIMEOUT_PARAM)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { timeout = in.readOptionalTimeValue(); } } @@ -57,7 +57,7 @@ public ActionRequestValidationException validate() { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - if (out.getTransportVersion().onOrAfter(TransportVersions.CCR_STATS_API_TIMEOUT_PARAM)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalTimeValue(timeout); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/notifications/Level.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/notifications/Level.java index 2db973f8122c1..f559370350972 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/notifications/Level.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/notifications/Level.java @@ -9,9 +9,23 @@ import java.util.Locale; public enum Level { - INFO, - WARNING, - ERROR; + INFO { + public org.apache.logging.log4j.Level log4jLevel() { + return org.apache.logging.log4j.Level.INFO; + } + }, + WARNING { + public org.apache.logging.log4j.Level log4jLevel() { + return org.apache.logging.log4j.Level.WARN; + } + }, + ERROR { + public org.apache.logging.log4j.Level log4jLevel() { + return org.apache.logging.log4j.Level.ERROR; + } + }; + + public abstract org.apache.logging.log4j.Level log4jLevel(); /** * Case-insensitive from string method. diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java index 46425a526c53f..fc9087d97bd79 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.xcontent.ToXContentFragment; @@ -195,7 +196,8 @@ public record CacheStats( long misses, long evictions, long hitsTimeInMillis, - long missesTimeInMillis + long missesTimeInMillis, + long cacheSizeInBytes ) implements Writeable, ToXContentFragment { public CacheStats(StreamInput in) throws IOException { @@ -206,7 +208,8 @@ public CacheStats(StreamInput in) throws IOException { in.readVLong(), in.readVLong(), in.getTransportVersion().onOrAfter(TransportVersions.ENRICH_CACHE_ADDITIONAL_STATS) ? in.readLong() : -1, - in.getTransportVersion().onOrAfter(TransportVersions.ENRICH_CACHE_ADDITIONAL_STATS) ? in.readLong() : -1 + in.getTransportVersion().onOrAfter(TransportVersions.ENRICH_CACHE_ADDITIONAL_STATS) ? in.readLong() : -1, + in.getTransportVersion().onOrAfter(TransportVersions.ENRICH_CACHE_STATS_SIZE_ADDED) ? in.readLong() : -1 ); } @@ -219,6 +222,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("evictions", evictions); builder.humanReadableField("hits_time_in_millis", "hits_time", new TimeValue(hitsTimeInMillis)); builder.humanReadableField("misses_time_in_millis", "misses_time", new TimeValue(missesTimeInMillis)); + builder.humanReadableField("size_in_bytes", "size", ByteSizeValue.ofBytes(cacheSizeInBytes)); return builder; } @@ -233,6 +237,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(hitsTimeInMillis); out.writeLong(missesTimeInMillis); } + if (out.getTransportVersion().onOrAfter(TransportVersions.ENRICH_CACHE_STATS_SIZE_ADDED)) { + out.writeLong(cacheSizeInBytes); + } } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java index 60f9b7b001060..9cbcd6c62dc3b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java @@ -106,7 +106,7 @@ public ShrinkAction(StreamInput in) throws IOException { this.numberOfShards = null; this.maxPrimaryShardSize = ByteSizeValue.readFrom(in); } - this.allowWriteAfterShrink = in.getTransportVersion().onOrAfter(TransportVersions.ILM_SHRINK_ENABLE_WRITE) && in.readBoolean(); + this.allowWriteAfterShrink = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) && in.readBoolean(); } public Integer getNumberOfShards() { @@ -130,7 +130,7 @@ public void writeTo(StreamOutput out) throws IOException { } else { maxPrimaryShardSize.writeTo(out); } - if (out.getTransportVersion().onOrAfter(TransportVersions.ILM_SHRINK_ENABLE_WRITE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeBoolean(this.allowWriteAfterShrink); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/InferenceFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/InferenceFeatureSetUsage.java index 8464821275aa8..61409f59f9d85 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/InferenceFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/InferenceFeatureSetUsage.java @@ -54,14 +54,30 @@ public void add() { count++; } + public String service() { + return service; + } + + public TaskType taskType() { + return taskType; + } + + public long count() { + return count; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + addXContentFragment(builder, params); + builder.endObject(); + return builder; + } + + public void addXContentFragment(XContentBuilder builder, Params params) throws IOException { builder.field("service", service); builder.field("task_type", taskType.name()); builder.field("count", count); - builder.endObject(); - return builder; } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/InferenceRequestStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/InferenceRequestStats.java new file mode 100644 index 0000000000000..74d44b1a24173 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/InferenceRequestStats.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.inference; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +public class InferenceRequestStats implements SerializableStats { + private final InferenceFeatureSetUsage.ModelStats modelStats; + private final String modelId; + + public InferenceRequestStats(String service, TaskType taskType, @Nullable String modelId, long count) { + this(new InferenceFeatureSetUsage.ModelStats(service, taskType, count), modelId); + } + + private InferenceRequestStats(InferenceFeatureSetUsage.ModelStats modelStats, @Nullable String modelId) { + this.modelStats = new InferenceFeatureSetUsage.ModelStats(modelStats); + this.modelId = modelId; + } + + public InferenceRequestStats(StreamInput in) throws IOException { + this.modelStats = new InferenceFeatureSetUsage.ModelStats(in); + this.modelId = in.readOptionalString(); + } + + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + builder.field("service", modelStats.service()); + builder.field("task_type", modelStats.taskType().toString()); + builder.field("count", modelStats.count()); + + if (modelId != null) { + builder.field("model_id", modelId); + } + + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + modelStats.writeTo(out); + out.writeOptionalString(modelId); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + InferenceRequestStats that = (InferenceRequestStats) o; + return Objects.equals(modelStats, that.modelStats) && Objects.equals(modelId, that.modelId); + } + + @Override + public int hashCode() { + return Objects.hash(modelStats, modelId); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/SerializableStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/SerializableStats.java new file mode 100644 index 0000000000000..7704304b11365 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/SerializableStats.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.inference; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.ToXContentObject; + +public interface SerializableStats extends ToXContentObject, Writeable { + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java index 229285510249c..53e404b48dc2e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java @@ -122,15 +122,11 @@ public Request(StreamInput in) throws IOException { this.inputType = InputType.UNSPECIFIED; } - if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_COHERE_RERANK)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { this.query = in.readOptionalString(); - } else { - this.query = null; - } - - if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_TIMEOUT_ADDED)) { this.inferenceTimeout = in.readTimeValue(); } else { + this.query = null; this.inferenceTimeout = DEFAULT_TIMEOUT; } } @@ -209,11 +205,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeEnum(getInputTypeToWrite(inputType, out.getTransportVersion())); } - if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_COHERE_RERANK)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalString(query); - } - - if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_TIMEOUT_ADDED)) { out.writeTimeValue(inferenceTimeout); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java index f82ee8b73c7a2..9196a57c868ba 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.inference.results; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -27,7 +28,6 @@ import java.util.Objects; import java.util.stream.Collectors; -import static org.elasticsearch.TransportVersions.ML_INFERENCE_RERANK_NEW_RESPONSE_FORMAT; import static org.elasticsearch.TransportVersions.ML_RERANK_DOC_OPTIONAL; public class RankedDocsResults implements InferenceServiceResults { @@ -113,7 +113,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public static RankedDoc of(StreamInput in) throws IOException { if (in.getTransportVersion().onOrAfter(ML_RERANK_DOC_OPTIONAL)) { return new RankedDoc(in.readInt(), in.readFloat(), in.readOptionalString()); - } else if (in.getTransportVersion().onOrAfter(ML_INFERENCE_RERANK_NEW_RESPONSE_FORMAT)) { + } else if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { return new RankedDoc(in.readInt(), in.readFloat(), in.readString()); } else { return new RankedDoc(Integer.parseInt(in.readString()), Float.parseFloat(in.readString()), in.readString()); @@ -126,7 +126,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeInt(index); out.writeFloat(relevanceScore); out.writeOptionalString(text); - } else if (out.getTransportVersion().onOrAfter(ML_INFERENCE_RERANK_NEW_RESPONSE_FORMAT)) { + } else if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeInt(index); out.writeFloat(relevanceScore); out.writeString(text == null ? "" : text); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java index b5469fadd95b6..c9b30a826248a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java @@ -43,7 +43,7 @@ public DefaultAuthenticationFailureHandler(final Map> failu if (failureResponseHeaders == null || failureResponseHeaders.isEmpty()) { this.defaultFailureResponseHeaders = Collections.singletonMap( "WWW-Authenticate", - Collections.singletonList("Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\"") + Collections.singletonList("Basic realm=\"" + XPackField.SECURITY + "\", charset=\"UTF-8\"") ); } else { this.defaultFailureResponseHeaders = Collections.unmodifiableMap( diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java index 63989ee86b3a0..bce9e6255a037 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java @@ -69,7 +69,7 @@ public int order() { public Map> getAuthenticationFailureHeaders() { return Collections.singletonMap( "WWW-Authenticate", - Collections.singletonList("Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\"") + Collections.singletonList("Basic realm=\"" + XPackField.SECURITY + "\", charset=\"UTF-8\"") ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java index 461619f2279f6..17088cff8718b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java @@ -91,12 +91,12 @@ public static Set resolveRoles( .flatMap(m -> { Set roleNames = m.getRoleNames(scriptService, model); logger.trace( - () -> format("Applying role-mapping [{}] to user-model [{}] produced role-names [{}]", m.getName(), model, roleNames) + () -> format("Applying role-mapping [%s] to user-model [%s] produced role-names [%s]", m.getName(), model, roleNames) ); return roleNames.stream(); }) .collect(Collectors.toSet()); - logger.debug(() -> format("Mapping user [{}] to roles [{}]", user, roles)); + logger.debug(() -> format("Mapping user [%s] to roles [%s]", user, roles)); return roles; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java index 7174b2f616c2a..df13bcb9342bf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java @@ -86,6 +86,7 @@ public final class IndexPrivilege extends Privilege { TransportClusterSearchShardsAction.TYPE.name(), TransportSearchShardsAction.TYPE.name(), TransportResolveClusterAction.NAME, + "indices:data/read/esql/resolve_fields", "indices:data/read/esql", "indices:data/read/esql/compute" ); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Exceptions.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Exceptions.java index 37f1dce5af7ba..b323e7ef20171 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Exceptions.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Exceptions.java @@ -18,13 +18,13 @@ private Exceptions() {} public static ElasticsearchSecurityException authenticationError(String msg, Throwable cause, Object... args) { ElasticsearchSecurityException e = new ElasticsearchSecurityException(msg, RestStatus.UNAUTHORIZED, cause, args); - e.addHeader("WWW-Authenticate", "Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\""); + e.addHeader("WWW-Authenticate", "Basic realm=\"" + XPackField.SECURITY + "\", charset=\"UTF-8\""); return e; } public static ElasticsearchSecurityException authenticationError(String msg, Object... args) { ElasticsearchSecurityException e = new ElasticsearchSecurityException(msg, RestStatus.UNAUTHORIZED, args); - e.addHeader("WWW-Authenticate", "Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\""); + e.addHeader("WWW-Authenticate", "Basic realm=\"" + XPackField.SECURITY + "\", charset=\"UTF-8\""); return e; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java index a317b79901751..daec14df094c4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java @@ -88,7 +88,7 @@ public static SnapshotLifecyclePolicyMetadata parse(XContentParser parser, Strin return PARSER.apply(parser, name); } - SnapshotLifecyclePolicyMetadata( + public SnapshotLifecyclePolicyMetadata( SnapshotLifecyclePolicy policy, Map headers, long version, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PutTransformAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PutTransformAction.java index 496e826651572..f9fde6b6816e0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PutTransformAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PutTransformAction.java @@ -14,6 +14,9 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.core.common.validation.SourceDestValidator; import org.elasticsearch.xpack.core.transform.TransformField; @@ -22,6 +25,7 @@ import org.elasticsearch.xpack.core.transform.utils.TransformStrings; import java.io.IOException; +import java.util.Map; import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -154,6 +158,11 @@ public boolean equals(Object obj) { && this.deferValidation == other.deferValidation && ackTimeout().equals(other.ackTimeout()); } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers); + } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/StartTransformAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/StartTransformAction.java index 838a0650c8afa..f02aaf553b8a9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/StartTransformAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/StartTransformAction.java @@ -14,6 +14,9 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.transform.TransformField; @@ -22,6 +25,7 @@ import java.io.IOException; import java.time.Instant; import java.util.Collections; +import java.util.Map; import java.util.Objects; public class StartTransformAction extends ActionType { @@ -89,6 +93,11 @@ public boolean equals(Object obj) { // the base class does not implement equals, therefore we need to check timeout ourselves return Objects.equals(id, other.id) && Objects.equals(from, other.from) && ackTimeout().equals(other.ackTimeout()); } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers); + } } public static class Response extends BaseTasksResponse implements ToXContentObject { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/ValidateTransformAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/ValidateTransformAction.java index 55c21b91b11d8..eae7d8a909c35 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/ValidateTransformAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/ValidateTransformAction.java @@ -14,6 +14,9 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.xpack.core.common.validation.SourceDestValidator; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; @@ -94,6 +97,11 @@ public int hashCode() { // the base class does not implement hashCode, therefore we need to hash timeout ourselves return Objects.hash(ackTimeout(), config, deferValidation); } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers); + } } public static class Response extends ActionResponse { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java index 34abaeb4cdf29..e39ddc170c0a9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java @@ -363,13 +363,11 @@ public void onFailure(Exception e) { indexId ) ).build(); - IndexMetadata metadata = runAsSnapshot( - threadPool, - () -> repository.getSnapshotIndexMetaData( - PlainActionFuture.get(listener -> repository.getRepositoryData(EsExecutors.DIRECT_EXECUTOR_SERVICE, listener)), - snapshotId, - indexId - ) + + IndexMetadata metadata = repository.getSnapshotIndexMetaData( + safeAwait(listener -> repository.getRepositoryData(EsExecutors.DIRECT_EXECUTOR_SERVICE, listener)), + snapshotId, + indexId ); IndexShard restoredShard = newShard( shardRouting, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java index 7c1e71139cb0c..b4389377dff34 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java @@ -55,7 +55,7 @@ public void testSerializationBwc() throws IOException { if (randomBoolean()) { request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(20, 25, 30))); } - assertSerialization(request, TransportVersionUtils.getPreviousVersion(TransportVersions.CCR_STATS_API_TIMEOUT_PARAM)); + assertSerialization(request, TransportVersionUtils.getPreviousVersion(TransportVersions.V_8_14_0)); assertSerialization(request, TransportVersions.MINIMUM_CCS_VERSION); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/LevelTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/LevelTests.java index fcc88862eed0c..7d720d401e064 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/LevelTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/LevelTests.java @@ -32,4 +32,10 @@ public void testValidOrdinals() { assertThat(Level.WARNING.ordinal(), equalTo(1)); assertThat(Level.ERROR.ordinal(), equalTo(2)); } + + public void testLog4JLevel() { + assertThat(Level.INFO.log4jLevel(), equalTo(org.apache.logging.log4j.Level.INFO)); + assertThat(Level.WARNING.log4jLevel(), equalTo(org.apache.logging.log4j.Level.WARN)); + assertThat(Level.ERROR.log4jLevel(), equalTo(org.apache.logging.log4j.Level.ERROR)); + } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/AbstractStepTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/AbstractStepTestCase.java index 9767a7546915e..cb020c1dcce20 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/AbstractStepTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/AbstractStepTestCase.java @@ -6,11 +6,14 @@ */ package org.elasticsearch.xpack.core.ilm; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.AdminClient; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.IndicesAdminClient; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateObserver; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; @@ -18,6 +21,9 @@ import org.junit.Before; import org.mockito.Mockito; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + public abstract class AbstractStepTestCase extends ESTestCase { protected Client client; @@ -67,4 +73,28 @@ public void testStepNameNotError() { StepKey nextStepKey = instance.getKey(); assertFalse(ErrorStep.NAME.equals(nextStepKey.name())); } + + protected void performActionAndWait( + AsyncActionStep step, + IndexMetadata indexMetadata, + ClusterState currentClusterState, + ClusterStateObserver observer + ) throws Exception { + final var future = new PlainActionFuture(); + step.performAction(indexMetadata, currentClusterState, observer, future); + try { + future.get(SAFE_AWAIT_TIMEOUT.millis(), TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + if (e.getCause() instanceof Exception exception) { + throw exception; + } else { + fail(e, "unexpected"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + fail(e, "unexpected"); + } catch (Exception e) { + fail(e, "unexpected"); + } + } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/AbstractUnfollowIndexStepTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/AbstractUnfollowIndexStepTestCase.java index 9fd7a27a1ca98..3116cf5227ce3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/AbstractUnfollowIndexStepTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/AbstractUnfollowIndexStepTestCase.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.core.ilm; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.index.IndexVersion; import org.mockito.Mockito; @@ -40,15 +39,16 @@ protected final T copyInstance(T instance) { } public final void testNotAFollowerIndex() throws Exception { - IndexMetadata indexMetadata = IndexMetadata.builder("follower-index") - .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) - .numberOfShards(1) - .numberOfReplicas(0) - .build(); - - T step = newInstance(randomStepKey(), randomStepKey()); - - PlainActionFuture.get(f -> step.performAction(indexMetadata, null, null, f)); + performActionAndWait( + newInstance(randomStepKey(), randomStepKey()), + IndexMetadata.builder("follower-index") + .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE, "true")) + .numberOfShards(1) + .numberOfReplicas(0) + .build(), + null, + null + ); Mockito.verifyNoMoreInteractions(client); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupSnapshotStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupSnapshotStepTests.java index c7e9cc707aa02..922826032a224 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupSnapshotStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupSnapshotStepTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.delete.TransportDeleteSnapshotAction; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.LifecycleExecutionState; @@ -58,19 +57,18 @@ public void testPerformActionDoesntFailIfSnapshotInfoIsMissing() throws Exceptio String policyName = "test-ilm-policy"; { - IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexName) + final var indexMetadata = IndexMetadata.builder(indexName) .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName)) .numberOfShards(randomIntBetween(1, 5)) - .numberOfReplicas(randomIntBetween(0, 5)); - - IndexMetadata indexMetadata = indexMetadataBuilder.build(); - - ClusterState clusterState = ClusterState.builder(emptyClusterState()) - .metadata(Metadata.builder().put(indexMetadata, true).build()) + .numberOfReplicas(randomIntBetween(0, 5)) .build(); - CleanupSnapshotStep cleanupSnapshotStep = createRandomInstance(); - PlainActionFuture.get(f -> cleanupSnapshotStep.performAction(indexMetadata, clusterState, null, f)); + performActionAndWait( + createRandomInstance(), + indexMetadata, + ClusterState.builder(emptyClusterState()).metadata(Metadata.builder().put(indexMetadata, true).build()).build(), + null + ); } { @@ -83,12 +81,12 @@ public void testPerformActionDoesntFailIfSnapshotInfoIsMissing() throws Exceptio IndexMetadata indexMetadata = indexMetadataBuilder.build(); - ClusterState clusterState = ClusterState.builder(emptyClusterState()) - .metadata(Metadata.builder().put(indexMetadata, true).build()) - .build(); - - CleanupSnapshotStep cleanupSnapshotStep = createRandomInstance(); - PlainActionFuture.get(f -> cleanupSnapshotStep.performAction(indexMetadata, clusterState, null, f)); + performActionAndWait( + createRandomInstance(), + indexMetadata, + ClusterState.builder(emptyClusterState()).metadata(Metadata.builder().put(indexMetadata, true).build()).build(), + null + ); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CloseFollowerIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CloseFollowerIndexStepTests.java index e1c3cb5aa614a..7ce078826b49a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CloseFollowerIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CloseFollowerIndexStepTests.java @@ -9,7 +9,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.index.IndexVersion; import org.mockito.Mockito; @@ -44,7 +43,7 @@ public void testCloseFollowingIndex() throws Exception { }).when(indicesClient).close(Mockito.any(), Mockito.any()); CloseFollowerIndexStep step = new CloseFollowerIndexStep(randomStepKey(), randomStepKey(), client); - PlainActionFuture.get(f -> step.performAction(indexMetadata, emptyClusterState(), null, f)); + performActionAndWait(step, indexMetadata, emptyClusterState(), null); } public void testRequestNotAcknowledged() { @@ -60,10 +59,7 @@ public void testRequestNotAcknowledged() { }).when(indicesClient).close(Mockito.any(), Mockito.any()); CloseFollowerIndexStep step = new CloseFollowerIndexStep(randomStepKey(), randomStepKey(), client); - Exception e = expectThrows( - Exception.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, emptyClusterState(), null, f)) - ); + Exception e = expectThrows(Exception.class, () -> performActionAndWait(step, indexMetadata, emptyClusterState(), null)); assertThat(e.getMessage(), is("close index request failed to be acknowledged")); } @@ -81,13 +77,7 @@ public void testCloseFollowingIndexFailed() { }).when(indicesClient).close(Mockito.any(), Mockito.any()); CloseFollowerIndexStep step = new CloseFollowerIndexStep(randomStepKey(), randomStepKey(), client); - assertSame( - error, - expectThrows( - Exception.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, emptyClusterState(), null, f)) - ) - ); + assertSame(error, expectThrows(Exception.class, () -> performActionAndWait(step, indexMetadata, emptyClusterState(), null))); Mockito.verify(indicesClient).close(Mockito.any(), Mockito.any()); Mockito.verifyNoMoreInteractions(indicesClient); } @@ -101,7 +91,7 @@ public void testCloseFollowerIndexIsNoopForAlreadyClosedIndex() throws Exception .numberOfReplicas(0) .build(); CloseFollowerIndexStep step = new CloseFollowerIndexStep(randomStepKey(), randomStepKey(), client); - PlainActionFuture.get(f -> step.performAction(indexMetadata, emptyClusterState(), null, f)); + performActionAndWait(step, indexMetadata, emptyClusterState(), null); Mockito.verifyNoMoreInteractions(client); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CloseIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CloseIndexStepTests.java index d00ccbdfbb0fd..02fb49ac71adf 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CloseIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CloseIndexStepTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.AdminClient; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.IndicesAdminClient; @@ -129,14 +128,7 @@ public void testPerformActionFailure() { return null; }).when(indicesClient).close(Mockito.any(), Mockito.any()); - assertSame( - exception, - expectThrows( - Exception.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, null, null, f)) - ) - ); - + assertSame(exception, expectThrows(Exception.class, () -> performActionAndWait(step, indexMetadata, null, null))); Mockito.verify(client, Mockito.only()).admin(); Mockito.verify(adminClient, Mockito.only()).indices(); Mockito.verify(indicesClient, Mockito.only()).close(Mockito.any(), Mockito.any()); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CreateSnapshotStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CreateSnapshotStepTests.java index 9d74227afe9fe..de0bfa0440179 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CreateSnapshotStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CreateSnapshotStepTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.LifecycleExecutionState; @@ -85,7 +84,7 @@ public void testPerformActionFailure() { CreateSnapshotStep createSnapshotStep = createRandomInstance(); Exception e = expectThrows( IllegalStateException.class, - () -> PlainActionFuture.get(f -> createSnapshotStep.performAction(indexMetadata, clusterState, null, f)) + () -> performActionAndWait(createSnapshotStep, indexMetadata, clusterState, null) ); assertThat(e.getMessage(), is("snapshot name was not generated for policy [" + policyName + "] and index [" + indexName + "]")); } @@ -104,7 +103,7 @@ public void testPerformActionFailure() { CreateSnapshotStep createSnapshotStep = createRandomInstance(); Exception e = expectThrows( IllegalStateException.class, - () -> PlainActionFuture.get(f -> createSnapshotStep.performAction(indexMetadata, clusterState, null, f)) + () -> performActionAndWait(createSnapshotStep, indexMetadata, clusterState, null) ); assertThat( e.getMessage(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DeleteStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DeleteStepTests.java index af4dc67d5dcbd..7e9db3a4f1645 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DeleteStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DeleteStepTests.java @@ -9,7 +9,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.datastreams.DeleteDataStreamAction; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.DataStream; @@ -87,7 +86,7 @@ public void testDeleted() throws Exception { ClusterState clusterState = ClusterState.builder(emptyClusterState()) .metadata(Metadata.builder().put(indexMetadata, true).build()) .build(); - PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)); + performActionAndWait(step, indexMetadata, clusterState, null); Mockito.verify(client, Mockito.only()).admin(); Mockito.verify(adminClient, Mockito.only()).indices(); @@ -113,13 +112,7 @@ public void testExceptionThrown() { ClusterState clusterState = ClusterState.builder(emptyClusterState()) .metadata(Metadata.builder().put(indexMetadata, true).build()) .build(); - assertSame( - exception, - expectThrows( - Exception.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)) - ) - ); + assertSame(exception, expectThrows(Exception.class, () -> performActionAndWait(step, indexMetadata, clusterState, null))); } public void testPerformActionCallsFailureListenerIfIndexIsTheDataStreamWriteIndex() { @@ -255,7 +248,7 @@ public void testDeleteWorksIfWriteIndexIsTheOnlyIndexInDataStream() throws Excep // Try on the normal data stream - It should delete the data stream DeleteStep step = createRandomInstance(); - PlainActionFuture.get(f -> step.performAction(index1, clusterState, null, f)); + performActionAndWait(step, index1, clusterState, null); Mockito.verify(client, Mockito.only()).execute(any(), any(), any()); Mockito.verify(adminClient, Mockito.never()).indices(); @@ -328,7 +321,7 @@ public void testDeleteWorksIfWriteIndexIsTheOnlyIndexInDataStreamWithFailureStor // Again, the deletion should work since the data stream would be fully deleted anyway if the failure store were disabled. DeleteStep step = createRandomInstance(); - PlainActionFuture.get(f -> step.performAction(index1, clusterState, null, f)); + performActionAndWait(step, index1, clusterState, null); Mockito.verify(client, Mockito.only()).execute(any(), any(), any()); Mockito.verify(adminClient, Mockito.never()).indices(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleStepTests.java index cd8472172ef7b..877d33b3cfda3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleStepTests.java @@ -8,7 +8,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.downsample.DownsampleAction; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -120,7 +119,7 @@ public void testPerformAction() throws Exception { mockClientDownsampleCall(index); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexMetadata, true)).build(); - PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)); + performActionAndWait(step, indexMetadata, clusterState, null); } public void testPerformActionFailureInvalidExecutionState() { @@ -171,7 +170,7 @@ public void testPerformActionOnDataStream() throws Exception { ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) .metadata(Metadata.builder().put(newInstance(dataStreamName, List.of(indexMetadata.getIndex()))).put(indexMetadata, true)) .build(); - PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)); + performActionAndWait(step, indexMetadata, clusterState, null); } /** diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeStepTests.java index b16983c6a7ac6..c4857a31b7a7a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeStepTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.support.DefaultShardOperationFailedException; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -82,7 +81,7 @@ public void testPerformActionComplete() throws Exception { }).when(indicesClient).forceMerge(any(), any()); ForceMergeStep step = new ForceMergeStep(stepKey, nextStepKey, client, maxNumSegments); - PlainActionFuture.get(f -> step.performAction(indexMetadata, null, null, f)); + performActionAndWait(step, indexMetadata, null, null); } public void testPerformActionThrowsException() { @@ -109,13 +108,7 @@ public void testPerformActionThrowsException() { }).when(indicesClient).forceMerge(any(), any()); ForceMergeStep step = new ForceMergeStep(stepKey, nextStepKey, client, maxNumSegments); - assertSame( - exception, - expectThrows( - Exception.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, null, null, f)) - ) - ); + assertSame(exception, expectThrows(Exception.class, () -> performActionAndWait(step, indexMetadata, null, null))); } public void testForcemergeFailsOnSomeShards() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStepTests.java index f905ca38e1c5c..2b5a0535caa0e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStepTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.LifecycleExecutionState; @@ -113,7 +112,7 @@ public void testPerformActionFailure() { MountSnapshotStep mountSnapshotStep = createRandomInstance(); Exception e = expectThrows( IllegalStateException.class, - () -> PlainActionFuture.get(f -> mountSnapshotStep.performAction(indexMetadata, clusterState, null, f)) + () -> performActionAndWait(mountSnapshotStep, indexMetadata, clusterState, null) ); assertThat( e.getMessage(), @@ -139,7 +138,7 @@ public void testPerformActionFailure() { MountSnapshotStep mountSnapshotStep = createRandomInstance(); Exception e = expectThrows( IllegalStateException.class, - () -> PlainActionFuture.get(f -> mountSnapshotStep.performAction(indexMetadata, clusterState, null, f)) + () -> performActionAndWait(mountSnapshotStep, indexMetadata, clusterState, null) ); assertThat(e.getMessage(), is("snapshot name was not generated for policy [" + policyName + "] and index [" + indexName + "]")); } @@ -182,7 +181,7 @@ public void testPerformAction() throws Exception { RESTORED_INDEX_PREFIX, randomStorageType() ); - PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)); + performActionAndWait(step, indexMetadata, clusterState, null); } } @@ -217,7 +216,7 @@ public void testResponseStatusHandling() throws Exception { RESTORED_INDEX_PREFIX, randomStorageType() ); - PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)); + performActionAndWait(step, indexMetadata, clusterState, null); } } @@ -232,7 +231,7 @@ public void testResponseStatusHandling() throws Exception { RESTORED_INDEX_PREFIX, randomStorageType() ); - PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)); + performActionAndWait(step, indexMetadata, clusterState, null); } } } @@ -306,7 +305,7 @@ public void doTestMountWithoutSnapshotIndexNameInState(String prefix) throws Exc RESTORED_INDEX_PREFIX, randomStorageType() ); - PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)); + performActionAndWait(step, indexMetadata, clusterState, null); } } } @@ -348,7 +347,7 @@ public void testIgnoreTotalShardsPerNodeInFrozenPhase() throws Exception { RESTORED_INDEX_PREFIX, randomStorageType() ); - PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)); + performActionAndWait(step, indexMetadata, clusterState, null); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/OpenIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/OpenIndexStepTests.java index 28b6ab42bb4d1..7f3d408ee3d49 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/OpenIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/OpenIndexStepTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.AdminClient; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.IndicesAdminClient; @@ -80,7 +79,7 @@ public void testPerformAction() throws Exception { return null; }).when(indicesClient).open(Mockito.any(), Mockito.any()); - PlainActionFuture.get(f -> step.performAction(indexMetadata, null, null, f)); + performActionAndWait(step, indexMetadata, null, null); Mockito.verify(client, Mockito.only()).admin(); Mockito.verify(adminClient, Mockito.only()).indices(); @@ -112,13 +111,7 @@ public void testPerformActionFailure() { return null; }).when(indicesClient).open(Mockito.any(), Mockito.any()); - assertSame( - exception, - expectThrows( - Exception.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, null, null, f)) - ) - ); + assertSame(exception, expectThrows(Exception.class, () -> performActionAndWait(step, indexMetadata, null, null))); Mockito.verify(client, Mockito.only()).admin(); Mockito.verify(adminClient, Mockito.only()).indices(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PauseFollowerIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PauseFollowerIndexStepTests.java index 1ae22b669b064..51ebc98176955 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PauseFollowerIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PauseFollowerIndexStepTests.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.core.ilm; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -55,7 +54,7 @@ public void testPauseFollowingIndex() throws Exception { }).when(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); PauseFollowerIndexStep step = new PauseFollowerIndexStep(randomStepKey(), randomStepKey(), client); - PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)); + performActionAndWait(step, indexMetadata, clusterState, null); } public void testRequestNotAcknowledged() { @@ -75,10 +74,7 @@ public void testRequestNotAcknowledged() { }).when(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); PauseFollowerIndexStep step = new PauseFollowerIndexStep(randomStepKey(), randomStepKey(), client); - Exception e = expectThrows( - Exception.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)) - ); + Exception e = expectThrows(Exception.class, () -> performActionAndWait(step, indexMetadata, clusterState, null)); assertThat(e.getMessage(), is("pause follow request failed to be acknowledged")); } @@ -102,13 +98,7 @@ public void testPauseFollowingIndexFailed() { }).when(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); PauseFollowerIndexStep step = new PauseFollowerIndexStep(randomStepKey(), randomStepKey(), client); - assertSame( - error, - expectThrows( - Exception.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)) - ) - ); + assertSame(error, expectThrows(Exception.class, () -> performActionAndWait(step, indexMetadata, clusterState, null))); Mockito.verify(client).execute(Mockito.same(PauseFollowAction.INSTANCE), Mockito.any(), Mockito.any()); Mockito.verifyNoMoreInteractions(client); @@ -134,7 +124,7 @@ public final void testNoShardFollowPersistentTasks() throws Exception { PauseFollowerIndexStep step = newInstance(randomStepKey(), randomStepKey()); - PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)); + performActionAndWait(step, indexMetadata, clusterState, null); Mockito.verifyNoMoreInteractions(client); } @@ -157,7 +147,7 @@ public final void testNoShardFollowTasksForManagedIndex() throws Exception { .build(); PauseFollowerIndexStep step = newInstance(randomStepKey(), randomStepKey()); - PlainActionFuture.get(f -> step.performAction(managedIndex, clusterState, null, f)); + performActionAndWait(step, managedIndex, clusterState, null); Mockito.verifyNoMoreInteractions(client); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RolloverStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RolloverStepTests.java index f25a862362540..4af25d094f5fe 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RolloverStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RolloverStepTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverInfo; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; @@ -87,7 +86,7 @@ public void testPerformAction() throws Exception { mockClientRolloverCall(alias); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexMetadata, true)).build(); - PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)); + performActionAndWait(step, indexMetadata, clusterState, null); Mockito.verify(client, Mockito.only()).admin(); Mockito.verify(adminClient, Mockito.only()).indices(); @@ -122,7 +121,7 @@ public void testPerformActionOnDataStream() throws Exception { .build(); boolean useFailureStore = randomBoolean(); IndexMetadata indexToOperateOn = useFailureStore ? failureIndexMetadata : indexMetadata; - PlainActionFuture.get(f -> step.performAction(indexToOperateOn, clusterState, null, f)); + performActionAndWait(step, indexToOperateOn, clusterState, null); Mockito.verify(client, Mockito.only()).admin(); Mockito.verify(adminClient, Mockito.only()).indices(); @@ -173,7 +172,7 @@ public void testSkipRolloverIfDataStreamIsAlreadyRolledOver() throws Exception { .build(); boolean useFailureStore = randomBoolean(); IndexMetadata indexToOperateOn = useFailureStore ? failureFirstGenerationIndex : firstGenerationIndex; - PlainActionFuture.get(f -> step.performAction(indexToOperateOn, clusterState, null, f)); + performActionAndWait(step, indexToOperateOn, clusterState, null); verifyNoMoreInteractions(client); verifyNoMoreInteractions(adminClient); @@ -206,7 +205,7 @@ public void testPerformActionWithIndexingComplete() throws Exception { RolloverStep step = createRandomInstance(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexMetadata, true)).build(); - PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)); + performActionAndWait(step, indexMetadata, clusterState, null); } public void testPerformActionSkipsRolloverForAlreadyRolledIndex() throws Exception { @@ -227,7 +226,7 @@ public void testPerformActionSkipsRolloverForAlreadyRolledIndex() throws Excepti RolloverStep step = createRandomInstance(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexMetadata, true)).build(); - PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)); + performActionAndWait(step, indexMetadata, clusterState, null); Mockito.verify(indicesClient, Mockito.never()).rolloverIndex(Mockito.any(), Mockito.any()); } @@ -248,13 +247,7 @@ public void testPerformActionFailure() { }).when(indicesClient).rolloverIndex(Mockito.any(), Mockito.any()); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexMetadata, true)).build(); - assertSame( - exception, - expectThrows( - Exception.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)) - ) - ); + assertSame(exception, expectThrows(Exception.class, () -> performActionAndWait(step, indexMetadata, clusterState, null))); Mockito.verify(client, Mockito.only()).admin(); Mockito.verify(adminClient, Mockito.only()).indices(); @@ -271,10 +264,7 @@ public void testPerformActionInvalidNullOrEmptyAlias() { RolloverStep step = createRandomInstance(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexMetadata, true)).build(); - Exception e = expectThrows( - IllegalArgumentException.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)) - ); + Exception e = expectThrows(IllegalArgumentException.class, () -> performActionAndWait(step, indexMetadata, clusterState, null)); assertThat( e.getMessage(), Matchers.is( @@ -299,10 +289,7 @@ public void testPerformActionAliasDoesNotPointToIndex() { RolloverStep step = createRandomInstance(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexMetadata, true)).build(); - Exception e = expectThrows( - IllegalArgumentException.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)) - ); + Exception e = expectThrows(IllegalArgumentException.class, () -> performActionAndWait(step, indexMetadata, clusterState, null)); assertThat( e.getMessage(), Matchers.is( diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SetSingleNodeAllocateStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SetSingleNodeAllocateStepTests.java index 78dcaa1be92af..1fb7b7c36827e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SetSingleNodeAllocateStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SetSingleNodeAllocateStepTests.java @@ -9,7 +9,6 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.transport.NoNodeAvailableException; import org.elasticsearch.cluster.ClusterState; @@ -231,10 +230,7 @@ public void testPerformActionWithClusterExcludeFilters() throws IOException { SetSingleNodeAllocateStep step = createRandomInstance(); - expectThrows( - NoNodeAvailableException.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)) - ); + expectThrows(NoNodeAvailableException.class, () -> performActionAndWait(step, indexMetadata, clusterState, null)); Mockito.verifyNoMoreInteractions(client); } @@ -331,13 +327,7 @@ public void testPerformActionAttrsRequestFails() { return null; }).when(indicesClient).updateSettings(Mockito.any(), Mockito.any()); - assertSame( - exception, - expectThrows( - Exception.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)) - ) - ); + assertSame(exception, expectThrows(Exception.class, () -> performActionAndWait(step, indexMetadata, clusterState, null))); Mockito.verify(client, Mockito.only()).admin(); Mockito.verify(adminClient, Mockito.only()).indices(); @@ -389,7 +379,7 @@ public void testPerformActionAttrsNoShard() { IndexNotFoundException e = expectThrows( IndexNotFoundException.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)) + () -> performActionAndWait(step, indexMetadata, clusterState, null) ); assertEquals(indexMetadata.getIndex(), e.getIndex()); @@ -676,7 +666,7 @@ private void assertNodeSelected( return null; }).when(indicesClient).updateSettings(Mockito.any(), Mockito.any()); - PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)); + performActionAndWait(step, indexMetadata, clusterState, null); Mockito.verify(client, Mockito.only()).admin(); Mockito.verify(adminClient, Mockito.only()).indices(); @@ -701,10 +691,7 @@ private void assertNoValidNode(IndexMetadata indexMetadata, Index index, Discove SetSingleNodeAllocateStep step = createRandomInstance(); - expectThrows( - NoNodeAvailableException.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, clusterState, null, f)) - ); + expectThrows(NoNodeAvailableException.class, () -> performActionAndWait(step, indexMetadata, clusterState, null)); Mockito.verifyNoMoreInteractions(client); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrinkSetAliasStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrinkSetAliasStepTests.java index d12cd17d957d4..7a03343b461de 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrinkSetAliasStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrinkSetAliasStepTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.index.IndexVersion; @@ -95,7 +94,7 @@ public void testPerformAction() throws Exception { return null; }).when(indicesClient).aliases(Mockito.any(), Mockito.any()); - PlainActionFuture.get(f -> step.performAction(indexMetadata, emptyClusterState(), null, f)); + performActionAndWait(step, indexMetadata, emptyClusterState(), null); Mockito.verify(client, Mockito.only()).admin(); Mockito.verify(adminClient, Mockito.only()).indices(); @@ -118,13 +117,7 @@ public void testPerformActionFailure() { return null; }).when(indicesClient).aliases(Mockito.any(), Mockito.any()); - assertSame( - exception, - expectThrows( - Exception.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, emptyClusterState(), null, f)) - ) - ); + assertSame(exception, expectThrows(Exception.class, () -> performActionAndWait(step, indexMetadata, emptyClusterState(), null))); Mockito.verify(client, Mockito.only()).admin(); Mockito.verify(adminClient, Mockito.only()).indices(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrinkStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrinkStepTests.java index db8ac28dd1b98..257df32b4d950 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrinkStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrinkStepTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -123,7 +122,7 @@ public void testPerformAction() throws Exception { return null; }).when(indicesClient).resizeIndex(Mockito.any(), Mockito.any()); - PlainActionFuture.get(f -> step.performAction(sourceIndexMetadata, emptyClusterState(), null, f)); + performActionAndWait(step, sourceIndexMetadata, emptyClusterState(), null); Mockito.verify(client, Mockito.only()).admin(); Mockito.verify(adminClient, Mockito.only()).indices(); @@ -186,7 +185,7 @@ public void testPerformActionIsCompleteForUnAckedRequests() throws Exception { return null; }).when(indicesClient).resizeIndex(Mockito.any(), Mockito.any()); - PlainActionFuture.get(f -> step.performAction(indexMetadata, emptyClusterState(), null, f)); + performActionAndWait(step, indexMetadata, emptyClusterState(), null); Mockito.verify(client, Mockito.only()).admin(); Mockito.verify(adminClient, Mockito.only()).indices(); @@ -212,13 +211,7 @@ public void testPerformActionFailure() throws Exception { return null; }).when(indicesClient).resizeIndex(Mockito.any(), Mockito.any()); - assertSame( - exception, - expectThrows( - Exception.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, emptyClusterState(), null, f)) - ) - ); + assertSame(exception, expectThrows(Exception.class, () -> performActionAndWait(step, indexMetadata, emptyClusterState(), null))); Mockito.verify(client, Mockito.only()).admin(); Mockito.verify(adminClient, Mockito.only()).indices(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/UnfollowFollowerIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/UnfollowFollowerIndexStepTests.java index 1e0a9d7ec6dd8..71f7ea2925f16 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/UnfollowFollowerIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/UnfollowFollowerIndexStepTests.java @@ -8,7 +8,6 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.index.IndexVersion; @@ -46,7 +45,7 @@ public void testUnFollow() throws Exception { }).when(client).execute(Mockito.same(UnfollowAction.INSTANCE), Mockito.any(), Mockito.any()); UnfollowFollowerIndexStep step = new UnfollowFollowerIndexStep(randomStepKey(), randomStepKey(), client); - PlainActionFuture.get(f -> step.performAction(indexMetadata, null, null, f)); + performActionAndWait(step, indexMetadata, null, null); } public void testRequestNotAcknowledged() { @@ -65,10 +64,7 @@ public void testRequestNotAcknowledged() { }).when(client).execute(Mockito.same(UnfollowAction.INSTANCE), Mockito.any(), Mockito.any()); UnfollowFollowerIndexStep step = new UnfollowFollowerIndexStep(randomStepKey(), randomStepKey(), client); - Exception e = expectThrows( - Exception.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, null, null, f)) - ); + Exception e = expectThrows(Exception.class, () -> performActionAndWait(step, indexMetadata, null, null)); assertThat(e.getMessage(), is("unfollow request failed to be acknowledged")); } @@ -91,13 +87,7 @@ public void testUnFollowUnfollowFailed() { }).when(client).execute(Mockito.same(UnfollowAction.INSTANCE), Mockito.any(), Mockito.any()); UnfollowFollowerIndexStep step = new UnfollowFollowerIndexStep(randomStepKey(), randomStepKey(), client); - assertSame( - error, - expectThrows( - RuntimeException.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, null, null, f)) - ) - ); + assertSame(error, expectThrows(RuntimeException.class, () -> performActionAndWait(step, indexMetadata, null, null))); } public void testFailureToReleaseRetentionLeases() throws Exception { @@ -120,6 +110,6 @@ public void testFailureToReleaseRetentionLeases() throws Exception { }).when(client).execute(Mockito.same(UnfollowAction.INSTANCE), Mockito.any(), Mockito.any()); UnfollowFollowerIndexStep step = new UnfollowFollowerIndexStep(randomStepKey(), randomStepKey(), client); - PlainActionFuture.get(f -> step.performAction(indexMetadata, null, null, f)); + performActionAndWait(step, indexMetadata, null, null); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/UpdateSettingsStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/UpdateSettingsStepTests.java index e71fe8720e2c7..750ab67a9e1a9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/UpdateSettingsStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/UpdateSettingsStepTests.java @@ -8,7 +8,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; @@ -58,7 +57,7 @@ private static IndexMetadata getIndexMetadata() { .build(); } - public void testPerformAction() { + public void testPerformAction() throws Exception { IndexMetadata indexMetadata = getIndexMetadata(); UpdateSettingsStep step = createRandomInstance(); @@ -73,7 +72,7 @@ public void testPerformAction() { return null; }).when(indicesClient).updateSettings(Mockito.any(), Mockito.any()); - assertNull(PlainActionFuture.get(f -> step.performAction(indexMetadata, emptyClusterState(), null, f))); + performActionAndWait(step, indexMetadata, emptyClusterState(), null); Mockito.verify(client, Mockito.only()).admin(); Mockito.verify(adminClient, Mockito.only()).indices(); @@ -95,13 +94,7 @@ public void testPerformActionFailure() { return null; }).when(indicesClient).updateSettings(Mockito.any(), Mockito.any()); - assertSame( - exception, - expectThrows( - Exception.class, - () -> PlainActionFuture.get(f -> step.performAction(indexMetadata, emptyClusterState(), null, f)) - ) - ); + assertSame(exception, expectThrows(Exception.class, () -> performActionAndWait(step, indexMetadata, emptyClusterState(), null))); Mockito.verify(client, Mockito.only()).admin(); Mockito.verify(adminClient, Mockito.only()).indices(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/InferenceRequestStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/InferenceRequestStatsTests.java new file mode 100644 index 0000000000000..518612b1b0397 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/InferenceRequestStatsTests.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.inference; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; + +import java.io.IOException; + +import static org.hamcrest.Matchers.is; + +public class InferenceRequestStatsTests extends AbstractBWCWireSerializationTestCase { + + public static InferenceRequestStats createRandom() { + var modelId = randomBoolean() ? randomAlphaOfLength(10) : null; + + return new InferenceRequestStats(randomAlphaOfLength(10), randomFrom(TaskType.values()), modelId, randomInt()); + } + + public void testToXContent_DoesNotWriteModelId_WhenItIsNull() throws IOException { + var stats = new InferenceRequestStats("service", TaskType.TEXT_EMBEDDING, null, 1); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + stats.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, is(""" + {"service":"service","task_type":"text_embedding","count":1}""")); + } + + public void testToXContent_WritesModelId_WhenItIsDefined() throws IOException { + var stats = new InferenceRequestStats("service", TaskType.TEXT_EMBEDDING, "model_id", 2); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + stats.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, is(""" + {"service":"service","task_type":"text_embedding","count":2,"model_id":"model_id"}""")); + } + + @Override + protected InferenceRequestStats mutateInstanceForVersion(InferenceRequestStats instance, TransportVersion version) { + return instance; + } + + @Override + protected Writeable.Reader instanceReader() { + return InferenceRequestStats::new; + } + + @Override + protected InferenceRequestStats createTestInstance() { + return createRandom(); + } + + @Override + protected InferenceRequestStats mutateInstance(InferenceRequestStats instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/InferenceActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/InferenceActionRequestTests.java index d4d4146c6a5ba..f41e117e75b9f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/InferenceActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/InferenceActionRequestTests.java @@ -315,7 +315,7 @@ protected InferenceAction.Request mutateInstanceForVersion(InferenceAction.Reque InputType.UNSPECIFIED, InferenceAction.Request.DEFAULT_TIMEOUT ); - } else if (version.before(TransportVersions.ML_INFERENCE_COHERE_RERANK)) { + } else if (version.before(TransportVersions.V_8_14_0)) { return new InferenceAction.Request( instance.getTaskType(), instance.getInferenceEntityId(), @@ -325,16 +325,6 @@ protected InferenceAction.Request mutateInstanceForVersion(InferenceAction.Reque instance.getInputType(), InferenceAction.Request.DEFAULT_TIMEOUT ); - } else if (version.before(TransportVersions.ML_INFERENCE_TIMEOUT_ADDED)) { - return new InferenceAction.Request( - instance.getTaskType(), - instance.getInferenceEntityId(), - instance.getQuery(), - instance.getInput(), - instance.getTaskSettings(), - instance.getInputType(), - InferenceAction.Request.DEFAULT_TIMEOUT - ); } return instance; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AdaptiveAllocationSettingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AdaptiveAllocationSettingsTests.java new file mode 100644 index 0000000000000..c86648f10f08b --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AdaptiveAllocationSettingsTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ml.inference.assignment; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; + +public class AdaptiveAllocationSettingsTests extends AbstractWireSerializingTestCase { + + public static AdaptiveAllocationsSettings testInstance() { + return new AdaptiveAllocationsSettings( + randomBoolean() ? null : randomBoolean(), + randomBoolean() ? null : randomIntBetween(1, 2), + randomBoolean() ? null : randomIntBetween(2, 4) + ); + } + + public static AdaptiveAllocationsSettings mutate(AdaptiveAllocationsSettings instance) { + boolean mutatedEnabled = Boolean.FALSE.equals(instance.getEnabled()); + return new AdaptiveAllocationsSettings(mutatedEnabled, instance.getMinNumberOfAllocations(), instance.getMaxNumberOfAllocations()); + } + + @Override + protected Writeable.Reader instanceReader() { + return AdaptiveAllocationsSettings::new; + } + + @Override + protected AdaptiveAllocationsSettings createTestInstance() { + return testInstance(); + } + + @Override + protected AdaptiveAllocationsSettings mutateInstance(AdaptiveAllocationsSettings instance) throws IOException { + return mutate(instance); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandlerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandlerTests.java index 46cb1b8e66930..b8b3087fd72b8 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandlerTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandlerTests.java @@ -33,7 +33,7 @@ public class DefaultAuthenticationFailureHandlerTests extends ESTestCase { public void testAuthenticationRequired() { final boolean testDefault = randomBoolean(); - final String basicAuthScheme = "Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\""; + final String basicAuthScheme = "Basic realm=\"" + XPackField.SECURITY + "\", charset=\"UTF-8\""; final String bearerAuthScheme = "Bearer realm=\"" + XPackField.SECURITY + "\""; final DefaultAuthenticationFailureHandler failureHandler; if (testDefault) { @@ -69,7 +69,7 @@ public void testMissingToken() { } public void testExceptionProcessingRequest() { - final String basicAuthScheme = "Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\""; + final String basicAuthScheme = "Basic realm=\"" + XPackField.SECURITY + "\", charset=\"UTF-8\""; final String bearerAuthScheme = "Bearer realm=\"" + XPackField.SECURITY + "\""; final String negotiateAuthScheme = randomFrom("Negotiate", "Negotiate Ijoijksdk"); final Map> failureResponseHeaders = new HashMap<>(); @@ -134,7 +134,7 @@ public void testExceptionProcessingRequest() { } public void testSortsWWWAuthenticateHeaderValues() { - final String basicAuthScheme = "Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\""; + final String basicAuthScheme = "Basic realm=\"" + XPackField.SECURITY + "\", charset=\"UTF-8\""; final String bearerAuthScheme = "Bearer realm=\"" + XPackField.SECURITY + "\""; final String negotiateAuthScheme = randomFrom("Negotiate", "Negotiate Ijoijksdk"); final String apiKeyAuthScheme = "ApiKey"; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/SecurityAssertions.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/SecurityAssertions.java index 3b40b96d26e10..cb989a970332d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/SecurityAssertions.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/SecurityAssertions.java @@ -22,6 +22,6 @@ public static void assertContainsWWWAuthenticateHeader(ElasticsearchSecurityExce assertThat(e.status(), is(RestStatus.UNAUTHORIZED)); assertThat(e.getHeaderKeys(), hasSize(1)); assertThat(e.getHeader("WWW-Authenticate"), notNullValue()); - assertThat(e.getHeader("WWW-Authenticate"), contains("Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\"")); + assertThat(e.getHeader("WWW-Authenticate"), contains("Basic realm=\"" + XPackField.SECURITY + "\", charset=\"UTF-8\"")); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/action/RestTermsEnumActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/action/RestTermsEnumActionTests.java index 2ea372a84b66c..9df49c72baee0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/action/RestTermsEnumActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/action/RestTermsEnumActionTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; @@ -71,7 +72,8 @@ public static void stubTermEnumAction() { final TransportAction transportAction = new TransportAction<>( TermsEnumAction.NAME, new ActionFilters(Collections.emptySet()), - taskManager + taskManager, + EsExecutors.DIRECT_EXECUTOR_SERVICE ) { @Override protected void doExecute(Task task, ActionRequest request, ActionListener listener) {} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json deleted file mode 100644 index 167efbd3ffaf5..0000000000000 --- a/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "template": { - "mappings": { - "date_detection": false, - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "keyword" - }, - "data_stream.type": { - "type": "constant_keyword", - "value": "logs" - }, - "data_stream.dataset": { - "type": "constant_keyword" - }, - "data_stream.namespace": { - "type": "constant_keyword" - } - } - } - }, - "_meta": { - "description": "default mappings for the logs index template installed by x-pack", - "managed": true - }, - "version": ${xpack.stack.template.version}, - "deprecated": ${xpack.stack.template.deprecated} -} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json index 240abf9934db5..e9a9f2611ad7b 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json @@ -5,7 +5,7 @@ "lifecycle": { "name": "logs" }, - "mode": "${xpack.stack.template.logs.index.mode}", + "mode": "${xpack.stack.template.logsdb.index.mode}", "codec": "best_compression", "mapping": { "ignore_malformed": true, diff --git a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-symbols.json b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-symbols.json index 9271718bd27ed..a23fa60021a05 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-symbols.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-symbols.json @@ -72,11 +72,8 @@ "store": false }, /* - pairs of (32bit PC offset, 32bit line number) followed by 64bit PC range base at the end. - To find line number for a given PC: find lowest offset such as offsetBase+PC >= offset, then read corresponding line number. - offsetBase could seemingly be available from exec_pc_range (it's the first value of the pair), but it's not the case. - Ranges are stored as points, which cannot be retrieve when disabling _source. - See https://www.elastic.co/guide/en/elasticsearch/reference/current/point.html . + To find the line number for a given address: find the first offset in Symbol.linetable.offsets such that offset <= base+address, + then read corresponding line number (at the same index) in Symbol.linetable.lines. Linetable: base for offsets (64bit PC range base) */ diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java index 5e6f8b6b5b18e..e66c88f70a93e 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java @@ -315,7 +315,8 @@ public TA( IndicesService indicesService, DownsampleMetrics downsampleMetrics ) { - super(NAME, actionFilters, transportService.getTaskManager()); + // TODO: consider moving to Downsample.DOWSAMPLE_TASK_THREAD_POOL_NAME and simplify realNodeOperation + super(NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.client = client; this.indicesService = indicesService; this.downsampleMetrics = downsampleMetrics; diff --git a/x-pack/plugin/enrich/qa/rest/build.gradle b/x-pack/plugin/enrich/qa/rest/build.gradle index e8473d15ed9ef..fdaddbc1f9290 100644 --- a/x-pack/plugin/enrich/qa/rest/build.gradle +++ b/x-pack/plugin/enrich/qa/rest/build.gradle @@ -8,7 +8,7 @@ import org.elasticsearch.gradle.internal.info.BuildParams restResources { restApi { - include '_common', 'bulk', 'indices', 'index', 'ingest.delete_pipeline', 'ingest.put_pipeline', 'enrich', 'get' + include '_common', 'bulk', 'indices', 'index', 'ingest.delete_pipeline', 'ingest.put_pipeline', 'enrich', 'get', 'capabilities' } restTests { includeXpack 'enrich' diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java index e36707b0b8bc4..35c2071188864 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java @@ -50,10 +50,11 @@ */ public final class EnrichCache { - private final Cache>> cache; + private final Cache cache; private final LongSupplier relativeNanoTimeProvider; private final AtomicLong hitsTimeInNanos = new AtomicLong(0); private final AtomicLong missesTimeInNanos = new AtomicLong(0); + private final AtomicLong sizeInBytes = new AtomicLong(0); private volatile Metadata metadata; EnrichCache(long maxSize) { @@ -63,7 +64,9 @@ public final class EnrichCache { // non-private for unit testing only EnrichCache(long maxSize, LongSupplier relativeNanoTimeProvider) { this.relativeNanoTimeProvider = relativeNanoTimeProvider; - this.cache = CacheBuilder.>>builder().setMaximumWeight(maxSize).build(); + this.cache = CacheBuilder.builder().setMaximumWeight(maxSize).removalListener(notification -> { + sizeInBytes.getAndAdd(-1 * notification.getValue().sizeInBytes); + }).build(); } /** @@ -86,12 +89,11 @@ public void computeIfAbsent( hitsTimeInNanos.addAndGet(cacheRequestTime); listener.onResponse(response); } else { - final long retrieveStart = relativeNanoTimeProvider.getAsLong(); searchResponseFetcher.accept(searchRequest, ActionListener.wrap(resp -> { - List> value = toCacheValue(resp); + CacheValue value = toCacheValue(resp); put(searchRequest, value); - List> copy = deepCopy(value, false); + List> copy = deepCopy(value.hits, false); long databaseQueryAndCachePutTime = relativeNanoTimeProvider.getAsLong() - retrieveStart; missesTimeInNanos.addAndGet(cacheRequestTime + databaseQueryAndCachePutTime); listener.onResponse(copy); @@ -104,20 +106,21 @@ public void computeIfAbsent( String enrichIndex = getEnrichIndexKey(searchRequest); CacheKey cacheKey = new CacheKey(enrichIndex, searchRequest); - List> response = cache.get(cacheKey); + CacheValue response = cache.get(cacheKey); if (response != null) { - return deepCopy(response, false); + return deepCopy(response.hits, false); } else { return null; } } // non-private for unit testing only - void put(SearchRequest searchRequest, List> response) { + void put(SearchRequest searchRequest, CacheValue cacheValue) { String enrichIndex = getEnrichIndexKey(searchRequest); CacheKey cacheKey = new CacheKey(enrichIndex, searchRequest); - cache.put(cacheKey, response); + cache.put(cacheKey, cacheValue); + sizeInBytes.addAndGet(cacheValue.sizeInBytes); } void setMetadata(Metadata metadata) { @@ -133,7 +136,8 @@ public EnrichStatsAction.Response.CacheStats getStats(String localNodeId) { cacheStats.getMisses(), cacheStats.getEvictions(), TimeValue.nsecToMSec(hitsTimeInNanos.get()), - TimeValue.nsecToMSec(missesTimeInNanos.get()) + TimeValue.nsecToMSec(missesTimeInNanos.get()), + sizeInBytes.get() ); } @@ -146,12 +150,14 @@ private String getEnrichIndexKey(SearchRequest searchRequest) { return ia.getIndices().get(0).getName(); } - static List> toCacheValue(SearchResponse response) { + static CacheValue toCacheValue(SearchResponse response) { List> result = new ArrayList<>(response.getHits().getHits().length); + long size = 0; for (SearchHit hit : response.getHits()) { result.add(deepCopy(hit.getSourceAsMap(), true)); + size += hit.getSourceRef() != null ? hit.getSourceRef().ramBytesUsed() : 0; } - return Collections.unmodifiableList(result); + return new CacheValue(Collections.unmodifiableList(result), size); } @SuppressWarnings("unchecked") @@ -205,4 +211,6 @@ public int hashCode() { } } + // Visibility for testing + record CacheValue(List> hits, Long sizeInBytes) {} } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java index 5cb9c0cf9c051..ca00f49100279 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java @@ -85,6 +85,8 @@ public class EnrichPolicyRunner implements Runnable { static final String ENRICH_MATCH_FIELD_NAME = "enrich_match_field"; static final String ENRICH_README_FIELD_NAME = "enrich_readme"; + public static final String ENRICH_MIN_NUMBER_OF_REPLICAS_NAME = "enrich.min_number_of_replicas"; + static final String ENRICH_INDEX_README_TEXT = "This index is managed by Elasticsearch and should not be modified in any way."; private final String policyName; @@ -137,7 +139,7 @@ public void run() { // This call does not set the origin to ensure that the user executing the policy has permission to access the source index client.admin().indices().getIndex(getIndexRequest, listener.delegateFailureAndWrap((l, getIndexResponse) -> { validateMappings(getIndexResponse); - prepareAndCreateEnrichIndex(toMappings(getIndexResponse)); + prepareAndCreateEnrichIndex(toMappings(getIndexResponse), clusterService.getSettings()); })); } catch (Exception e) { listener.onFailure(e); @@ -434,10 +436,11 @@ static boolean isIndexableField(MapperService mapperService, String field, Strin } } - private void prepareAndCreateEnrichIndex(List> mappings) { + private void prepareAndCreateEnrichIndex(List> mappings, Settings settings) { + int numberOfReplicas = settings.getAsInt(ENRICH_MIN_NUMBER_OF_REPLICAS_NAME, 0); Settings enrichIndexSettings = Settings.builder() .put("index.number_of_shards", 1) - .put("index.number_of_replicas", 0) + .put("index.number_of_replicas", numberOfReplicas) // No changes will be made to an enrich index after policy execution, so need to enable automatic refresh interval: .put("index.refresh_interval", -1) // This disables eager global ordinals loading for all fields: diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestEnrichStatsAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestEnrichStatsAction.java index 3d64e7c1380fe..2c78556df489d 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestEnrichStatsAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestEnrichStatsAction.java @@ -16,12 +16,15 @@ import org.elasticsearch.xpack.core.enrich.action.EnrichStatsAction; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; @ServerlessScope(Scope.INTERNAL) public class RestEnrichStatsAction extends BaseRestHandler { + private static final Set SUPPORTED_CAPABILITIES = Set.of("size-in-bytes"); + @Override public List routes() { return List.of(new Route(GET, "/_enrich/_stats")); @@ -32,6 +35,11 @@ public String getName() { return "enrich_stats"; } + @Override + public Set supportedCapabilities() { + return SUPPORTED_CAPABILITIES; + } + @Override protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) { final var request = new EnrichStatsAction.Request(RestUtils.getMasterNodeTimeout(restRequest)); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichCacheTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichCacheTests.java index f2f2948db41ee..19af929017a3b 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichCacheTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichCacheTests.java @@ -79,7 +79,7 @@ public void testCaching() { new SearchSourceBuilder().query(new MatchQueryBuilder("match_field", "2")) ); // Emulated search response (content doesn't matter, since it isn't used, it just a cache entry) - List> searchResponse = List.of(Map.of("test", "entry")); + EnrichCache.CacheValue searchResponse = new EnrichCache.CacheValue(List.of(Map.of("test", "entry")), 1L); EnrichCache enrichCache = new EnrichCache(3); enrichCache.setMetadata(metadata); @@ -91,6 +91,7 @@ public void testCaching() { assertThat(cacheStats.hits(), equalTo(0L)); assertThat(cacheStats.misses(), equalTo(0L)); assertThat(cacheStats.evictions(), equalTo(0L)); + assertThat(cacheStats.cacheSizeInBytes(), equalTo(3L)); assertThat(enrichCache.get(searchRequest1), notNullValue()); assertThat(enrichCache.get(searchRequest2), notNullValue()); @@ -101,6 +102,7 @@ public void testCaching() { assertThat(cacheStats.hits(), equalTo(3L)); assertThat(cacheStats.misses(), equalTo(1L)); assertThat(cacheStats.evictions(), equalTo(0L)); + assertThat(cacheStats.cacheSizeInBytes(), equalTo(3L)); enrichCache.put(searchRequest4, searchResponse); cacheStats = enrichCache.getStats("_id"); @@ -108,6 +110,7 @@ public void testCaching() { assertThat(cacheStats.hits(), equalTo(3L)); assertThat(cacheStats.misses(), equalTo(1L)); assertThat(cacheStats.evictions(), equalTo(1L)); + assertThat(cacheStats.cacheSizeInBytes(), equalTo(3L)); // Simulate enrich policy execution, which should make current cache entries unused. metadata = Metadata.builder() @@ -149,6 +152,7 @@ public void testCaching() { assertThat(cacheStats.hits(), equalTo(6L)); assertThat(cacheStats.misses(), equalTo(6L)); assertThat(cacheStats.evictions(), equalTo(4L)); + assertThat(cacheStats.cacheSizeInBytes(), equalTo(3L)); } public void testComputeIfAbsent() throws InterruptedException { @@ -331,7 +335,7 @@ public void testEnrichIndexNotExist() { new SearchSourceBuilder().query(new MatchQueryBuilder("test", "query")) ); // Emulated search response (content doesn't matter, since it isn't used, it just a cache entry) - List> searchResponse = List.of(Map.of("test", "entry")); + EnrichCache.CacheValue searchResponse = new EnrichCache.CacheValue(List.of(Map.of("test", "entry")), 1L); EnrichCache enrichCache = new EnrichCache(1); enrichCache.setMetadata(metadata); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/EnrichStatsResponseTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/EnrichStatsResponseTests.java index 14e3008cda02f..aec184472d41e 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/EnrichStatsResponseTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/EnrichStatsResponseTests.java @@ -51,6 +51,7 @@ protected EnrichStatsAction.Response createTestInstance() { randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong(), randomNonNegativeLong() ) ); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/monitoring/collector/enrich/EnrichStatsCollectorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/monitoring/collector/enrich/EnrichStatsCollectorTests.java index a38b2605c1ff0..2a069eb596760 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/monitoring/collector/enrich/EnrichStatsCollectorTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/monitoring/collector/enrich/EnrichStatsCollectorTests.java @@ -93,6 +93,7 @@ public void testDoCollect() throws Exception { randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong(), randomNonNegativeLong() ) ); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java index 0f7d92564c8ab..a3bc7ea621d8a 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java @@ -29,6 +29,10 @@ * - nestedParent - if nested, what's the parent (which might not be the immediate one) */ public class FieldAttribute extends TypedAttribute { + // TODO: This constant should not be used if possible; use .synthetic() + // https://github.com/elastic/elasticsearch/issues/105821 + public static final String SYNTHETIC_ATTRIBUTE_NAME_PREFIX = "$$"; + static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Attribute.class, "FieldAttribute", @@ -72,12 +76,11 @@ public FieldAttribute( boolean synthetic ) { super(source, name, type, qualifier, nullability, id, synthetic); - this.path = parent != null ? parent.name() : StringUtils.EMPTY; + this.path = parent != null ? parent.fieldName() : StringUtils.EMPTY; this.parent = parent; this.field = field; } - @SuppressWarnings("unchecked") public FieldAttribute(StreamInput in) throws IOException { /* * The funny casting dance with `(StreamInput & PlanStreamInput) in` is required @@ -131,6 +134,20 @@ public String path() { return path; } + /** + * The full name of the field in the index, including all parent fields. E.g. {@code parent.subfield.this_field}. + */ + public String fieldName() { + // Before 8.15, the field name was the same as the attribute's name. + // On later versions, the attribute can be renamed when creating synthetic attributes. + // TODO: We should use synthetic() to check for that case. + // https://github.com/elastic/elasticsearch/issues/105821 + if (name().startsWith(SYNTHETIC_ATTRIBUTE_NAME_PREFIX) == false) { + return name(); + } + return Strings.hasText(path) ? path + "." + field.getName() : field.getName(); + } + public String qualifiedPath() { // return only the qualifier is there's no path return qualifier() != null ? qualifier() + (Strings.hasText(path) ? "." + path : StringUtils.EMPTY) : path; diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java index b6704f0569b27..fd7382b0098c9 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java @@ -13,6 +13,7 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IgnoredFieldMapper; +import org.elasticsearch.index.mapper.IndexModeFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -45,7 +46,9 @@ public class MetadataAttribute extends TypedAttribute { IgnoredFieldMapper.NAME, tuple(DataType.KEYWORD, true), SourceFieldMapper.NAME, - tuple(DataType.SOURCE, false) + tuple(DataType.SOURCE, false), + IndexModeFieldMapper.NAME, + tuple(DataType.KEYWORD, true) ); private final boolean searchable; diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/Equals.java deleted file mode 100644 index 533ce4b76b595..0000000000000 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/Equals.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison; - -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; -import org.elasticsearch.xpack.esql.core.tree.NodeInfo; -import org.elasticsearch.xpack.esql.core.tree.Source; - -import java.io.IOException; -import java.time.ZoneId; - -public class Equals extends BinaryComparison implements Negatable { - - public Equals(Source source, Expression left, Expression right) { - super(source, left, right, BinaryComparisonOperation.EQ, null); - } - - public Equals(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonOperation.EQ, zoneId); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public String getWriteableName() { - throw new UnsupportedOperationException(); - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, Equals::new, left(), right(), zoneId()); - } - - @Override - protected Equals replaceChildren(Expression newLeft, Expression newRight) { - return new Equals(source(), newLeft, newRight, zoneId()); - } - - @Override - public Equals swapLeftAndRight() { - return new Equals(source(), right(), left(), zoneId()); - } - - @Override - public BinaryComparison negate() { - return new NotEquals(source(), left(), right(), zoneId()); - } - - @Override - public BinaryComparison reverse() { - return this; - } - - @Override - protected boolean isCommutative() { - return true; - } -} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/GreaterThan.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/GreaterThan.java deleted file mode 100644 index f4ffa1a12ae5b..0000000000000 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/GreaterThan.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison; - -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; -import org.elasticsearch.xpack.esql.core.tree.NodeInfo; -import org.elasticsearch.xpack.esql.core.tree.Source; - -import java.io.IOException; -import java.time.ZoneId; - -public class GreaterThan extends BinaryComparison implements Negatable { - - public GreaterThan(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonOperation.GT, zoneId); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public String getWriteableName() { - throw new UnsupportedOperationException(); - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, GreaterThan::new, left(), right(), zoneId()); - } - - @Override - protected GreaterThan replaceChildren(Expression newLeft, Expression newRight) { - return new GreaterThan(source(), newLeft, newRight, zoneId()); - } - - @Override - public LessThan swapLeftAndRight() { - return new LessThan(source(), right(), left(), zoneId()); - } - - @Override - public LessThanOrEqual negate() { - return new LessThanOrEqual(source(), left(), right(), zoneId()); - } - - @Override - public BinaryComparison reverse() { - return new LessThan(source(), left(), right(), zoneId()); - } -} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/GreaterThanOrEqual.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/GreaterThanOrEqual.java deleted file mode 100644 index 28aa4124f0987..0000000000000 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/GreaterThanOrEqual.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison; - -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; -import org.elasticsearch.xpack.esql.core.tree.NodeInfo; -import org.elasticsearch.xpack.esql.core.tree.Source; - -import java.io.IOException; -import java.time.ZoneId; - -public class GreaterThanOrEqual extends BinaryComparison implements Negatable { - - public GreaterThanOrEqual(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonOperation.GTE, zoneId); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public String getWriteableName() { - throw new UnsupportedOperationException(); - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, GreaterThanOrEqual::new, left(), right(), zoneId()); - } - - @Override - protected GreaterThanOrEqual replaceChildren(Expression newLeft, Expression newRight) { - return new GreaterThanOrEqual(source(), newLeft, newRight, zoneId()); - } - - @Override - public LessThanOrEqual swapLeftAndRight() { - return new LessThanOrEqual(source(), right(), left(), zoneId()); - } - - @Override - public LessThan negate() { - return new LessThan(source(), left(), right(), zoneId()); - } - - @Override - public BinaryComparison reverse() { - return new LessThanOrEqual(source(), left(), right(), zoneId()); - } -} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/In.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/In.java deleted file mode 100644 index bd645064289a5..0000000000000 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/In.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison; - -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.Expressions; -import org.elasticsearch.xpack.esql.core.expression.Foldables; -import org.elasticsearch.xpack.esql.core.expression.Nullability; -import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; -import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; -import org.elasticsearch.xpack.esql.core.tree.NodeInfo; -import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.core.type.DataTypeConverter; -import org.elasticsearch.xpack.esql.core.util.CollectionUtils; - -import java.io.IOException; -import java.time.ZoneId; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Objects; - -import static org.elasticsearch.common.logging.LoggerMessageFormat.format; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; -import static org.elasticsearch.xpack.esql.core.util.StringUtils.ordinal; - -public class In extends ScalarFunction { - - private final Expression value; - private final List list; - private final ZoneId zoneId; - - public In(Source source, Expression value, List list) { - this(source, value, list, null); - } - - public In(Source source, Expression value, List list, ZoneId zoneId) { - super(source, CollectionUtils.combine(list, value)); - this.value = value; - this.list = new ArrayList<>(new LinkedHashSet<>(list)); - this.zoneId = zoneId; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public String getWriteableName() { - throw new UnsupportedOperationException(); - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, In::new, value(), list(), zoneId()); - } - - @Override - public Expression replaceChildren(List newChildren) { - return new In(source(), newChildren.get(newChildren.size() - 1), newChildren.subList(0, newChildren.size() - 1), zoneId()); - } - - public ZoneId zoneId() { - return zoneId; - } - - public Expression value() { - return value; - } - - public List list() { - return list; - } - - @Override - public DataType dataType() { - return DataType.BOOLEAN; - } - - @Override - public Nullability nullable() { - return Nullability.UNKNOWN; - } - - @Override - public boolean foldable() { - return Expressions.foldable(children()) || (Expressions.foldable(list) && list().stream().allMatch(Expressions::isNull)); - } - - @Override - public Boolean fold() { - // Optimization for early return and Query folding to LocalExec - if (Expressions.isNull(value) || list.size() == 1 && Expressions.isNull(list.get(0))) { - return null; - } - return apply(value.fold(), foldAndConvertListOfValues(list, value.dataType())); - } - - private static Boolean apply(Object input, List values) { - Boolean result = Boolean.FALSE; - for (Object v : values) { - Boolean compResult = Comparisons.eq(input, v); - if (compResult == null) { - result = null; - } else if (compResult == Boolean.TRUE) { - return Boolean.TRUE; - } - } - return result; - } - - @Override - protected Expression canonicalize() { - // order values for commutative operators - List canonicalValues = Expressions.canonicalize(list); - Collections.sort(canonicalValues, (l, r) -> Integer.compare(l.hashCode(), r.hashCode())); - return new In(source(), value, canonicalValues, zoneId); - } - - protected List foldAndConvertListOfValues(List expressions, DataType dataType) { - List values = new ArrayList<>(expressions.size()); - for (Expression e : expressions) { - values.add(DataTypeConverter.convert(Foldables.valueOf(e), dataType)); - } - return values; - } - - protected boolean areCompatible(DataType left, DataType right) { - return DataType.areCompatible(left, right); - } - - @Override - protected TypeResolution resolveType() { - TypeResolution resolution = TypeResolutions.isExact(value, functionName(), DEFAULT); - if (resolution.unresolved()) { - return resolution; - } - - for (Expression ex : list) { - if (ex.foldable() == false) { - return new TypeResolution( - format( - null, - "Comparisons against fields are not (currently) supported; offender [{}] in [{}]", - Expressions.name(ex), - sourceText() - ) - ); - } - } - - DataType dt = value.dataType(); - for (int i = 0; i < list.size(); i++) { - Expression listValue = list.get(i); - if (areCompatible(dt, listValue.dataType()) == false) { - return new TypeResolution( - format( - null, - "{} argument of [{}] must be [{}], found value [{}] type [{}]", - ordinal(i + 1), - sourceText(), - dt.typeName(), - Expressions.name(listValue), - listValue.dataType().typeName() - ) - ); - } - } - - return super.resolveType(); - } - - @Override - public int hashCode() { - return Objects.hash(value, list); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - - In other = (In) obj; - return Objects.equals(value, other.value) && Objects.equals(list, other.list); - } -} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/LessThan.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/LessThan.java deleted file mode 100644 index 150db16521480..0000000000000 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/LessThan.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison; - -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; -import org.elasticsearch.xpack.esql.core.tree.NodeInfo; -import org.elasticsearch.xpack.esql.core.tree.Source; - -import java.io.IOException; -import java.time.ZoneId; - -public class LessThan extends BinaryComparison implements Negatable { - - public LessThan(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonOperation.LT, zoneId); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public String getWriteableName() { - throw new UnsupportedOperationException(); - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, LessThan::new, left(), right(), zoneId()); - } - - @Override - protected LessThan replaceChildren(Expression newLeft, Expression newRight) { - return new LessThan(source(), newLeft, newRight, zoneId()); - } - - @Override - public GreaterThan swapLeftAndRight() { - return new GreaterThan(source(), right(), left(), zoneId()); - } - - @Override - public GreaterThanOrEqual negate() { - return new GreaterThanOrEqual(source(), left(), right(), zoneId()); - } - - @Override - public BinaryComparison reverse() { - return new GreaterThan(source(), left(), right(), zoneId()); - } -} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/LessThanOrEqual.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/LessThanOrEqual.java deleted file mode 100644 index a0e5abd4317b3..0000000000000 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/LessThanOrEqual.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison; - -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; -import org.elasticsearch.xpack.esql.core.tree.NodeInfo; -import org.elasticsearch.xpack.esql.core.tree.Source; - -import java.io.IOException; -import java.time.ZoneId; - -public class LessThanOrEqual extends BinaryComparison implements Negatable { - - public LessThanOrEqual(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonOperation.LTE, zoneId); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public String getWriteableName() { - throw new UnsupportedOperationException(); - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, LessThanOrEqual::new, left(), right(), zoneId()); - } - - @Override - protected LessThanOrEqual replaceChildren(Expression newLeft, Expression newRight) { - return new LessThanOrEqual(source(), newLeft, newRight, zoneId()); - } - - @Override - public GreaterThanOrEqual swapLeftAndRight() { - return new GreaterThanOrEqual(source(), right(), left(), zoneId()); - } - - @Override - public GreaterThan negate() { - return new GreaterThan(source(), left(), right(), zoneId()); - } - - @Override - public BinaryComparison reverse() { - return new GreaterThanOrEqual(source(), left(), right(), zoneId()); - } -} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/NotEquals.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/NotEquals.java deleted file mode 100644 index 6d52195ec9452..0000000000000 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/NotEquals.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison; - -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; -import org.elasticsearch.xpack.esql.core.tree.NodeInfo; -import org.elasticsearch.xpack.esql.core.tree.Source; - -import java.io.IOException; -import java.time.ZoneId; - -public class NotEquals extends BinaryComparison implements Negatable { - - public NotEquals(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonOperation.NEQ, zoneId); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public String getWriteableName() { - throw new UnsupportedOperationException(); - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, NotEquals::new, left(), right(), zoneId()); - } - - @Override - protected NotEquals replaceChildren(Expression newLeft, Expression newRight) { - return new NotEquals(source(), newLeft, newRight, zoneId()); - } - - @Override - public NotEquals swapLeftAndRight() { - return new NotEquals(source(), right(), left(), zoneId()); - } - - @Override - public BinaryComparison negate() { - return new Equals(source(), left(), right(), zoneId()); - } - - @Override - public BinaryComparison reverse() { - return this; - } - - @Override - protected boolean isCommutative() { - return true; - } -} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/NullEquals.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/NullEquals.java deleted file mode 100644 index bb2196a5ae3b9..0000000000000 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/NullEquals.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison; - -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.Nullability; -import org.elasticsearch.xpack.esql.core.tree.NodeInfo; -import org.elasticsearch.xpack.esql.core.tree.Source; - -import java.io.IOException; -import java.time.ZoneId; - -/** - * Implements the MySQL {@code <=>} operator - */ -public class NullEquals extends BinaryComparison { - - public NullEquals(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonOperation.NULLEQ, zoneId); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public String getWriteableName() { - throw new UnsupportedOperationException(); - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, NullEquals::new, left(), right(), zoneId()); - } - - @Override - protected NullEquals replaceChildren(Expression newLeft, Expression newRight) { - return new NullEquals(source(), newLeft, newRight, zoneId()); - } - - @Override - public NullEquals swapLeftAndRight() { - return new NullEquals(source(), right(), left(), zoneId()); - } - - @Override - public Nullability nullable() { - return Nullability.FALSE; - } - - @Override - public BinaryComparison reverse() { - return this; - } - - @Override - protected boolean isCommutative() { - return true; - } -} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java index 6eab4a0cd9a75..2df3a8eba46d5 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java @@ -7,15 +7,11 @@ package org.elasticsearch.xpack.esql.core.planner; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.time.DateFormatter; -import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; -import org.elasticsearch.xpack.esql.core.expression.TypedAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.Range; import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MatchQueryPredicate; import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MultiMatchQueryPredicate; @@ -25,15 +21,6 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.GreaterThan; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.GreaterThanOrEqual; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.In; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.LessThan; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.LessThanOrEqual; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.NotEquals; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.NullEquals; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.Like; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLike; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch; @@ -47,29 +34,16 @@ import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery; import org.elasticsearch.xpack.esql.core.querydsl.query.RangeQuery; import org.elasticsearch.xpack.esql.core.querydsl.query.RegexQuery; -import org.elasticsearch.xpack.esql.core.querydsl.query.TermQuery; -import org.elasticsearch.xpack.esql.core.querydsl.query.TermsQuery; import org.elasticsearch.xpack.esql.core.querydsl.query.WildcardQuery; import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.Check; import org.elasticsearch.xpack.esql.core.util.CollectionUtils; -import org.elasticsearch.xpack.versionfield.Version; import java.time.OffsetTime; -import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.temporal.TemporalAccessor; -import java.util.ArrayList; import java.util.Arrays; -import java.util.LinkedHashSet; import java.util.List; -import java.util.Set; - -import static org.elasticsearch.xpack.esql.core.type.DataType.IP; -import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; -import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; -import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber; public final class ExpressionTranslators { @@ -219,104 +193,6 @@ private static Query translate(IsNull isNull, TranslatorHandler handler) { } } - // assume the Optimizer properly orders the predicates to ease the translation - public static class BinaryComparisons extends ExpressionTranslator { - - @Override - protected Query asQuery(BinaryComparison bc, TranslatorHandler handler) { - return doTranslate(bc, handler); - } - - public static void checkBinaryComparison(BinaryComparison bc) { - Check.isTrue( - bc.right().foldable(), - "Line {}:{}: Comparisons against fields are not (currently) supported; offender [{}] in [{}]", - bc.right().sourceLocation().getLineNumber(), - bc.right().sourceLocation().getColumnNumber(), - Expressions.name(bc.right()), - bc.symbol() - ); - } - - public static Query doTranslate(BinaryComparison bc, TranslatorHandler handler) { - checkBinaryComparison(bc); - return handler.wrapFunctionQuery(bc, bc.left(), () -> translate(bc, handler)); - } - - static Query translate(BinaryComparison bc, TranslatorHandler handler) { - TypedAttribute attribute = checkIsPushableAttribute(bc.left()); - Source source = bc.source(); - String name = handler.nameOf(attribute); - Object value = valueOf(bc.right()); - String format = null; - boolean isDateLiteralComparison = false; - - // for a date constant comparison, we need to use a format for the date, to make sure that the format is the same - // no matter the timezone provided by the user - if (value instanceof ZonedDateTime || value instanceof OffsetTime) { - DateFormatter formatter; - if (value instanceof ZonedDateTime) { - formatter = DateFormatter.forPattern(DATE_FORMAT); - // RangeQueryBuilder accepts an Object as its parameter, but it will call .toString() on the ZonedDateTime instance - // which can have a slightly different format depending on the ZoneId used to create the ZonedDateTime - // Since RangeQueryBuilder can handle date as String as well, we'll format it as String and provide the format as well. - value = formatter.format((ZonedDateTime) value); - } else { - formatter = DateFormatter.forPattern(TIME_FORMAT); - value = formatter.format((OffsetTime) value); - } - format = formatter.pattern(); - isDateLiteralComparison = true; - } else if (attribute.dataType() == IP && value instanceof BytesRef bytesRef) { - value = DocValueFormat.IP.format(bytesRef); - } else if (attribute.dataType() == VERSION) { - // VersionStringFieldMapper#indexedValueForSearch() only accepts as input String or BytesRef with the String (i.e. not - // encoded) representation of the version as it'll do the encoding itself. - if (value instanceof BytesRef bytesRef) { - value = new Version(bytesRef).toString(); - } else if (value instanceof Version version) { - value = version.toString(); - } - } else if (attribute.dataType() == UNSIGNED_LONG && value instanceof Long ul) { - value = unsignedLongAsNumber(ul); - } - - ZoneId zoneId = null; - if (DataType.isDateTime(attribute.dataType())) { - zoneId = bc.zoneId(); - } - if (bc instanceof GreaterThan) { - return new RangeQuery(source, name, value, false, null, false, format, zoneId); - } - if (bc instanceof GreaterThanOrEqual) { - return new RangeQuery(source, name, value, true, null, false, format, zoneId); - } - if (bc instanceof LessThan) { - return new RangeQuery(source, name, null, false, value, false, format, zoneId); - } - if (bc instanceof LessThanOrEqual) { - return new RangeQuery(source, name, null, false, value, true, format, zoneId); - } - if (bc instanceof Equals || bc instanceof NullEquals || bc instanceof NotEquals) { - name = pushableAttributeName(attribute); - - Query query; - if (isDateLiteralComparison) { - // dates equality uses a range query because it's the one that has a "format" parameter - query = new RangeQuery(source, name, value, true, value, true, format, zoneId); - } else { - query = new TermQuery(source, name, value); - } - if (bc instanceof NotEquals) { - query = new NotQuery(source, query); - } - return query; - } - - throw new QlIllegalArgumentException("Don't know how to translate binary comparison [{}] in [{}]", bc.right().nodeString(), bc); - } - } - public static class Ranges extends ExpressionTranslator { @Override @@ -366,54 +242,7 @@ private static RangeQuery translate(Range r, TranslatorHandler handler) { } } - public static class InComparisons extends ExpressionTranslator { - - @Override - protected Query asQuery(In in, TranslatorHandler handler) { - return doTranslate(in, handler); - } - - public static Query doTranslate(In in, TranslatorHandler handler) { - return handler.wrapFunctionQuery(in, in.value(), () -> translate(in, handler)); - } - - private static boolean needsTypeSpecificValueHandling(DataType fieldType) { - return DataType.isDateTime(fieldType) || fieldType == IP || fieldType == VERSION || fieldType == UNSIGNED_LONG; - } - - private static Query translate(In in, TranslatorHandler handler) { - TypedAttribute attribute = checkIsPushableAttribute(in.value()); - - Set terms = new LinkedHashSet<>(); - List queries = new ArrayList<>(); - - for (Expression rhs : in.list()) { - if (DataType.isNull(rhs.dataType()) == false) { - if (needsTypeSpecificValueHandling(attribute.dataType())) { - // delegates to BinaryComparisons translator to ensure consistent handling of date and time values - Query query = BinaryComparisons.translate(new Equals(in.source(), in.value(), rhs, in.zoneId()), handler); - - if (query instanceof TermQuery) { - terms.add(((TermQuery) query).value()); - } else { - queries.add(query); - } - } else { - terms.add(valueOf(rhs)); - } - } - } - - if (terms.isEmpty() == false) { - String fieldName = pushableAttributeName(attribute); - queries.add(new TermsQuery(in.source(), fieldName, terms)); - } - - return queries.stream().reduce((q1, q2) -> or(in.source(), q1, q2)).get(); - } - } - - private static Query or(Source source, Query left, Query right) { + public static Query or(Source source, Query left, Query right) { return boolQuery(source, left, right, false); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/rule/Rule.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/rule/Rule.java index 6121c9b36442b..163b1f89f2abb 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/rule/Rule.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/rule/Rule.java @@ -6,8 +6,8 @@ */ package org.elasticsearch.xpack.esql.core.rule; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.xpack.esql.core.tree.Node; import org.elasticsearch.xpack.esql.core.util.ReflectionUtils; diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 503c076b4f7a2..cca9ba2a54b63 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -9,7 +9,9 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import java.io.IOException; import java.math.BigInteger; @@ -20,6 +22,7 @@ import java.util.Comparator; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -27,9 +30,9 @@ import static java.util.stream.Collectors.toUnmodifiableMap; public enum DataType { - UNSUPPORTED(builder().typeName("UNSUPPORTED")), - NULL(builder().esType("null")), - BOOLEAN(builder().esType("boolean").size(1)), + UNSUPPORTED(builder().typeName("UNSUPPORTED").unknownSize()), + NULL(builder().esType("null").estimatedSize(0)), + BOOLEAN(builder().esType("boolean").estimatedSize(1)), /** * These are numeric fields labeled as metric counters in time-series indices. Although stored @@ -38,37 +41,62 @@ public enum DataType { * These fields are strictly for use in retrieval from indices, rate aggregation, and casting to their * parent numeric type. */ - COUNTER_LONG(builder().esType("counter_long").size(Long.BYTES).docValues().counter()), - COUNTER_INTEGER(builder().esType("counter_integer").size(Integer.BYTES).docValues().counter()), - COUNTER_DOUBLE(builder().esType("counter_double").size(Double.BYTES).docValues().counter()), - - LONG(builder().esType("long").size(Long.BYTES).wholeNumber().docValues().counter(COUNTER_LONG)), - INTEGER(builder().esType("integer").size(Integer.BYTES).wholeNumber().docValues().counter(COUNTER_INTEGER)), - SHORT(builder().esType("short").size(Short.BYTES).wholeNumber().docValues().widenSmallNumeric(INTEGER)), - BYTE(builder().esType("byte").size(Byte.BYTES).wholeNumber().docValues().widenSmallNumeric(INTEGER)), - UNSIGNED_LONG(builder().esType("unsigned_long").size(Long.BYTES).wholeNumber().docValues()), - DOUBLE(builder().esType("double").size(Double.BYTES).rationalNumber().docValues().counter(COUNTER_DOUBLE)), - FLOAT(builder().esType("float").size(Float.BYTES).rationalNumber().docValues().widenSmallNumeric(DOUBLE)), - HALF_FLOAT(builder().esType("half_float").size(Float.BYTES).rationalNumber().docValues().widenSmallNumeric(DOUBLE)), - SCALED_FLOAT(builder().esType("scaled_float").size(Long.BYTES).rationalNumber().docValues().widenSmallNumeric(DOUBLE)), + COUNTER_LONG(builder().esType("counter_long").estimatedSize(Long.BYTES).docValues().counter()), + COUNTER_INTEGER(builder().esType("counter_integer").estimatedSize(Integer.BYTES).docValues().counter()), + COUNTER_DOUBLE(builder().esType("counter_double").estimatedSize(Double.BYTES).docValues().counter()), + + LONG(builder().esType("long").estimatedSize(Long.BYTES).wholeNumber().docValues().counter(COUNTER_LONG)), + INTEGER(builder().esType("integer").estimatedSize(Integer.BYTES).wholeNumber().docValues().counter(COUNTER_INTEGER)), + SHORT(builder().esType("short").estimatedSize(Short.BYTES).wholeNumber().docValues().widenSmallNumeric(INTEGER)), + BYTE(builder().esType("byte").estimatedSize(Byte.BYTES).wholeNumber().docValues().widenSmallNumeric(INTEGER)), + UNSIGNED_LONG(builder().esType("unsigned_long").estimatedSize(Long.BYTES).wholeNumber().docValues()), + DOUBLE(builder().esType("double").estimatedSize(Double.BYTES).rationalNumber().docValues().counter(COUNTER_DOUBLE)), + FLOAT(builder().esType("float").estimatedSize(Float.BYTES).rationalNumber().docValues().widenSmallNumeric(DOUBLE)), + HALF_FLOAT(builder().esType("half_float").estimatedSize(Float.BYTES).rationalNumber().docValues().widenSmallNumeric(DOUBLE)), + SCALED_FLOAT(builder().esType("scaled_float").estimatedSize(Long.BYTES).rationalNumber().docValues().widenSmallNumeric(DOUBLE)), KEYWORD(builder().esType("keyword").unknownSize().docValues()), TEXT(builder().esType("text").unknownSize()), - DATETIME(builder().esType("date").typeName("DATETIME").size(Long.BYTES).docValues()), - IP(builder().esType("ip").size(45).docValues()), - VERSION(builder().esType("version").unknownSize().docValues()), - OBJECT(builder().esType("object")), - NESTED(builder().esType("nested")), + DATETIME(builder().esType("date").typeName("DATETIME").estimatedSize(Long.BYTES).docValues()), + // IP addresses, both IPv4 and IPv6, are encoded using 16 bytes. + IP(builder().esType("ip").estimatedSize(16).docValues()), + // 8.15.2-SNAPSHOT is 15 bytes, most are shorter, some can be longer + VERSION(builder().esType("version").estimatedSize(15).docValues()), + OBJECT(builder().esType("object").unknownSize()), + NESTED(builder().esType("nested").unknownSize()), SOURCE(builder().esType(SourceFieldMapper.NAME).unknownSize()), - DATE_PERIOD(builder().typeName("DATE_PERIOD").size(3 * Integer.BYTES)), - TIME_DURATION(builder().typeName("TIME_DURATION").size(Integer.BYTES + Long.BYTES)), - GEO_POINT(builder().esType("geo_point").size(Double.BYTES * 2).docValues()), - CARTESIAN_POINT(builder().esType("cartesian_point").size(Double.BYTES * 2).docValues()), - CARTESIAN_SHAPE(builder().esType("cartesian_shape").unknownSize().docValues()), - GEO_SHAPE(builder().esType("geo_shape").unknownSize().docValues()), - - DOC_DATA_TYPE(builder().esType("_doc").size(Integer.BYTES * 3)), + DATE_PERIOD(builder().typeName("DATE_PERIOD").estimatedSize(3 * Integer.BYTES)), + TIME_DURATION(builder().typeName("TIME_DURATION").estimatedSize(Integer.BYTES + Long.BYTES)), + // WKB for points is typically 21 bytes. + GEO_POINT(builder().esType("geo_point").estimatedSize(21).docValues()), + CARTESIAN_POINT(builder().esType("cartesian_point").estimatedSize(21).docValues()), + // wild estimate for size, based on some test data (airport_city_boundaries) + CARTESIAN_SHAPE(builder().esType("cartesian_shape").estimatedSize(200).docValues()), + GEO_SHAPE(builder().esType("geo_shape").estimatedSize(200).docValues()), + + /** + * Fields with this type represent a Lucene doc id. This field is a bit magic in that: + *
      + *
    • One copy of it is always added at the start of every query
      • + *
    • It is implicitly dropped before being returned to the user
      • + *
    • It is not "target-able" by any functions
      • + *
    • Users shouldn't know it's there at all
      • + *
    • It is used as an input for things that interact with Lucene like + * loading field values
      • + *
    + */ + DOC_DATA_TYPE(builder().esType("_doc").estimatedSize(Integer.BYTES * 3)), + /** + * Fields with this type represent values from the {@link TimeSeriesIdFieldMapper}. + * Every document in {@link IndexMode#TIME_SERIES} index will have a single value + * for this field and the segments themselves are sorted on this value. + */ TSID_DATA_TYPE(builder().esType("_tsid").unknownSize().docValues()), + /** + * Fields with this type are the partial result of running a non-time-series aggregation + * inside alongside time-series aggregations. These fields are not parsable from the + * mapping and should be hidden from users. + */ PARTIAL_AGG(builder().esType("partial_agg").unknownSize()); private final String typeName; @@ -77,7 +105,7 @@ public enum DataType { private final String esType; - private final int size; + private final Optional estimatedSize; /** * True if the type represents a "whole number", as in, does not have a decimal part. @@ -113,10 +141,11 @@ public enum DataType { DataType(Builder builder) { String typeString = builder.typeName != null ? builder.typeName : builder.esType; + assert builder.estimatedSize != null : "Missing size for type " + typeString; this.typeName = typeString.toLowerCase(Locale.ROOT); this.name = typeString.toUpperCase(Locale.ROOT); this.esType = builder.esType; - this.size = builder.size; + this.estimatedSize = builder.estimatedSize; this.isWholeNumber = builder.isWholeNumber; this.isRationalNumber = builder.isRationalNumber; this.docValues = builder.docValues; @@ -211,8 +240,12 @@ public static boolean isString(DataType t) { return t == KEYWORD || t == TEXT; } + public static boolean isPrimitiveAndSupported(DataType t) { + return isPrimitive(t) && t != UNSUPPORTED; + } + public static boolean isPrimitive(DataType t) { - return t != OBJECT && t != NESTED && t != UNSUPPORTED; + return t != OBJECT && t != NESTED; } public static boolean isNull(DataType t) { @@ -223,25 +256,69 @@ public static boolean isNullOrNumeric(DataType t) { return t.isNumeric() || isNull(t); } - public static boolean isSigned(DataType t) { - return t.isNumeric() && t.equals(UNSIGNED_LONG) == false; - } - public static boolean isDateTime(DataType type) { return type == DATETIME; } + public static boolean isNullOrTimeDuration(DataType t) { + return t == TIME_DURATION || isNull(t); + } + + public static boolean isNullOrDatePeriod(DataType t) { + return t == DATE_PERIOD || isNull(t); + } + + public static boolean isTemporalAmount(DataType t) { + return t == DATE_PERIOD || t == TIME_DURATION; + } + + public static boolean isNullOrTemporalAmount(DataType t) { + return isTemporalAmount(t) || isNull(t); + } + + public static boolean isDateTimeOrTemporal(DataType t) { + return isDateTime(t) || isTemporalAmount(t); + } + public static boolean areCompatible(DataType left, DataType right) { if (left == right) { return true; } else { - return (left == NULL || right == NULL) - || (isString(left) && isString(right)) - || (left.isNumeric() && right.isNumeric()) - || (isDateTime(left) && isDateTime(right)); + return (left == NULL || right == NULL) || (isString(left) && isString(right)) || (left.isNumeric() && right.isNumeric()); } } + /** + * Supported types that can be contained in a block. + */ + public static boolean isRepresentable(DataType t) { + return t != OBJECT + && t != NESTED + && t != UNSUPPORTED + && t != DATE_PERIOD + && t != TIME_DURATION + && t != BYTE + && t != SHORT + && t != FLOAT + && t != SCALED_FLOAT + && t != SOURCE + && t != HALF_FLOAT + && t != PARTIAL_AGG + && t.isCounter() == false; + } + + public static boolean isSpatialPoint(DataType t) { + return t == GEO_POINT || t == CARTESIAN_POINT; + } + + public static boolean isSpatialGeo(DataType t) { + return t == GEO_POINT || t == GEO_SHAPE; + } + + public static boolean isSpatial(DataType t) { + return t == GEO_POINT || t == CARTESIAN_POINT || t == GEO_SHAPE || t == CARTESIAN_SHAPE; + } + public String nameUpper() { return name; } @@ -282,8 +359,12 @@ public boolean isNumeric() { return isWholeNumber || isRationalNumber; } - public int size() { - return size; + /** + * @return the estimated size, in bytes, of this data type. If there's no reasonable way to estimate the size, + * the optional will be empty. + */ + public Optional estimatedSize() { + return estimatedSize; } public boolean hasDocValues() { @@ -352,7 +433,7 @@ private static class Builder { private String typeName; - private int size; + private Optional estimatedSize; /** * True if the type represents a "whole number", as in, does not have a decimal part. @@ -398,13 +479,13 @@ Builder typeName(String typeName) { return this; } - Builder size(int size) { - this.size = size; + Builder estimatedSize(int size) { + this.estimatedSize = Optional.of(size); return this; } Builder unknownSize() { - this.size = Integer.MAX_VALUE; + this.estimatedSize = Optional.empty(); return this; } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java index bd87a92f3289d..0bccf3407aa2d 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java @@ -37,7 +37,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTime; -import static org.elasticsearch.xpack.esql.core.type.DataType.isPrimitive; +import static org.elasticsearch.xpack.esql.core.type.DataType.isPrimitiveAndSupported; import static org.elasticsearch.xpack.esql.core.type.DataType.isString; import static org.elasticsearch.xpack.esql.core.util.NumericUtils.UNSIGNED_LONG_MAX; import static org.elasticsearch.xpack.esql.core.util.NumericUtils.inUnsignedLongRange; @@ -77,6 +77,8 @@ public static DataType commonType(DataType left, DataType right) { return right; } if (left.isNumeric() && right.isNumeric()) { + int lsize = left.estimatedSize().orElseThrow(); + int rsize = right.estimatedSize().orElseThrow(); // if one is int if (left.isWholeNumber()) { // promote the highest int @@ -84,7 +86,7 @@ public static DataType commonType(DataType left, DataType right) { if (left == UNSIGNED_LONG || right == UNSIGNED_LONG) { return UNSIGNED_LONG; } - return left.size() > right.size() ? left : right; + return lsize > rsize ? left : right; } // promote the rational return right; @@ -94,7 +96,7 @@ public static DataType commonType(DataType left, DataType right) { return left; } // promote the highest rational - return left.size() > right.size() ? left : right; + return lsize > rsize ? left : right; } if (isString(left)) { if (right.isNumeric()) { @@ -124,7 +126,7 @@ public static boolean canConvert(DataType from, DataType to) { return true; } // only primitives are supported so far - return isPrimitive(from) && isPrimitive(to) && converterFor(from, to) != null; + return isPrimitiveAndSupported(from) && isPrimitiveAndSupported(to) && converterFor(from, to) != null; } /** diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java index 47246a4e190dd..4ba3658697c0d 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java @@ -354,6 +354,9 @@ public static Number parseIntegral(String string) throws InvalidArgumentExceptio } return bi; } + if (bi.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) { + throw new InvalidArgumentException("Magnitude of negative number [{}] is too large", string); + } // try to downsize to int if possible (since that's the most common type) if (bi.intValue() == bi.longValue()) { // ternary operator would always promote to Long return bi.intValueExact(); diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/InTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/InTests.java deleted file mode 100644 index a6abe4e923c17..0000000000000 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/InTests.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison; - -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.esql.core.TestUtils; -import org.elasticsearch.xpack.esql.core.expression.Literal; - -import java.util.Arrays; - -import static org.elasticsearch.xpack.esql.core.expression.Literal.NULL; -import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; - -public class InTests extends ESTestCase { - - private static final Literal ONE = L(1); - private static final Literal TWO = L(2); - private static final Literal THREE = L(3); - - public void testInWithContainedValue() { - In in = new In(EMPTY, TWO, Arrays.asList(ONE, TWO, THREE)); - assertTrue(in.fold()); - } - - public void testInWithNotContainedValue() { - In in = new In(EMPTY, THREE, Arrays.asList(ONE, TWO)); - assertFalse(in.fold()); - } - - public void testHandleNullOnLeftValue() { - In in = new In(EMPTY, NULL, Arrays.asList(ONE, TWO, THREE)); - assertNull(in.fold()); - in = new In(EMPTY, NULL, Arrays.asList(ONE, NULL, THREE)); - assertNull(in.fold()); - - } - - public void testHandleNullsOnRightValue() { - In in = new In(EMPTY, THREE, Arrays.asList(ONE, NULL, THREE)); - assertTrue(in.fold()); - in = new In(EMPTY, ONE, Arrays.asList(TWO, NULL, THREE)); - assertNull(in.fold()); - } - - private static Literal L(Object value) { - return TestUtils.of(EMPTY, value); - } -} diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TypesTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TypesTests.java index 1974eb3669f4b..bc682b46ba0ea 100644 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TypesTests.java +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TypesTests.java @@ -118,7 +118,7 @@ public void testDottedField() { assertThat(mapping.size(), is(2)); EsField field = mapping.get("manager"); - assertThat(DataType.isPrimitive(field.getDataType()), is(false)); + assertThat(DataType.isPrimitiveAndSupported(field.getDataType()), is(false)); assertThat(field.getDataType(), is(OBJECT)); Map children = field.getProperties(); assertThat(children.size(), is(2)); @@ -133,7 +133,7 @@ public void testMultiField() { assertThat(mapping.size(), is(1)); EsField field = mapping.get("text"); - assertThat(DataType.isPrimitive(field.getDataType()), is(true)); + assertThat(DataType.isPrimitiveAndSupported(field.getDataType()), is(true)); assertThat(field.getDataType(), is(TEXT)); Map fields = field.getProperties(); assertThat(fields.size(), is(4)); @@ -147,7 +147,7 @@ public void testMultiFieldTooManyOptions() { assertThat(mapping.size(), is(1)); EsField field = mapping.get("text"); - assertThat(DataType.isPrimitive(field.getDataType()), is(true)); + assertThat(DataType.isPrimitiveAndSupported(field.getDataType()), is(true)); assertThat(field, instanceOf(TextEsField.class)); Map fields = field.getProperties(); assertThat(fields.size(), is(4)); @@ -161,7 +161,7 @@ public void testNestedDoc() { assertThat(mapping.size(), is(1)); EsField field = mapping.get("dep"); - assertThat(DataType.isPrimitive(field.getDataType()), is(false)); + assertThat(DataType.isPrimitiveAndSupported(field.getDataType()), is(false)); assertThat(field.getDataType(), is(NESTED)); Map children = field.getProperties(); assertThat(children.size(), is(4)); diff --git a/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/SpecReader.java b/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/SpecReader.java index 422a5b744eed0..c96f360cc95f0 100644 --- a/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/SpecReader.java +++ b/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/SpecReader.java @@ -79,7 +79,7 @@ public static List readURLSpec(URL source, Parser parser) throws Excep Object result = parser.parse(line); // only if the parser is ready, add the object - otherwise keep on serving it lines if (result != null) { - testCases.add(new Object[] { fileName, groupName, testName, Integer.valueOf(lineNumber), result }); + testCases.add(makeTestCase(fileName, groupName, testName, lineNumber, result)); testName = null; } } @@ -102,4 +102,13 @@ public interface Parser { public static boolean shouldSkipLine(String line) { return line.isEmpty() || line.startsWith("//") || line.startsWith("#"); } + + private static Object[] makeTestCase(String fileName, String groupName, String testName, int lineNumber, Object result) { + var testNameParts = testName.split("#", 2); + + testName = testNameParts[0]; + var instructions = testNameParts.length == 2 ? testNameParts[1] : ""; + + return new Object[] { fileName, groupName, testName, lineNumber, result, instructions }; + } } diff --git a/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/TestUtils.java b/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/TestUtils.java index 5f774ad9dd60e..c587821e82986 100644 --- a/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/TestUtils.java +++ b/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/TestUtils.java @@ -25,13 +25,6 @@ import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.Range; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.GreaterThan; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.GreaterThanOrEqual; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.LessThan; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.LessThanOrEqual; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.NotEquals; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.NullEquals; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLike; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardLike; @@ -115,34 +108,6 @@ public static Literal of(Source source, Object value) { return new Literal(source, value, DataType.fromJava(value)); } - public static Equals equalsOf(Expression left, Expression right) { - return new Equals(EMPTY, left, right, randomZone()); - } - - public static NotEquals notEqualsOf(Expression left, Expression right) { - return new NotEquals(EMPTY, left, right, randomZone()); - } - - public static NullEquals nullEqualsOf(Expression left, Expression right) { - return new NullEquals(EMPTY, left, right, randomZone()); - } - - public static LessThan lessThanOf(Expression left, Expression right) { - return new LessThan(EMPTY, left, right, randomZone()); - } - - public static LessThanOrEqual lessThanOrEqualOf(Expression left, Expression right) { - return new LessThanOrEqual(EMPTY, left, right, randomZone()); - } - - public static GreaterThan greaterThanOf(Expression left, Expression right) { - return new GreaterThan(EMPTY, left, right, randomZone()); - } - - public static GreaterThanOrEqual greaterThanOrEqualOf(Expression left, Expression right) { - return new GreaterThanOrEqual(EMPTY, left, right, randomZone()); - } - public static Range rangeOf(Expression value, Expression lower, boolean includeLower, Expression upper, boolean includeUpper) { return new Range(EMPTY, value, lower, includeLower, upper, includeUpper, randomZone()); } diff --git a/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/ArrowResponse.java b/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/ArrowResponse.java index 8c2243284a538..7a8328060a390 100644 --- a/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/ArrowResponse.java +++ b/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/ArrowResponse.java @@ -326,7 +326,7 @@ protected void encodeChunk(int sizeHint, RecyclerBytesStreamOutput out) throws I */ static final Map ESQL_CONVERTERS = Map.ofEntries( // For reference: - // - EsqlDataTypes: list of ESQL data types (not all are present in outputs) + // - DataType: list of ESQL data types (not all are present in outputs) // - PositionToXContent: conversions for ESQL JSON output // - EsqlDataTypeConverter: conversions to ESQL datatypes // Missing: multi-valued values diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index 1694115aaa71d..8803fd81147ef 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -36,11 +36,13 @@ dependencies { testImplementation project(':test:framework') testImplementation(testArtifact(project(xpackModule('core')))) testImplementation project(path: xpackModule('enrich')) + testImplementation project(path: xpackModule('spatial')) testImplementation project(path: ':modules:reindex') testImplementation project(path: ':modules:parent-join') testImplementation project(path: ':modules:analysis-common') testImplementation project(path: ':modules:ingest-common') + testImplementation project(path: ':modules:legacy-geo') testImplementation('net.nextencia:rrdiagram:0.9.4') testImplementation('org.webjars.npm:fontsource__roboto-mono:4.5.7') @@ -309,4 +311,32 @@ tasks.named('stringTemplates').configure { it.inputFile = enrichResultBuilderInput it.outputFile = "org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForBoolean.java" } + + + File inInputFile = file("src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InEvaluator.java.st") + template { + it.properties = booleanProperties + it.inputFile = inInputFile + it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InBooleanEvaluator.java" + } + template { + it.properties = intProperties + it.inputFile = inInputFile + it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InIntEvaluator.java" + } + template { + it.properties = longProperties + it.inputFile = inInputFile + it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InLongEvaluator.java" + } + template { + it.properties = doubleProperties + it.inputFile = inInputFile + it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InDoubleEvaluator.java" + } + template { + it.properties = bytesRefProperties + it.inputFile = inInputFile + it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InBytesRefEvaluator.java" + } } diff --git a/x-pack/plugin/esql/compute/build.gradle b/x-pack/plugin/esql/compute/build.gradle index e5816d0b7c78b..ac053bdb827dc 100644 --- a/x-pack/plugin/esql/compute/build.gradle +++ b/x-pack/plugin/esql/compute/build.gradle @@ -36,32 +36,40 @@ spotless { } } -def prop(Type, type, Wrapper, TYPE, BYTES, Array, Hash) { +def prop(Name, Type, type, Wrapper, TYPE, BYTES, Array, Hash) { return [ + // Name of the DataType. Use in DataType names + "Name" : Name, + // PascalCased type. Use in ElementType names "Type" : Type, + // Variable type. May be a primitive "type" : type, + // Wrapper type. Only for primitive types "Wrapper": Wrapper, + // SCREAMING_SNAKE_CASE type. Use in ElementType names "TYPE" : TYPE, "BYTES" : BYTES, "Array" : Array, "Hash" : Hash, - "int" : type == "int" ? "true" : "", - "float" : type == "float" ? "true" : "", - "long" : type == "long" ? "true" : "", - "double" : type == "double" ? "true" : "", - "BytesRef" : type == "BytesRef" ? "true" : "", - "boolean" : type == "boolean" ? "true" : "", + "int" : Name == "Int" ? "true" : "", + "float" : Name == "Float" ? "true" : "", + "long" : Name == "Long" ? "true" : "", + "double" : Name == "Double" ? "true" : "", + "boolean" : Name == "Boolean" ? "true" : "", + "BytesRef" : Name == "BytesRef" ? "true" : "", + "Ip" : Name == "Ip" ? "true" : "", ] } tasks.named('stringTemplates').configure { - var intProperties = prop("Int", "int", "Integer", "INT", "Integer.BYTES", "IntArray", "LongHash") - var floatProperties = prop("Float", "float", "Float", "FLOAT", "Float.BYTES", "FloatArray", "LongHash") - var longProperties = prop("Long", "long", "Long", "LONG", "Long.BYTES", "LongArray", "LongHash") - var doubleProperties = prop("Double", "double", "Double", "DOUBLE", "Double.BYTES", "DoubleArray", "LongHash") - var bytesRefProperties = prop("BytesRef", "BytesRef", "", "BYTES_REF", "org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF", "", "BytesRefHash") - var booleanProperties = prop("Boolean", "boolean", "Boolean", "BOOLEAN", "Byte.BYTES", "BitArray", "") + var intProperties = prop("Int", "Int", "int", "Integer", "INT", "Integer.BYTES", "IntArray", "LongHash") + var floatProperties = prop("Float", "Float", "float", "Float", "FLOAT", "Float.BYTES", "FloatArray", "LongHash") + var longProperties = prop("Long", "Long", "long", "Long", "LONG", "Long.BYTES", "LongArray", "LongHash") + var doubleProperties = prop("Double", "Double", "double", "Double", "DOUBLE", "Double.BYTES", "DoubleArray", "LongHash") + var booleanProperties = prop("Boolean", "Boolean", "boolean", "Boolean", "BOOLEAN", "Byte.BYTES", "BitArray", "") + var bytesRefProperties = prop("BytesRef", "BytesRef", "BytesRef", "", "BYTES_REF", "org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF", "", "BytesRefHash") + var ipProperties = prop("Ip", "BytesRef", "BytesRef", "", "BYTES_REF", "16", "", "") // primitive vectors File vectorInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st") @@ -554,6 +562,16 @@ tasks.named('stringTemplates').configure { it.inputFile = topAggregatorInputFile it.outputFile = "org/elasticsearch/compute/aggregation/TopDoubleAggregator.java" } + template { + it.properties = booleanProperties + it.inputFile = topAggregatorInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/TopBooleanAggregator.java" + } + template { + it.properties = ipProperties + it.inputFile = topAggregatorInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/TopIpAggregator.java" + } File multivalueDedupeInputFile = file("src/main/java/org/elasticsearch/compute/operator/mvdedupe/X-MultivalueDedupe.java.st") template { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanArrayState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanArrayState.java index 79f4a88d403c6..793e6cc1b37ef 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanArrayState.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanArrayState.java @@ -49,7 +49,7 @@ boolean get(int groupId) { } boolean getOrDefault(int groupId) { - return groupId < values.size() ? values.get(groupId) : init; + return groupId < size ? values.get(groupId) : init; } void set(int groupId, boolean value) { @@ -102,7 +102,7 @@ public void toIntermediate( ) { for (int i = 0; i < selected.getPositionCount(); i++) { int group = selected.getInt(i); - if (group < values.size()) { + if (group < size) { valuesBuilder.appendBoolean(values.get(group)); } else { valuesBuilder.appendBoolean(false); // TODO can we just use null? diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopBooleanAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopBooleanAggregator.java new file mode 100644 index 0000000000000..32391c4827303 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopBooleanAggregator.java @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.sort.BooleanBucketedSort; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.search.sort.SortOrder; + +/** + * Aggregates the top N field values for boolean. + *

    + * This class is generated. Edit `X-TopAggregator.java.st` to edit this file. + *

    + */ +@Aggregator({ @IntermediateState(name = "top", type = "BOOLEAN_BLOCK") }) +@GroupingAggregator +class TopBooleanAggregator { + public static SingleState initSingle(BigArrays bigArrays, int limit, boolean ascending) { + return new SingleState(bigArrays, limit, ascending); + } + + public static void combine(SingleState state, boolean v) { + state.add(v); + } + + public static void combineIntermediate(SingleState state, BooleanBlock values) { + int start = values.getFirstValueIndex(0); + int end = start + values.getValueCount(0); + for (int i = start; i < end; i++) { + combine(state, values.getBoolean(i)); + } + } + + public static Block evaluateFinal(SingleState state, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory()); + } + + public static GroupingState initGrouping(BigArrays bigArrays, int limit, boolean ascending) { + return new GroupingState(bigArrays, limit, ascending); + } + + public static void combine(GroupingState state, int groupId, boolean v) { + state.add(groupId, v); + } + + public static void combineIntermediate(GroupingState state, int groupId, BooleanBlock values, int valuesPosition) { + int start = values.getFirstValueIndex(valuesPosition); + int end = start + values.getValueCount(valuesPosition); + for (int i = start; i < end; i++) { + combine(state, groupId, values.getBoolean(i)); + } + } + + public static void combineStates(GroupingState current, int groupId, GroupingState state, int statePosition) { + current.merge(groupId, state, statePosition); + } + + public static Block evaluateFinal(GroupingState state, IntVector selected, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory(), selected); + } + + public static class GroupingState implements Releasable { + private final BooleanBucketedSort sort; + + private GroupingState(BigArrays bigArrays, int limit, boolean ascending) { + this.sort = new BooleanBucketedSort(bigArrays, ascending ? SortOrder.ASC : SortOrder.DESC, limit); + } + + public void add(int groupId, boolean value) { + sort.collect(value, groupId); + } + + public void merge(int groupId, GroupingState other, int otherGroupId) { + sort.merge(groupId, other.sort, otherGroupId); + } + + void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory(), selected); + } + + Block toBlock(BlockFactory blockFactory, IntVector selected) { + return sort.toBlock(blockFactory, selected); + } + + void enableGroupIdTracking(SeenGroupIds seen) { + // we figure out seen values from nulls on the values block + } + + @Override + public void close() { + Releasables.closeExpectNoException(sort); + } + } + + public static class SingleState implements Releasable { + private final GroupingState internalState; + + private SingleState(BigArrays bigArrays, int limit, boolean ascending) { + this.internalState = new GroupingState(bigArrays, limit, ascending); + } + + public void add(boolean value) { + internalState.add(0, value); + } + + public void merge(GroupingState other) { + internalState.merge(0, other, 0); + } + + void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory()); + } + + Block toBlock(BlockFactory blockFactory) { + try (var intValues = blockFactory.newConstantIntVector(0, 1)) { + return internalState.toBlock(blockFactory, intValues); + } + } + + @Override + public void close() { + Releasables.closeExpectNoException(internalState); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopDoubleAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopDoubleAggregator.java index 3bd76b79d62f2..d9a7a302f07c8 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopDoubleAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopDoubleAggregator.java @@ -23,6 +23,9 @@ /** * Aggregates the top N field values for double. + *

    + * This class is generated. Edit `X-TopAggregator.java.st` to edit this file. + *

    */ @Aggregator({ @IntermediateState(name = "top", type = "DOUBLE_BLOCK") }) @GroupingAggregator diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopFloatAggregator.java index 066c82e9448fb..8b65261e10f46 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopFloatAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopFloatAggregator.java @@ -23,6 +23,9 @@ /** * Aggregates the top N field values for float. + *

    + * This class is generated. Edit `X-TopAggregator.java.st` to edit this file. + *

    */ @Aggregator({ @IntermediateState(name = "top", type = "FLOAT_BLOCK") }) @GroupingAggregator diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIntAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIntAggregator.java index 2f5149c594d94..5c6b79f710af5 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIntAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIntAggregator.java @@ -23,6 +23,9 @@ /** * Aggregates the top N field values for int. + *

    + * This class is generated. Edit `X-TopAggregator.java.st` to edit this file. + *

    */ @Aggregator({ @IntermediateState(name = "top", type = "INT_BLOCK") }) @GroupingAggregator diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIpAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIpAggregator.java new file mode 100644 index 0000000000000..219f7385b56df --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIpAggregator.java @@ -0,0 +1,143 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.sort.IpBucketedSort; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.search.sort.SortOrder; + +/** + * Aggregates the top N field values for BytesRef. + *

    + * This class is generated. Edit `X-TopAggregator.java.st` to edit this file. + *

    + */ +@Aggregator({ @IntermediateState(name = "top", type = "BYTES_REF_BLOCK") }) +@GroupingAggregator +class TopIpAggregator { + public static SingleState initSingle(BigArrays bigArrays, int limit, boolean ascending) { + return new SingleState(bigArrays, limit, ascending); + } + + public static void combine(SingleState state, BytesRef v) { + state.add(v); + } + + public static void combineIntermediate(SingleState state, BytesRefBlock values) { + int start = values.getFirstValueIndex(0); + int end = start + values.getValueCount(0); + var scratch = new BytesRef(); + for (int i = start; i < end; i++) { + combine(state, values.getBytesRef(i, scratch)); + } + } + + public static Block evaluateFinal(SingleState state, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory()); + } + + public static GroupingState initGrouping(BigArrays bigArrays, int limit, boolean ascending) { + return new GroupingState(bigArrays, limit, ascending); + } + + public static void combine(GroupingState state, int groupId, BytesRef v) { + state.add(groupId, v); + } + + public static void combineIntermediate(GroupingState state, int groupId, BytesRefBlock values, int valuesPosition) { + int start = values.getFirstValueIndex(valuesPosition); + int end = start + values.getValueCount(valuesPosition); + var scratch = new BytesRef(); + for (int i = start; i < end; i++) { + combine(state, groupId, values.getBytesRef(i, scratch)); + } + } + + public static void combineStates(GroupingState current, int groupId, GroupingState state, int statePosition) { + current.merge(groupId, state, statePosition); + } + + public static Block evaluateFinal(GroupingState state, IntVector selected, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory(), selected); + } + + public static class GroupingState implements Releasable { + private final IpBucketedSort sort; + + private GroupingState(BigArrays bigArrays, int limit, boolean ascending) { + this.sort = new IpBucketedSort(bigArrays, ascending ? SortOrder.ASC : SortOrder.DESC, limit); + } + + public void add(int groupId, BytesRef value) { + sort.collect(value, groupId); + } + + public void merge(int groupId, GroupingState other, int otherGroupId) { + sort.merge(groupId, other.sort, otherGroupId); + } + + void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory(), selected); + } + + Block toBlock(BlockFactory blockFactory, IntVector selected) { + return sort.toBlock(blockFactory, selected); + } + + void enableGroupIdTracking(SeenGroupIds seen) { + // we figure out seen values from nulls on the values block + } + + @Override + public void close() { + Releasables.closeExpectNoException(sort); + } + } + + public static class SingleState implements Releasable { + private final GroupingState internalState; + + private SingleState(BigArrays bigArrays, int limit, boolean ascending) { + this.internalState = new GroupingState(bigArrays, limit, ascending); + } + + public void add(BytesRef value) { + internalState.add(0, value); + } + + public void merge(GroupingState other) { + internalState.merge(0, other, 0); + } + + void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory()); + } + + Block toBlock(BlockFactory blockFactory) { + try (var intValues = blockFactory.newConstantIntVector(0, 1)) { + return internalState.toBlock(blockFactory, intValues); + } + } + + @Override + public void close() { + Releasables.closeExpectNoException(internalState); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopLongAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopLongAggregator.java index d6bafaa30c425..44cef8df7257b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopLongAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopLongAggregator.java @@ -23,6 +23,9 @@ /** * Aggregates the top N field values for long. + *

    + * This class is generated. Edit `X-TopAggregator.java.st` to edit this file. + *

    */ @Aggregator({ @IntermediateState(name = "top", type = "LONG_BLOCK") }) @GroupingAggregator diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBlock.java index 8ae2984018640..d9f40a3012531 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBlock.java @@ -97,10 +97,10 @@ default void writeTo(StreamOutput out) throws IOException { if (vector != null) { out.writeByte(SERIALIZE_BLOCK_VECTOR); vector.writeTo(out); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_BLOCK) && this instanceof BooleanArrayBlock b) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof BooleanArrayBlock b) { out.writeByte(SERIALIZE_BLOCK_ARRAY); b.writeArrayBlock(out); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_ARRAY) && this instanceof BooleanBigArrayBlock b) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof BooleanBigArrayBlock b) { out.writeByte(SERIALIZE_BLOCK_BIG_ARRAY); b.writeArrayBlock(out); } else { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVector.java index 5cf900cfc4a71..0eca12fd73e39 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVector.java @@ -101,10 +101,10 @@ default void writeTo(StreamOutput out) throws IOException { if (isConstant() && positions > 0) { out.writeByte(SERIALIZE_VECTOR_CONSTANT); out.writeBoolean(getBoolean(0)); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_VECTOR) && this instanceof BooleanArrayVector v) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof BooleanArrayVector v) { out.writeByte(SERIALIZE_VECTOR_ARRAY); v.writeArrayVector(positions, out); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_VECTOR) && this instanceof BooleanBigArrayVector v) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof BooleanBigArrayVector v) { out.writeByte(SERIALIZE_VECTOR_BIG_ARRAY); v.writeArrayVector(positions, out); } else { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlock.java index d7c28a24482e0..041c69a89872b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlock.java @@ -107,10 +107,10 @@ default void writeTo(StreamOutput out) throws IOException { if (vector != null) { out.writeByte(SERIALIZE_BLOCK_VECTOR); vector.writeTo(out); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_BLOCK) && this instanceof BytesRefArrayBlock b) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof BytesRefArrayBlock b) { out.writeByte(SERIALIZE_BLOCK_ARRAY); b.writeArrayBlock(out); - } else if (version.onOrAfter(TransportVersions.ESQL_ORDINAL_BLOCK) && this instanceof OrdinalBytesRefBlock b && b.isDense()) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof OrdinalBytesRefBlock b && b.isDense()) { out.writeByte(SERIALIZE_BLOCK_ORDINAL); b.writeOrdinalBlock(out); } else { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVector.java index 3739dccb0f956..004e0886fc801 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVector.java @@ -108,10 +108,10 @@ default void writeTo(StreamOutput out) throws IOException { if (isConstant() && positions > 0) { out.writeByte(SERIALIZE_VECTOR_CONSTANT); out.writeBytesRef(getBytesRef(0, new BytesRef())); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_VECTOR) && this instanceof BytesRefArrayVector v) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof BytesRefArrayVector v) { out.writeByte(SERIALIZE_VECTOR_ARRAY); v.writeArrayVector(positions, out); - } else if (version.onOrAfter(TransportVersions.ESQL_ORDINAL_BLOCK) && this instanceof OrdinalBytesRefVector v && v.isDense()) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof OrdinalBytesRefVector v && v.isDense()) { out.writeByte(SERIALIZE_VECTOR_ORDINAL); v.writeOrdinalVector(out); } else { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBlock.java index 95f318703df62..bbb27548047b8 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBlock.java @@ -97,10 +97,10 @@ default void writeTo(StreamOutput out) throws IOException { if (vector != null) { out.writeByte(SERIALIZE_BLOCK_VECTOR); vector.writeTo(out); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_BLOCK) && this instanceof DoubleArrayBlock b) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof DoubleArrayBlock b) { out.writeByte(SERIALIZE_BLOCK_ARRAY); b.writeArrayBlock(out); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_ARRAY) && this instanceof DoubleBigArrayBlock b) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof DoubleBigArrayBlock b) { out.writeByte(SERIALIZE_BLOCK_BIG_ARRAY); b.writeArrayBlock(out); } else { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVector.java index 10d4f4abe5f6a..d6993402d1d08 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVector.java @@ -102,10 +102,10 @@ default void writeTo(StreamOutput out) throws IOException { if (isConstant() && positions > 0) { out.writeByte(SERIALIZE_VECTOR_CONSTANT); out.writeDouble(getDouble(0)); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_VECTOR) && this instanceof DoubleArrayVector v) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof DoubleArrayVector v) { out.writeByte(SERIALIZE_VECTOR_ARRAY); v.writeArrayVector(positions, out); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_VECTOR) && this instanceof DoubleBigArrayVector v) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof DoubleBigArrayVector v) { out.writeByte(SERIALIZE_VECTOR_BIG_ARRAY); v.writeArrayVector(positions, out); } else { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBlock.java index 3d2def604a61e..06f58b9cd90e8 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBlock.java @@ -97,10 +97,10 @@ default void writeTo(StreamOutput out) throws IOException { if (vector != null) { out.writeByte(SERIALIZE_BLOCK_VECTOR); vector.writeTo(out); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_BLOCK) && this instanceof FloatArrayBlock b) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof FloatArrayBlock b) { out.writeByte(SERIALIZE_BLOCK_ARRAY); b.writeArrayBlock(out); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_ARRAY) && this instanceof FloatBigArrayBlock b) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof FloatBigArrayBlock b) { out.writeByte(SERIALIZE_BLOCK_BIG_ARRAY); b.writeArrayBlock(out); } else { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVector.java index 5fd2ae7b9c719..4476ca6c522ec 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVector.java @@ -101,10 +101,10 @@ default void writeTo(StreamOutput out) throws IOException { if (isConstant() && positions > 0) { out.writeByte(SERIALIZE_VECTOR_CONSTANT); out.writeFloat(getFloat(0)); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_VECTOR) && this instanceof FloatArrayVector v) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof FloatArrayVector v) { out.writeByte(SERIALIZE_VECTOR_ARRAY); v.writeArrayVector(positions, out); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_VECTOR) && this instanceof FloatBigArrayVector v) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof FloatBigArrayVector v) { out.writeByte(SERIALIZE_VECTOR_BIG_ARRAY); v.writeArrayVector(positions, out); } else { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlock.java index 21d40170151a5..988bda41cc84e 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlock.java @@ -97,10 +97,10 @@ default void writeTo(StreamOutput out) throws IOException { if (vector != null) { out.writeByte(SERIALIZE_BLOCK_VECTOR); vector.writeTo(out); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_BLOCK) && this instanceof IntArrayBlock b) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof IntArrayBlock b) { out.writeByte(SERIALIZE_BLOCK_ARRAY); b.writeArrayBlock(out); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_ARRAY) && this instanceof IntBigArrayBlock b) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof IntBigArrayBlock b) { out.writeByte(SERIALIZE_BLOCK_BIG_ARRAY); b.writeArrayBlock(out); } else { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVector.java index 384d5813d5750..016c52cc0656f 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVector.java @@ -111,10 +111,10 @@ default void writeTo(StreamOutput out) throws IOException { if (isConstant() && positions > 0) { out.writeByte(SERIALIZE_VECTOR_CONSTANT); out.writeInt(getInt(0)); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_VECTOR) && this instanceof IntArrayVector v) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof IntArrayVector v) { out.writeByte(SERIALIZE_VECTOR_ARRAY); v.writeArrayVector(positions, out); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_VECTOR) && this instanceof IntBigArrayVector v) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof IntBigArrayVector v) { out.writeByte(SERIALIZE_VECTOR_BIG_ARRAY); v.writeArrayVector(positions, out); } else { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBlock.java index 5a11ee8e2a6e3..245fd09b951b4 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBlock.java @@ -97,10 +97,10 @@ default void writeTo(StreamOutput out) throws IOException { if (vector != null) { out.writeByte(SERIALIZE_BLOCK_VECTOR); vector.writeTo(out); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_BLOCK) && this instanceof LongArrayBlock b) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof LongArrayBlock b) { out.writeByte(SERIALIZE_BLOCK_ARRAY); b.writeArrayBlock(out); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_ARRAY) && this instanceof LongBigArrayBlock b) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof LongBigArrayBlock b) { out.writeByte(SERIALIZE_BLOCK_BIG_ARRAY); b.writeArrayBlock(out); } else { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVector.java index a74146b692e31..36381ffaa1a4a 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVector.java @@ -102,10 +102,10 @@ default void writeTo(StreamOutput out) throws IOException { if (isConstant() && positions > 0) { out.writeByte(SERIALIZE_VECTOR_CONSTANT); out.writeLong(getLong(0)); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_VECTOR) && this instanceof LongArrayVector v) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof LongArrayVector v) { out.writeByte(SERIALIZE_VECTOR_ARRAY); v.writeArrayVector(positions, out); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_VECTOR) && this instanceof LongBigArrayVector v) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof LongBigArrayVector v) { out.writeByte(SERIALIZE_VECTOR_BIG_ARRAY); v.writeArrayVector(positions, out); } else { diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBooleanGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBooleanGroupingAggregatorFunction.java index b72ff8354cb12..f404fccd45d51 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBooleanGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBooleanGroupingAggregatorFunction.java @@ -161,7 +161,9 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page assert max.getPositionCount() == seen.getPositionCount(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = Math.toIntExact(groups.getInt(groupPosition)); - MaxBooleanAggregator.combineIntermediate(state, groupId, max.getBoolean(groupPosition + positionOffset), seen.getBoolean(groupPosition + positionOffset)); + if (seen.getBoolean(groupPosition + positionOffset)) { + state.set(groupId, MaxBooleanAggregator.combine(state.getOrDefault(groupId), max.getBoolean(groupPosition + positionOffset))); + } } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunction.java new file mode 100644 index 0000000000000..9f714246ea332 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunction.java @@ -0,0 +1,133 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link MaxIpAggregator}. + * This class is generated. Do not edit it. + */ +public final class MaxIpAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("max", ElementType.BYTES_REF), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final DriverContext driverContext; + + private final MaxIpAggregator.SingleState state; + + private final List channels; + + public MaxIpAggregatorFunction(DriverContext driverContext, List channels, + MaxIpAggregator.SingleState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static MaxIpAggregatorFunction create(DriverContext driverContext, + List channels) { + return new MaxIpAggregatorFunction(driverContext, channels, MaxIpAggregator.initSingle()); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + BytesRefBlock block = page.getBlock(channels.get(0)); + BytesRefVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(BytesRefVector vector) { + BytesRef scratch = new BytesRef(); + for (int i = 0; i < vector.getPositionCount(); i++) { + MaxIpAggregator.combine(state, vector.getBytesRef(i, scratch)); + } + } + + private void addRawBlock(BytesRefBlock block) { + BytesRef scratch = new BytesRef(); + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + MaxIpAggregator.combine(state, block.getBytesRef(i, scratch)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block maxUncast = page.getBlock(channels.get(0)); + if (maxUncast.areAllValuesNull()) { + return; + } + BytesRefVector max = ((BytesRefBlock) maxUncast).asVector(); + assert max.getPositionCount() == 1; + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert seen.getPositionCount() == 1; + BytesRef scratch = new BytesRef(); + MaxIpAggregator.combineIntermediate(state, max.getBytesRef(0, scratch), seen.getBoolean(0)); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = MaxIpAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..1fb734c243477 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunctionSupplier.java @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link MaxIpAggregator}. + * This class is generated. Do not edit it. + */ +public final class MaxIpAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public MaxIpAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public MaxIpAggregatorFunction aggregator(DriverContext driverContext) { + return MaxIpAggregatorFunction.create(driverContext, channels); + } + + @Override + public MaxIpGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return MaxIpGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "max of ips"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..c556b23215e6b --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpGroupingAggregatorFunction.java @@ -0,0 +1,210 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link MaxIpAggregator}. + * This class is generated. Do not edit it. + */ +public final class MaxIpGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("max", ElementType.BYTES_REF), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final MaxIpAggregator.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + public MaxIpGroupingAggregatorFunction(List channels, + MaxIpAggregator.GroupingState state, DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static MaxIpGroupingAggregatorFunction create(List channels, + DriverContext driverContext) { + return new MaxIpGroupingAggregatorFunction(channels, MaxIpAggregator.initGrouping(driverContext.bigArrays()), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + BytesRefBlock valuesBlock = page.getBlock(channels.get(0)); + BytesRefVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + MaxIpAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, BytesRefVector values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + MaxIpAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + MaxIpAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + MaxIpAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block maxUncast = page.getBlock(channels.get(0)); + if (maxUncast.areAllValuesNull()) { + return; + } + BytesRefVector max = ((BytesRefBlock) maxUncast).asVector(); + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert max.getPositionCount() == seen.getPositionCount(); + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + MaxIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(groupPosition + positionOffset, scratch), seen.getBoolean(groupPosition + positionOffset)); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + MaxIpAggregator.GroupingState inState = ((MaxIpGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + MaxIpAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = MaxIpAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpAggregatorFunction.java new file mode 100644 index 0000000000000..a47c901d70db4 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpAggregatorFunction.java @@ -0,0 +1,133 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link MinIpAggregator}. + * This class is generated. Do not edit it. + */ +public final class MinIpAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("max", ElementType.BYTES_REF), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final DriverContext driverContext; + + private final MinIpAggregator.SingleState state; + + private final List channels; + + public MinIpAggregatorFunction(DriverContext driverContext, List channels, + MinIpAggregator.SingleState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static MinIpAggregatorFunction create(DriverContext driverContext, + List channels) { + return new MinIpAggregatorFunction(driverContext, channels, MinIpAggregator.initSingle()); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + BytesRefBlock block = page.getBlock(channels.get(0)); + BytesRefVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(BytesRefVector vector) { + BytesRef scratch = new BytesRef(); + for (int i = 0; i < vector.getPositionCount(); i++) { + MinIpAggregator.combine(state, vector.getBytesRef(i, scratch)); + } + } + + private void addRawBlock(BytesRefBlock block) { + BytesRef scratch = new BytesRef(); + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + MinIpAggregator.combine(state, block.getBytesRef(i, scratch)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block maxUncast = page.getBlock(channels.get(0)); + if (maxUncast.areAllValuesNull()) { + return; + } + BytesRefVector max = ((BytesRefBlock) maxUncast).asVector(); + assert max.getPositionCount() == 1; + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert seen.getPositionCount() == 1; + BytesRef scratch = new BytesRef(); + MinIpAggregator.combineIntermediate(state, max.getBytesRef(0, scratch), seen.getBoolean(0)); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = MinIpAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..591a8501f874d --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpAggregatorFunctionSupplier.java @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link MinIpAggregator}. + * This class is generated. Do not edit it. + */ +public final class MinIpAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public MinIpAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public MinIpAggregatorFunction aggregator(DriverContext driverContext) { + return MinIpAggregatorFunction.create(driverContext, channels); + } + + @Override + public MinIpGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return MinIpGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "min of ips"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..5b51f041bd966 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpGroupingAggregatorFunction.java @@ -0,0 +1,210 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link MinIpAggregator}. + * This class is generated. Do not edit it. + */ +public final class MinIpGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("max", ElementType.BYTES_REF), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final MinIpAggregator.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + public MinIpGroupingAggregatorFunction(List channels, + MinIpAggregator.GroupingState state, DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static MinIpGroupingAggregatorFunction create(List channels, + DriverContext driverContext) { + return new MinIpGroupingAggregatorFunction(channels, MinIpAggregator.initGrouping(driverContext.bigArrays()), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + BytesRefBlock valuesBlock = page.getBlock(channels.get(0)); + BytesRefVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + MinIpAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, BytesRefVector values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + MinIpAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + MinIpAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + MinIpAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block maxUncast = page.getBlock(channels.get(0)); + if (maxUncast.areAllValuesNull()) { + return; + } + BytesRefVector max = ((BytesRefBlock) maxUncast).asVector(); + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert max.getPositionCount() == seen.getPositionCount(); + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + MinIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(groupPosition + positionOffset, scratch), seen.getBoolean(groupPosition + positionOffset)); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + MinIpAggregator.GroupingState inState = ((MinIpGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + MinIpAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = MinIpAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBooleanAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBooleanAggregatorFunction.java new file mode 100644 index 0000000000000..617ebfd004808 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBooleanAggregatorFunction.java @@ -0,0 +1,126 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link TopBooleanAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopBooleanAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("top", ElementType.BOOLEAN) ); + + private final DriverContext driverContext; + + private final TopBooleanAggregator.SingleState state; + + private final List channels; + + private final int limit; + + private final boolean ascending; + + public TopBooleanAggregatorFunction(DriverContext driverContext, List channels, + TopBooleanAggregator.SingleState state, int limit, boolean ascending) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + this.limit = limit; + this.ascending = ascending; + } + + public static TopBooleanAggregatorFunction create(DriverContext driverContext, + List channels, int limit, boolean ascending) { + return new TopBooleanAggregatorFunction(driverContext, channels, TopBooleanAggregator.initSingle(driverContext.bigArrays(), limit, ascending), limit, ascending); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + BooleanBlock block = page.getBlock(channels.get(0)); + BooleanVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(BooleanVector vector) { + for (int i = 0; i < vector.getPositionCount(); i++) { + TopBooleanAggregator.combine(state, vector.getBoolean(i)); + } + } + + private void addRawBlock(BooleanBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + TopBooleanAggregator.combine(state, block.getBoolean(i)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { + return; + } + BooleanBlock top = (BooleanBlock) topUncast; + assert top.getPositionCount() == 1; + TopBooleanAggregator.combineIntermediate(state, top); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = TopBooleanAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBooleanAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBooleanAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..74beed084543f --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBooleanAggregatorFunctionSupplier.java @@ -0,0 +1,45 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link TopBooleanAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopBooleanAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + private final int limit; + + private final boolean ascending; + + public TopBooleanAggregatorFunctionSupplier(List channels, int limit, + boolean ascending) { + this.channels = channels; + this.limit = limit; + this.ascending = ascending; + } + + @Override + public TopBooleanAggregatorFunction aggregator(DriverContext driverContext) { + return TopBooleanAggregatorFunction.create(driverContext, channels, limit, ascending); + } + + @Override + public TopBooleanGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return TopBooleanGroupingAggregatorFunction.create(channels, driverContext, limit, ascending); + } + + @Override + public String describe() { + return "top of booleans"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBooleanGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBooleanGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..53b5149e4da7e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBooleanGroupingAggregatorFunction.java @@ -0,0 +1,202 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link TopBooleanAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopBooleanGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("top", ElementType.BOOLEAN) ); + + private final TopBooleanAggregator.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + private final int limit; + + private final boolean ascending; + + public TopBooleanGroupingAggregatorFunction(List channels, + TopBooleanAggregator.GroupingState state, DriverContext driverContext, int limit, + boolean ascending) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + this.limit = limit; + this.ascending = ascending; + } + + public static TopBooleanGroupingAggregatorFunction create(List channels, + DriverContext driverContext, int limit, boolean ascending) { + return new TopBooleanGroupingAggregatorFunction(channels, TopBooleanAggregator.initGrouping(driverContext.bigArrays(), limit, ascending), driverContext, limit, ascending); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + BooleanBlock valuesBlock = page.getBlock(channels.get(0)); + BooleanVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, BooleanBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + TopBooleanAggregator.combine(state, groupId, values.getBoolean(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, BooleanVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + TopBooleanAggregator.combine(state, groupId, values.getBoolean(groupPosition + positionOffset)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, BooleanBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + TopBooleanAggregator.combine(state, groupId, values.getBoolean(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, BooleanVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + TopBooleanAggregator.combine(state, groupId, values.getBoolean(groupPosition + positionOffset)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { + return; + } + BooleanBlock top = (BooleanBlock) topUncast; + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + TopBooleanAggregator.combineIntermediate(state, groupId, top, groupPosition + positionOffset); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + TopBooleanAggregator.GroupingState inState = ((TopBooleanGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + TopBooleanAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = TopBooleanAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunction.java new file mode 100644 index 0000000000000..43f4d78d59cd9 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunction.java @@ -0,0 +1,130 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link TopIpAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopIpAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("top", ElementType.BYTES_REF) ); + + private final DriverContext driverContext; + + private final TopIpAggregator.SingleState state; + + private final List channels; + + private final int limit; + + private final boolean ascending; + + public TopIpAggregatorFunction(DriverContext driverContext, List channels, + TopIpAggregator.SingleState state, int limit, boolean ascending) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + this.limit = limit; + this.ascending = ascending; + } + + public static TopIpAggregatorFunction create(DriverContext driverContext, List channels, + int limit, boolean ascending) { + return new TopIpAggregatorFunction(driverContext, channels, TopIpAggregator.initSingle(driverContext.bigArrays(), limit, ascending), limit, ascending); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + BytesRefBlock block = page.getBlock(channels.get(0)); + BytesRefVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(BytesRefVector vector) { + BytesRef scratch = new BytesRef(); + for (int i = 0; i < vector.getPositionCount(); i++) { + TopIpAggregator.combine(state, vector.getBytesRef(i, scratch)); + } + } + + private void addRawBlock(BytesRefBlock block) { + BytesRef scratch = new BytesRef(); + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + TopIpAggregator.combine(state, block.getBytesRef(i, scratch)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { + return; + } + BytesRefBlock top = (BytesRefBlock) topUncast; + assert top.getPositionCount() == 1; + BytesRef scratch = new BytesRef(); + TopIpAggregator.combineIntermediate(state, top); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = TopIpAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..8f630c0306170 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunctionSupplier.java @@ -0,0 +1,44 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link TopIpAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopIpAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + private final int limit; + + private final boolean ascending; + + public TopIpAggregatorFunctionSupplier(List channels, int limit, boolean ascending) { + this.channels = channels; + this.limit = limit; + this.ascending = ascending; + } + + @Override + public TopIpAggregatorFunction aggregator(DriverContext driverContext) { + return TopIpAggregatorFunction.create(driverContext, channels, limit, ascending); + } + + @Override + public TopIpGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return TopIpGroupingAggregatorFunction.create(channels, driverContext, limit, ascending); + } + + @Override + public String describe() { + return "top of ips"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..d9e480c324676 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunction.java @@ -0,0 +1,208 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link TopIpAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopIpGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("top", ElementType.BYTES_REF) ); + + private final TopIpAggregator.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + private final int limit; + + private final boolean ascending; + + public TopIpGroupingAggregatorFunction(List channels, + TopIpAggregator.GroupingState state, DriverContext driverContext, int limit, + boolean ascending) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + this.limit = limit; + this.ascending = ascending; + } + + public static TopIpGroupingAggregatorFunction create(List channels, + DriverContext driverContext, int limit, boolean ascending) { + return new TopIpGroupingAggregatorFunction(channels, TopIpAggregator.initGrouping(driverContext.bigArrays(), limit, ascending), driverContext, limit, ascending); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + BytesRefBlock valuesBlock = page.getBlock(channels.get(0)); + BytesRefVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + TopIpAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, BytesRefVector values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + TopIpAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + TopIpAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + TopIpAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block topUncast = page.getBlock(channels.get(0)); + if (topUncast.areAllValuesNull()) { + return; + } + BytesRefBlock top = (BytesRefBlock) topUncast; + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + TopIpAggregator.combineIntermediate(state, groupId, top, groupPosition + positionOffset); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + TopIpAggregator.GroupingState inState = ((TopIpGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + TopIpAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = TopIpAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/IpArrayState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/IpArrayState.java new file mode 100644 index 0000000000000..63527f70fb621 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/IpArrayState.java @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.ByteArray; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasables; + +/** + * Aggregator state for an array of IPs. It is created in a mode where it + * won't track the {@code groupId}s that are sent to it and it is the + * responsibility of the caller to only fetch values for {@code groupId}s + * that it has sent using the {@code selected} parameter when building the + * results. This is fine when there are no {@code null} values in the input + * data. But once there are null values in the input data it is + * much more convenient to only send non-null values and + * the tracking built into the grouping code can't track that. In that case + * call {@link #enableGroupIdTracking} to transition the state into a mode + * where it'll track which {@code groupIds} have been written. + *

    + * This class is a specialized version of the {@code X-ArrayState.java.st} template. + *

    + */ +public final class IpArrayState extends AbstractArrayState implements GroupingAggregatorState { + private static final int IP_LENGTH = 16; + + private final byte[] init; + + private ByteArray values; + + IpArrayState(BigArrays bigArrays, BytesRef init) { + super(bigArrays); + assert init.length == IP_LENGTH; + this.values = bigArrays.newByteArray(IP_LENGTH, false); + this.init = new byte[IP_LENGTH]; + + System.arraycopy(init.bytes, init.offset, this.init, 0, IP_LENGTH); + this.values.set(0, this.init, 0, IP_LENGTH); + } + + BytesRef get(int groupId, BytesRef scratch) { + var ipIndex = getIndex(groupId); + values.get(ipIndex, IP_LENGTH, scratch); + return scratch; + } + + BytesRef getOrDefault(int groupId, BytesRef scratch) { + var ipIndex = getIndex(groupId); + if (ipIndex + IP_LENGTH <= values.size()) { + values.get(ipIndex, IP_LENGTH, scratch); + } else { + scratch.bytes = init; + scratch.offset = 0; + scratch.length = IP_LENGTH; + } + return scratch; + } + + void set(int groupId, BytesRef ip) { + assert ip.length == IP_LENGTH; + ensureCapacity(groupId); + var ipIndex = getIndex(groupId); + values.set(ipIndex, ip.bytes, ip.offset, ip.length); + trackGroupId(groupId); + } + + Block toValuesBlock(IntVector selected, DriverContext driverContext) { + var scratch = new BytesRef(); + if (false == trackingGroupIds()) { + try (var builder = driverContext.blockFactory().newBytesRefVectorBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + var value = get(group, scratch); + builder.appendBytesRef(value); + } + return builder.build().asBlock(); + } + } + try (var builder = driverContext.blockFactory().newBytesRefBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (hasValue(group)) { + var value = get(group, scratch); + builder.appendBytesRef(value); + } else { + builder.appendNull(); + } + } + return builder.build(); + } + } + + private void ensureCapacity(int groupId) { + var minIpIndex = getIndex(groupId); + var minSize = minIpIndex + IP_LENGTH; + if (minSize > values.size()) { + long prevSize = values.size(); + values = bigArrays.grow(values, minSize); + var prevLastIpIndex = prevSize - prevSize % IP_LENGTH; + var lastIpIndex = values.size() - values.size() % IP_LENGTH; + for (long i = prevLastIpIndex; i < lastIpIndex; i += IP_LENGTH) { + values.set(i, init, 0, IP_LENGTH); + } + } + } + + /** Extracts an intermediate view of the contents of this state. */ + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + assert blocks.length >= offset + 2; + try ( + var valuesBuilder = driverContext.blockFactory().newBytesRefBlockBuilder(selected.getPositionCount()); + var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount()) + ) { + var scratch = new BytesRef(); + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + int ipIndex = getIndex(group); + if (ipIndex < values.size()) { + var value = get(group, scratch); + valuesBuilder.appendBytesRef(value); + } else { + scratch.length = 0; + valuesBuilder.appendBytesRef(scratch); // TODO can we just use null? + } + hasValueBuilder.appendBoolean(i, hasValue(group)); + } + blocks[offset] = valuesBuilder.build(); + blocks[offset + 1] = hasValueBuilder.build().asBlock(); + } + } + + @Override + public void close() { + Releasables.close(values, super::close); + } + + /** + * Returns the index of the ip at {@code groupId} + */ + private int getIndex(int groupId) { + return groupId * IP_LENGTH; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxIpAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxIpAggregator.java new file mode 100644 index 0000000000000..1ddce7674ae7b --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxIpAggregator.java @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; + +@Aggregator({ @IntermediateState(name = "max", type = "BYTES_REF"), @IntermediateState(name = "seen", type = "BOOLEAN") }) +@GroupingAggregator +class MaxIpAggregator { + private static final BytesRef INIT_VALUE = new BytesRef(new byte[16]); + + private static boolean isBetter(BytesRef value, BytesRef otherValue) { + return value.compareTo(otherValue) > 0; + } + + public static SingleState initSingle() { + return new SingleState(); + } + + public static void combine(SingleState state, BytesRef value) { + state.add(value); + } + + public static void combineIntermediate(SingleState state, BytesRef value, boolean seen) { + if (seen) { + combine(state, value); + } + } + + public static Block evaluateFinal(SingleState state, DriverContext driverContext) { + return state.toBlock(driverContext); + } + + public static GroupingState initGrouping(BigArrays bigArrays) { + return new GroupingState(bigArrays); + } + + public static void combine(GroupingState state, int groupId, BytesRef value) { + state.add(groupId, value); + } + + public static void combineIntermediate(GroupingState state, int groupId, BytesRef value, boolean seen) { + if (seen) { + state.add(groupId, value); + } + } + + public static void combineStates(GroupingState state, int groupId, GroupingState otherState, int otherGroupId) { + state.combine(groupId, otherState, otherGroupId); + } + + public static Block evaluateFinal(GroupingState state, IntVector selected, DriverContext driverContext) { + return state.toBlock(selected, driverContext); + } + + public static class GroupingState implements Releasable { + private final BytesRef scratch = new BytesRef(); + private final IpArrayState internalState; + + private GroupingState(BigArrays bigArrays) { + this.internalState = new IpArrayState(bigArrays, INIT_VALUE); + } + + public void add(int groupId, BytesRef value) { + if (isBetter(value, internalState.getOrDefault(groupId, scratch))) { + internalState.set(groupId, value); + } + } + + public void combine(int groupId, GroupingState otherState, int otherGroupId) { + if (otherState.internalState.hasValue(otherGroupId)) { + add(groupId, otherState.internalState.get(otherGroupId, otherState.scratch)); + } + } + + void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + internalState.toIntermediate(blocks, offset, selected, driverContext); + } + + Block toBlock(IntVector selected, DriverContext driverContext) { + return internalState.toValuesBlock(selected, driverContext); + } + + void enableGroupIdTracking(SeenGroupIds seen) { + internalState.enableGroupIdTracking(seen); + } + + @Override + public void close() { + Releasables.close(internalState); + } + } + + public static class SingleState implements Releasable { + private final BytesRef internalState; + private boolean seen; + + private SingleState() { + this.internalState = BytesRef.deepCopyOf(INIT_VALUE); + this.seen = false; + } + + public void add(BytesRef value) { + if (isBetter(value, internalState)) { + seen = true; + System.arraycopy(value.bytes, value.offset, internalState.bytes, 0, internalState.length); + } + } + + void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = driverContext.blockFactory().newConstantBytesRefBlockWith(internalState, 1); + blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); + } + + Block toBlock(DriverContext driverContext) { + if (seen == false) { + return driverContext.blockFactory().newConstantNullBlock(1); + } + + return driverContext.blockFactory().newConstantBytesRefBlockWith(internalState, 1); + } + + @Override + public void close() { + // Nothing to close + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinIpAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinIpAggregator.java new file mode 100644 index 0000000000000..8313756851c1f --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinIpAggregator.java @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; + +@Aggregator({ @IntermediateState(name = "max", type = "BYTES_REF"), @IntermediateState(name = "seen", type = "BOOLEAN") }) +@GroupingAggregator +class MinIpAggregator { + private static final BytesRef INIT_VALUE = new BytesRef(new byte[] { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }); + + private static boolean isBetter(BytesRef value, BytesRef otherValue) { + return value.compareTo(otherValue) < 0; + } + + public static SingleState initSingle() { + return new SingleState(); + } + + public static void combine(SingleState state, BytesRef value) { + state.add(value); + } + + public static void combineIntermediate(SingleState state, BytesRef value, boolean seen) { + if (seen) { + combine(state, value); + } + } + + public static Block evaluateFinal(SingleState state, DriverContext driverContext) { + return state.toBlock(driverContext); + } + + public static GroupingState initGrouping(BigArrays bigArrays) { + return new GroupingState(bigArrays); + } + + public static void combine(GroupingState state, int groupId, BytesRef value) { + state.add(groupId, value); + } + + public static void combineIntermediate(GroupingState state, int groupId, BytesRef value, boolean seen) { + if (seen) { + state.add(groupId, value); + } + } + + public static void combineStates(GroupingState state, int groupId, GroupingState otherState, int otherGroupId) { + state.combine(groupId, otherState, otherGroupId); + } + + public static Block evaluateFinal(GroupingState state, IntVector selected, DriverContext driverContext) { + return state.toBlock(selected, driverContext); + } + + public static class GroupingState implements Releasable { + private final BytesRef scratch = new BytesRef(); + private final IpArrayState internalState; + + private GroupingState(BigArrays bigArrays) { + this.internalState = new IpArrayState(bigArrays, INIT_VALUE); + } + + public void add(int groupId, BytesRef value) { + if (isBetter(value, internalState.getOrDefault(groupId, scratch))) { + internalState.set(groupId, value); + } + } + + public void combine(int groupId, GroupingState otherState, int otherGroupId) { + if (otherState.internalState.hasValue(otherGroupId)) { + add(groupId, otherState.internalState.get(otherGroupId, otherState.scratch)); + } + } + + void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + internalState.toIntermediate(blocks, offset, selected, driverContext); + } + + Block toBlock(IntVector selected, DriverContext driverContext) { + return internalState.toValuesBlock(selected, driverContext); + } + + void enableGroupIdTracking(SeenGroupIds seen) { + internalState.enableGroupIdTracking(seen); + } + + @Override + public void close() { + Releasables.close(internalState); + } + } + + public static class SingleState implements Releasable { + private final BytesRef internalState; + private boolean seen; + + private SingleState() { + this.internalState = BytesRef.deepCopyOf(INIT_VALUE); + this.seen = false; + } + + public void add(BytesRef value) { + if (isBetter(value, internalState)) { + seen = true; + System.arraycopy(value.bytes, value.offset, internalState.bytes, 0, internalState.length); + } + } + + void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = driverContext.blockFactory().newConstantBytesRefBlockWith(internalState, 1); + blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); + } + + Block toBlock(DriverContext driverContext) { + if (seen == false) { + return driverContext.blockFactory().newConstantNullBlock(1); + } + + return driverContext.blockFactory().newConstantBytesRefBlockWith(internalState, 1); + } + + @Override + public void close() { + // Nothing to close + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ArrayState.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ArrayState.java.st index 10dbd9f423725..ad0ffc1d7e993 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ArrayState.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ArrayState.java.st @@ -70,7 +70,11 @@ $endif$ } $type$ getOrDefault(int groupId) { +$if(boolean)$ + return groupId < size ? values.get(groupId) : init; +$else$ return groupId < values.size() ? values.get(groupId) : init; +$endif$ } void set(int groupId, $type$ value) { @@ -139,7 +143,7 @@ $endif$ ) { for (int i = 0; i < selected.getPositionCount(); i++) { int group = selected.getInt(i); - if (group < values.size()) { + if (group < $if(boolean)$size$else$values.size()$endif$) { valuesBuilder.append$Type$(values.get(group)); } else { valuesBuilder.append$Type$($if(boolean)$false$else$0$endif$); // TODO can we just use null? diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-TopAggregator.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-TopAggregator.java.st index 41d0224f37214..b97d26ee6147d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-TopAggregator.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-TopAggregator.java.st @@ -7,6 +7,9 @@ package org.elasticsearch.compute.aggregation; +$if(Ip)$ +import org.apache.lucene.util.BytesRef; +$endif$ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.compute.ann.Aggregator; import org.elasticsearch.compute.ann.GroupingAggregator; @@ -20,7 +23,7 @@ import org.elasticsearch.compute.data.IntVector; $if(long)$ import org.elasticsearch.compute.data.$Type$Block; $endif$ -import org.elasticsearch.compute.data.sort.$Type$BucketedSort; +import org.elasticsearch.compute.data.sort.$Name$BucketedSort; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; @@ -28,10 +31,13 @@ import org.elasticsearch.search.sort.SortOrder; /** * Aggregates the top N field values for $type$. + *

    + * This class is generated. Edit `X-TopAggregator.java.st` to edit this file. + *

    */ @Aggregator({ @IntermediateState(name = "top", type = "$TYPE$_BLOCK") }) @GroupingAggregator -class Top$Type$Aggregator { +class Top$Name$Aggregator { public static SingleState initSingle(BigArrays bigArrays, int limit, boolean ascending) { return new SingleState(bigArrays, limit, ascending); } @@ -43,9 +49,16 @@ class Top$Type$Aggregator { public static void combineIntermediate(SingleState state, $Type$Block values) { int start = values.getFirstValueIndex(0); int end = start + values.getValueCount(0); +$if(Ip)$ + var scratch = new BytesRef(); + for (int i = start; i < end; i++) { + combine(state, values.get$Type$(i, scratch)); + } +$else$ for (int i = start; i < end; i++) { combine(state, values.get$Type$(i)); } +$endif$ } public static Block evaluateFinal(SingleState state, DriverContext driverContext) { @@ -63,9 +76,16 @@ class Top$Type$Aggregator { public static void combineIntermediate(GroupingState state, int groupId, $Type$Block values, int valuesPosition) { int start = values.getFirstValueIndex(valuesPosition); int end = start + values.getValueCount(valuesPosition); +$if(Ip)$ + var scratch = new BytesRef(); + for (int i = start; i < end; i++) { + combine(state, groupId, values.get$Type$(i, scratch)); + } +$else$ for (int i = start; i < end; i++) { combine(state, groupId, values.get$Type$(i)); } +$endif$ } public static void combineStates(GroupingState current, int groupId, GroupingState state, int statePosition) { @@ -77,10 +97,10 @@ class Top$Type$Aggregator { } public static class GroupingState implements Releasable { - private final $Type$BucketedSort sort; + private final $Name$BucketedSort sort; private GroupingState(BigArrays bigArrays, int limit, boolean ascending) { - this.sort = new $Type$BucketedSort(bigArrays, ascending ? SortOrder.ASC : SortOrder.DESC, limit); + this.sort = new $Name$BucketedSort(bigArrays, ascending ? SortOrder.ASC : SortOrder.DESC, limit); } public void add(int groupId, $type$ value) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/table/RowInTableLookup.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/table/RowInTableLookup.java index 4fa582e761e18..8bc9bacf8ff21 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/table/RowInTableLookup.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/table/RowInTableLookup.java @@ -37,6 +37,9 @@ public abstract sealed class RowInTableLookup implements Releasable permits Empt public abstract String toString(); public static RowInTableLookup build(BlockFactory blockFactory, Block[] keys) { + if (keys.length < 1) { + throw new IllegalArgumentException("expected [keys] to be non-empty"); + } int positions = keys[0].getPositionCount(); for (int k = 0; k < keys.length; k++) { if (positions != keys[k].getPositionCount()) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st index dc6f4ee1003cf..3d35832faebaf 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st @@ -129,15 +129,15 @@ $endif$ if (vector != null) { out.writeByte(SERIALIZE_BLOCK_VECTOR); vector.writeTo(out); - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_BLOCK) && this instanceof $Type$ArrayBlock b) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof $Type$ArrayBlock b) { out.writeByte(SERIALIZE_BLOCK_ARRAY); b.writeArrayBlock(out); $if(BytesRef)$ - } else if (version.onOrAfter(TransportVersions.ESQL_ORDINAL_BLOCK) && this instanceof OrdinalBytesRefBlock b && b.isDense()) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof OrdinalBytesRefBlock b && b.isDense()) { out.writeByte(SERIALIZE_BLOCK_ORDINAL); b.writeOrdinalBlock(out); $else$ - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_ARRAY) && this instanceof $Type$BigArrayBlock b) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof $Type$BigArrayBlock b) { out.writeByte(SERIALIZE_BLOCK_BIG_ARRAY); b.writeArrayBlock(out); $endif$ diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st index 28332648b5d3f..bc928dc7178c5 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st @@ -166,15 +166,15 @@ $if(BytesRef)$ $else$ out.write$Type$(get$Type$(0)); $endif$ - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_VECTOR) && this instanceof $Type$ArrayVector v) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof $Type$ArrayVector v) { out.writeByte(SERIALIZE_VECTOR_ARRAY); v.writeArrayVector(positions, out); $if(BytesRef)$ - } else if (version.onOrAfter(TransportVersions.ESQL_ORDINAL_BLOCK) && this instanceof OrdinalBytesRefVector v && v.isDense()) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof OrdinalBytesRefVector v && v.isDense()) { out.writeByte(SERIALIZE_VECTOR_ORDINAL); v.writeOrdinalVector(out); $else$ - } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_VECTOR) && this instanceof $Type$BigArrayVector v) { + } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof $Type$BigArrayVector v) { out.writeByte(SERIALIZE_VECTOR_BIG_ARRAY); v.writeArrayVector(positions, out); $endif$ diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/BooleanBucketedSort.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/BooleanBucketedSort.java new file mode 100644 index 0000000000000..eb66d2c836d2f --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/BooleanBucketedSort.java @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data.sort; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.IntArray; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.search.sort.SortOrder; + +import java.util.stream.IntStream; + +/** + * Aggregates the top N boolean values per bucket. + * This class collects by just keeping the count of true and false values. + */ +public class BooleanBucketedSort implements Releasable { + + private final BigArrays bigArrays; + private final SortOrder order; + private final int bucketSize; + /** + * An array containing all the values on all buckets. The structure is as follows: + *

    + * For each bucket, there are 2 values: The first keeps the count of true values, and the second the count of false values. + *

    + */ + private IntArray values; + + public BooleanBucketedSort(BigArrays bigArrays, SortOrder order, int bucketSize) { + this.bigArrays = bigArrays; + this.order = order; + this.bucketSize = bucketSize; + + boolean success = false; + try { + values = bigArrays.newIntArray(0, true); + success = true; + } finally { + if (success == false) { + close(); + } + } + } + + /** + * Collects a {@code value} into a {@code bucket}. + *

    + * It may or may not be inserted in the heap, depending on if it is better than the current root. + *

    + */ + public void collect(boolean value, int bucket) { + long rootIndex = (long) bucket * 2; + + long requiredSize = rootIndex + 2; + if (values.size() < requiredSize) { + grow(requiredSize); + } + + if (value) { + values.increment(rootIndex + 1, 1); + } else { + values.increment(rootIndex, 1); + } + } + + /** + * The order of the sort. + */ + public SortOrder getOrder() { + return order; + } + + /** + * The number of values to store per bucket. + */ + public int getBucketSize() { + return bucketSize; + } + + /** + * Merge the values from {@code other}'s {@code otherGroupId} into {@code groupId}. + */ + public void merge(int groupId, BooleanBucketedSort other, int otherGroupId) { + long otherRootIndex = (long) otherGroupId * 2; + + if (other.values.size() < otherRootIndex + 2) { + return; + } + + int falseValues = other.values.get(otherRootIndex); + int trueValues = other.values.get(otherRootIndex + 1); + + if (falseValues + trueValues == 0) { + return; + } + + long rootIndex = (long) groupId * 2; + + long requiredSize = rootIndex + 2; + if (values.size() < requiredSize) { + grow(requiredSize); + } + + values.increment(rootIndex, falseValues); + values.increment(rootIndex + 1, trueValues); + } + + /** + * Creates a block with the values from the {@code selected} groups. + */ + public Block toBlock(BlockFactory blockFactory, IntVector selected) { + // Check if the selected groups are all empty, to avoid allocating extra memory + if (bucketSize == 0 || IntStream.range(0, selected.getPositionCount()).map(selected::getInt).noneMatch(bucket -> { + long rootIndex = (long) bucket * 2; + + if (values.size() < rootIndex + 2) { + return false; + } + + var size = values.get(rootIndex) + values.get(rootIndex + 1); + return size > 0; + })) { + return blockFactory.newConstantNullBlock(selected.getPositionCount()); + } + + try (var builder = blockFactory.newBooleanBlockBuilder(selected.getPositionCount())) { + for (int s = 0; s < selected.getPositionCount(); s++) { + int bucket = selected.getInt(s); + + long rootIndex = (long) bucket * 2; + + if (values.size() < rootIndex + 2) { + builder.appendNull(); + continue; + } + + int falseValues = values.get(rootIndex); + int trueValues = values.get(rootIndex + 1); + long totalValues = (long) falseValues + trueValues; + + if (totalValues == 0) { + builder.appendNull(); + continue; + } + + if (totalValues == 1) { + builder.appendBoolean(trueValues > 0); + continue; + } + + builder.beginPositionEntry(); + if (order == SortOrder.ASC) { + int falseValuesToAdd = Math.min(falseValues, bucketSize); + int trueValuesToAdd = Math.min(trueValues, bucketSize - falseValuesToAdd); + for (int i = 0; i < falseValuesToAdd; i++) { + builder.appendBoolean(false); + } + for (int i = 0; i < trueValuesToAdd; i++) { + builder.appendBoolean(true); + } + } else { + int trueValuesToAdd = Math.min(trueValues, bucketSize); + int falseValuesToAdd = Math.min(falseValues, bucketSize - trueValuesToAdd); + for (int i = 0; i < trueValuesToAdd; i++) { + builder.appendBoolean(true); + } + for (int i = 0; i < falseValuesToAdd; i++) { + builder.appendBoolean(false); + } + } + builder.endPositionEntry(); + } + return builder.build(); + } + } + + /** + * Allocate storage for more buckets and store the "next gather offset" + * for those new buckets. + */ + private void grow(long minSize) { + values = bigArrays.grow(values, minSize); + } + + @Override + public final void close() { + Releasables.close(values); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/IpBucketedSort.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/IpBucketedSort.java new file mode 100644 index 0000000000000..0fd38c18d7504 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/IpBucketedSort.java @@ -0,0 +1,405 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data.sort; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.BitArray; +import org.elasticsearch.common.util.ByteArray; +import org.elasticsearch.common.util.ByteUtils; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; + +import java.util.Arrays; +import java.util.stream.IntStream; + +/** + * Aggregates the top N IP values per bucket. + * See {@link BucketedSort} for more information. + */ +public class IpBucketedSort implements Releasable { + private static final int IP_LENGTH = 16; + + // BytesRefs used in internal methods + private final BytesRef scratch1 = new BytesRef(); + private final BytesRef scratch2 = new BytesRef(); + /** + * Bytes used as temporal storage for scratches + */ + private final byte[] scratchBytes = new byte[IP_LENGTH]; + + private final BigArrays bigArrays; + private final SortOrder order; + private final int bucketSize; + /** + * {@code true} if the bucket is in heap mode, {@code false} if + * it is still gathering. + */ + private final BitArray heapMode; + /** + * An array containing all the values on all buckets. The structure is as follows: + *

    + * For each bucket, there are bucketSize elements, based on the bucket id (0, 1, 2...). + * Then, for each bucket, it can be in 2 states: + *

    + *
      + *
    • + * Gather mode: All buckets start in gather mode, and remain here while they have less than bucketSize elements. + * In gather mode, the elements are stored in the array from the highest index to the lowest index. + * The lowest index contains the offset to the next slot to be filled. + *

      + * This allows us to insert elements in O(1) time. + *

      + *

      + * When the bucketSize-th element is collected, the bucket transitions to heap mode, by heapifying its contents. + *

      + *
      • + *
    • + * Heap mode: The bucket slots are organized as a min heap structure. + *

      + * The root of the heap is the minimum value in the bucket, + * which allows us to quickly discard new values that are not in the top N. + *

      + *
      • + *
    + */ + private ByteArray values; + + public IpBucketedSort(BigArrays bigArrays, SortOrder order, int bucketSize) { + this.bigArrays = bigArrays; + this.order = order; + this.bucketSize = bucketSize; + heapMode = new BitArray(0, bigArrays); + + boolean success = false; + try { + values = bigArrays.newByteArray(0, false); + success = true; + } finally { + if (success == false) { + close(); + } + } + } + + /** + * Collects a {@code value} into a {@code bucket}. + *

    + * It may or may not be inserted in the heap, depending on if it is better than the current root. + *

    + */ + public void collect(BytesRef value, int bucket) { + assert value.length == IP_LENGTH; + long rootIndex = (long) bucket * bucketSize; + if (inHeapMode(bucket)) { + if (betterThan(value, get(rootIndex, scratch1))) { + set(rootIndex, value); + downHeap(rootIndex, 0); + } + return; + } + // Gathering mode + long requiredSize = (rootIndex + bucketSize) * IP_LENGTH; + if (values.size() < requiredSize) { + grow(requiredSize); + } + int next = getNextGatherOffset(rootIndex); + assert 0 <= next && next < bucketSize + : "Expected next to be in the range of valid buckets [0 <= " + next + " < " + bucketSize + "]"; + long index = next + rootIndex; + set(index, value); + if (next == 0) { + heapMode.set(bucket); + heapify(rootIndex); + } else { + setNextGatherOffset(rootIndex, next - 1); + } + } + + /** + * The order of the sort. + */ + public SortOrder getOrder() { + return order; + } + + /** + * The number of values to store per bucket. + */ + public int getBucketSize() { + return bucketSize; + } + + /** + * Get the first and last indexes (inclusive, exclusive) of the values for a bucket. + * Returns [0, 0] if the bucket has never been collected. + */ + private Tuple getBucketValuesIndexes(int bucket) { + long rootIndex = (long) bucket * bucketSize; + if (rootIndex >= values.size() / IP_LENGTH) { + // We've never seen this bucket. + return Tuple.tuple(0L, 0L); + } + long start = inHeapMode(bucket) ? rootIndex : (rootIndex + getNextGatherOffset(rootIndex) + 1); + long end = rootIndex + bucketSize; + return Tuple.tuple(start, end); + } + + /** + * Merge the values from {@code other}'s {@code otherGroupId} into {@code groupId}. + */ + public void merge(int groupId, IpBucketedSort other, int otherGroupId) { + var otherBounds = other.getBucketValuesIndexes(otherGroupId); + var scratch = new BytesRef(); + + // TODO: This can be improved for heapified buckets by making use of the heap structures + for (long i = otherBounds.v1(); i < otherBounds.v2(); i++) { + collect(other.get(i, scratch), groupId); + } + } + + /** + * Creates a block with the values from the {@code selected} groups. + */ + public Block toBlock(BlockFactory blockFactory, IntVector selected) { + // Check if the selected groups are all empty, to avoid allocating extra memory + if (IntStream.range(0, selected.getPositionCount()).map(selected::getInt).noneMatch(bucket -> { + var bounds = this.getBucketValuesIndexes(bucket); + var size = bounds.v2() - bounds.v1(); + + return size > 0; + })) { + return blockFactory.newConstantNullBlock(selected.getPositionCount()); + } + + // Used to sort the values in the bucket. + var bucketValues = new BytesRef[bucketSize]; + + try (var builder = blockFactory.newBytesRefBlockBuilder(selected.getPositionCount())) { + for (int s = 0; s < selected.getPositionCount(); s++) { + int bucket = selected.getInt(s); + + var bounds = getBucketValuesIndexes(bucket); + var size = bounds.v2() - bounds.v1(); + + if (size == 0) { + builder.appendNull(); + continue; + } + + if (size == 1) { + builder.appendBytesRef(get(bounds.v1(), scratch1)); + continue; + } + + for (int i = 0; i < size; i++) { + bucketValues[i] = get(bounds.v1() + i, new BytesRef()); + } + + // TODO: Make use of heap structures to faster iterate in order instead of copying and sorting + Arrays.sort(bucketValues, 0, (int) size); + + builder.beginPositionEntry(); + if (order == SortOrder.ASC) { + for (int i = 0; i < size; i++) { + builder.appendBytesRef(bucketValues[i]); + } + } else { + for (int i = (int) size - 1; i >= 0; i--) { + builder.appendBytesRef(bucketValues[i]); + } + } + builder.endPositionEntry(); + } + return builder.build(); + } + } + + /** + * Is this bucket a min heap {@code true} or in gathering mode {@code false}? + */ + private boolean inHeapMode(int bucket) { + return heapMode.get(bucket); + } + + /** + * Get the next index that should be "gathered" for a bucket rooted + * at {@code rootIndex}. + *

    + * Using the first 4 bytes of the element to store the next gather offset. + *

    + */ + private int getNextGatherOffset(long rootIndex) { + values.get(rootIndex * IP_LENGTH, Integer.BYTES, scratch1); + assert scratch1.length == Integer.BYTES; + return ByteUtils.readIntLE(scratch1.bytes, scratch1.offset); + } + + /** + * Set the next index that should be "gathered" for a bucket rooted + * at {@code rootIndex}. + *

    + * Using the first {@code Integer.BYTES} bytes of the element to store the next gather offset. + *

    + */ + private void setNextGatherOffset(long rootIndex, int offset) { + scratch1.bytes = scratchBytes; + scratch1.offset = 0; + scratch1.length = Integer.BYTES; + ByteUtils.writeIntLE(offset, scratch1.bytes, scratch1.offset); + values.set(rootIndex * IP_LENGTH, scratch1.bytes, scratch1.offset, scratch1.length); + } + + /** + * {@code true} if the entry at index {@code lhs} is "better" than + * the entry at {@code rhs}. "Better" in this means "lower" for + * {@link SortOrder#ASC} and "higher" for {@link SortOrder#DESC}. + */ + private boolean betterThan(BytesRef lhs, BytesRef rhs) { + return getOrder().reverseMul() * lhs.compareTo(rhs) < 0; + } + + /** + * Swap the data at two indices. + */ + private void swap(long lhs, long rhs) { + // var tmp = values.get(lhs); + values.get(lhs * IP_LENGTH, IP_LENGTH, scratch1); + assert scratch1.length == IP_LENGTH; + System.arraycopy(scratch1.bytes, scratch1.offset, scratchBytes, 0, scratch1.length); + + // values.set(lhs, values.get(rhs)); + values.get(rhs * IP_LENGTH, IP_LENGTH, scratch2); + assert scratch2.length == IP_LENGTH; + values.set(lhs * IP_LENGTH, scratch2.bytes, scratch2.offset, scratch2.length); + + // values.set(rhs, tmp); + scratch1.bytes = scratchBytes; + scratch1.offset = 0; + values.set(rhs * IP_LENGTH, scratch1.bytes, scratch1.offset, scratch1.length); + } + + /** + * Allocate storage for more buckets and store the "next gather offset" + * for those new buckets. + */ + private void grow(long minSize) { + long oldMax = values.size() / IP_LENGTH; + values = bigArrays.grow(values, minSize); + // Set the next gather offsets for all newly allocated buckets. + setNextGatherOffsets(oldMax - (oldMax % bucketSize)); + } + + /** + * Maintain the "next gather offsets" for newly allocated buckets. + */ + private void setNextGatherOffsets(long startingAt) { + int nextOffset = bucketSize - 1; + for (long bucketRoot = startingAt; bucketRoot < values.size() / IP_LENGTH; bucketRoot += bucketSize) { + setNextGatherOffset(bucketRoot, nextOffset); + } + } + + /** + * Heapify a bucket whose entries are in random order. + *

    + * This works by validating the heap property on each node, iterating + * "upwards", pushing any out of order parents "down". Check out the + * wikipedia + * entry on binary heaps for more about this. + *

    + *

    + * While this *looks* like it could easily be {@code O(n * log n)}, it is + * a fairly well studied algorithm attributed to Floyd. There's + * been a bunch of work that puts this at {@code O(n)}, close to 1.88n worst + * case. + *

    + * + * @param rootIndex the index the start of the bucket + */ + private void heapify(long rootIndex) { + int maxParent = bucketSize / 2 - 1; + for (int parent = maxParent; parent >= 0; parent--) { + downHeap(rootIndex, parent); + } + } + + /** + * Correct the heap invariant of a parent and its children. This + * runs in {@code O(log n)} time. + * @param rootIndex index of the start of the bucket + * @param parent Index within the bucket of the parent to check. + * For example, 0 is the "root". + */ + private void downHeap(long rootIndex, int parent) { + while (true) { + long parentIndex = rootIndex + parent; + int worst = parent; + long worstIndex = parentIndex; + int leftChild = parent * 2 + 1; + long leftIndex = rootIndex + leftChild; + if (leftChild < bucketSize) { + if (betterThan(get(worstIndex, scratch1), get(leftIndex, scratch2))) { + worst = leftChild; + worstIndex = leftIndex; + } + int rightChild = leftChild + 1; + long rightIndex = rootIndex + rightChild; + if (rightChild < bucketSize && betterThan(get(worstIndex, scratch1), get(rightIndex, scratch2))) { + worst = rightChild; + worstIndex = rightIndex; + } + } + if (worst == parent) { + break; + } + swap(worstIndex, parentIndex); + parent = worst; + } + } + + /** + * Get the IP value at {@code index} and store it in {@code scratch}. + * Returns {@code scratch}. + *

    + * {@code index} is an IP index, not a byte index. + *

    + */ + private BytesRef get(long index, BytesRef scratch) { + values.get(index * IP_LENGTH, IP_LENGTH, scratch); + assert scratch.length == IP_LENGTH; + return scratch; + } + + /** + * Set the IP value at {@code index}. + *

    + * {@code index} is an IP index, not a byte index. + *

    + */ + private void set(long index, BytesRef value) { + assert value.length == IP_LENGTH; + values.set(index * IP_LENGTH, value.bytes, value.offset, value.length); + } + + @Override + public final void close() { + Releasables.close(values, heapMode); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java index 11ccfb55a77aa..6f75298e95dd7 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java @@ -309,7 +309,7 @@ private Status(LuceneOperator operator) { processedQueries = Collections.emptySet(); processedShards = Collections.emptySet(); } - processingNanos = in.getTransportVersion().onOrAfter(TransportVersions.ESQL_TIMINGS) ? in.readVLong() : 0; + processingNanos = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) ? in.readVLong() : 0; sliceIndex = in.readVInt(); totalSlices = in.readVInt(); pagesEmitted = in.readVInt(); @@ -325,7 +325,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeCollection(processedQueries, StreamOutput::writeString); out.writeCollection(processedShards, StreamOutput::writeString); } - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_TIMINGS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeVLong(processingNanos); } out.writeVInt(sliceIndex); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AbstractPageMappingOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AbstractPageMappingOperator.java index 800b648711f26..05913b7dd5f69 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AbstractPageMappingOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AbstractPageMappingOperator.java @@ -112,13 +112,13 @@ public Status(long processNanos, int pagesProcessed) { } protected Status(StreamInput in) throws IOException { - processNanos = in.getTransportVersion().onOrAfter(TransportVersions.ESQL_TIMINGS) ? in.readVLong() : 0; + processNanos = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) ? in.readVLong() : 0; pagesProcessed = in.readVInt(); } @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_TIMINGS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeVLong(processNanos); } out.writeVInt(pagesProcessed); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java index 20d3f0166f1cb..260b72dc35d9c 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java @@ -265,7 +265,7 @@ public String toString() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ESQL_TIMINGS; + return TransportVersions.V_8_14_0; } } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java index 0fed88370a144..79359737b1b35 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java @@ -319,7 +319,7 @@ public String toString() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ESQL_ENRICH_OPERATOR_STATUS; + return TransportVersions.V_8_14_0; } } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java index 00c3771540867..414fbbbca8294 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java @@ -57,7 +57,7 @@ public DriverProfile(long tookNanos, long cpuNanos, long iterations, List(keys.length); Block[] blocks = new Block[keys.length]; diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ArrayStateTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ArrayStateTests.java index 1d8df3caf7b76..da10f94f6fb8a 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ArrayStateTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ArrayStateTests.java @@ -9,11 +9,14 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.compute.data.BlockTestUtils; import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.core.type.DataType; import java.util.ArrayList; import java.util.List; @@ -26,19 +29,31 @@ public static List params() { List params = new ArrayList<>(); for (boolean inOrder : new boolean[] { true, false }) { - params.add(new Object[] { ElementType.INT, 1000, inOrder }); - params.add(new Object[] { ElementType.LONG, 1000, inOrder }); - params.add(new Object[] { ElementType.DOUBLE, 1000, inOrder }); + params.add(new Object[] { DataType.INTEGER, 1000, inOrder }); + params.add(new Object[] { DataType.LONG, 1000, inOrder }); + params.add(new Object[] { DataType.FLOAT, 1000, inOrder }); + params.add(new Object[] { DataType.DOUBLE, 1000, inOrder }); + params.add(new Object[] { DataType.IP, 1000, inOrder }); } return params; } + private final DataType type; private final ElementType elementType; private final int valueCount; private final boolean inOrder; - public ArrayStateTests(ElementType elementType, int valueCount, boolean inOrder) { - this.elementType = elementType; + public ArrayStateTests(DataType type, int valueCount, boolean inOrder) { + this.type = type; + this.elementType = switch (type) { + case INTEGER -> ElementType.INT; + case LONG -> ElementType.LONG; + case FLOAT -> ElementType.FLOAT; + case DOUBLE -> ElementType.DOUBLE; + case BOOLEAN -> ElementType.BOOLEAN; + case IP -> ElementType.BYTES_REF; + default -> throw new IllegalArgumentException(); + }; this.valueCount = valueCount; this.inOrder = inOrder; } @@ -155,34 +170,47 @@ private void setAll(AbstractArrayState state, List values, int offset) { } private AbstractArrayState newState() { - return switch (elementType) { - case INT -> new IntArrayState(BigArrays.NON_RECYCLING_INSTANCE, 1); + return switch (type) { + case INTEGER -> new IntArrayState(BigArrays.NON_RECYCLING_INSTANCE, 1); case LONG -> new LongArrayState(BigArrays.NON_RECYCLING_INSTANCE, 1); + case FLOAT -> new FloatArrayState(BigArrays.NON_RECYCLING_INSTANCE, 1); case DOUBLE -> new DoubleArrayState(BigArrays.NON_RECYCLING_INSTANCE, 1); + case BOOLEAN -> new BooleanArrayState(BigArrays.NON_RECYCLING_INSTANCE, false); + case IP -> new IpArrayState(BigArrays.NON_RECYCLING_INSTANCE, new BytesRef(new byte[16])); default -> throw new IllegalArgumentException(); }; } - private void set(AbstractArrayState state, int groupdId, Object value) { - switch (elementType) { - case INT -> ((IntArrayState) state).set(groupdId, (Integer) value); - case LONG -> ((LongArrayState) state).set(groupdId, (Long) value); - case DOUBLE -> ((DoubleArrayState) state).set(groupdId, (Double) value); + private void set(AbstractArrayState state, int groupId, Object value) { + switch (type) { + case INTEGER -> ((IntArrayState) state).set(groupId, (Integer) value); + case LONG -> ((LongArrayState) state).set(groupId, (Long) value); + case FLOAT -> ((FloatArrayState) state).set(groupId, (Float) value); + case DOUBLE -> ((DoubleArrayState) state).set(groupId, (Double) value); + case BOOLEAN -> ((BooleanArrayState) state).set(groupId, (Boolean) value); + case IP -> ((IpArrayState) state).set(groupId, (BytesRef) value); default -> throw new IllegalArgumentException(); } } private Object get(AbstractArrayState state, int index) { - return switch (elementType) { - case INT -> ((IntArrayState) state).get(index); + return switch (type) { + case INTEGER -> ((IntArrayState) state).get(index); case LONG -> ((LongArrayState) state).get(index); + case FLOAT -> ((FloatArrayState) state).get(index); case DOUBLE -> ((DoubleArrayState) state).get(index); + case BOOLEAN -> ((BooleanArrayState) state).get(index); + case IP -> ((IpArrayState) state).get(index, new BytesRef()); default -> throw new IllegalArgumentException(); }; } private Object randomValue() { - return BlockTestUtils.randomValue(elementType); + return switch (type) { + case INTEGER, LONG, FLOAT, DOUBLE, BOOLEAN -> BlockTestUtils.randomValue(elementType); + case IP -> new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean()))); + default -> throw new IllegalArgumentException(); + }; } private Object nullableRandomValue() { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java index 3436d6b537611..f6558d54b2779 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java @@ -7,6 +7,7 @@ package org.elasticsearch.compute.aggregation; +import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.util.BitArray; import org.elasticsearch.compute.aggregation.blockhash.BlockHash; @@ -32,7 +33,9 @@ import org.elasticsearch.compute.operator.Operator; import org.elasticsearch.compute.operator.PositionMergingSourceOperator; import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.hamcrest.Matcher; import java.util.ArrayList; @@ -62,6 +65,18 @@ protected final int aggregatorIntermediateBlockCount() { protected abstract void assertSimpleGroup(List input, Block result, int position, Long group); + /** + * Returns the datatype this aggregator accepts. If null, all datatypes are accepted. + *

    + * Used to generate correct input for aggregators that require specific types. + * For example, IP aggregators require BytesRefs with a fixed size. + *

    + */ + @Nullable + protected DataType acceptedDataType() { + return null; + }; + @Override protected final Operator.OperatorFactory simpleWithMode(AggregatorMode mode) { List channels = mode.isInputPartial() ? range(1, 1 + aggregatorIntermediateBlockCount()).boxed().toList() : List.of(1); @@ -202,7 +217,12 @@ protected void appendNull(ElementType elementType, Block.Builder builder, int bl // Append a small random value to make sure we don't overflow on things like sums append(builder, switch (elementType) { case BOOLEAN -> randomBoolean(); - case BYTES_REF -> new BytesRef(randomAlphaOfLength(3)); + case BYTES_REF -> { + if (acceptedDataType() == DataType.IP) { + yield new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean()))); + } + yield new BytesRef(randomAlphaOfLength(3)); + } case FLOAT -> randomFloat(); case DOUBLE -> randomDouble(); case INT -> 1; diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunctionTests.java new file mode 100644 index 0000000000000..84488b5115e5d --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunctionTests.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.operator.SequenceBytesRefBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; + +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.equalTo; + +public class MaxIpAggregatorFunctionTests extends AggregatorFunctionTestCase { + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceBytesRefBlockSourceOperator( + blockFactory, + IntStream.range(0, size).mapToObj(l -> new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean())))) + ); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MaxIpAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "max of ips"; + } + + @Override + public void assertSimpleOutput(List input, Block result) { + Optional max = input.stream().flatMap(b -> allBytesRefs(b)).max(Comparator.naturalOrder()); + if (max.isEmpty()) { + assertThat(result.isNull(0), equalTo(true)); + return; + } + assertThat(result.isNull(0), equalTo(false)); + assertThat(BlockUtils.toJavaObject(result, 0), equalTo(max.get())); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxIpGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxIpGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..12e34fcf9a50e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxIpGroupingAggregatorFunctionTests.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongBytesRefTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.xpack.esql.core.type.DataType; + +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.equalTo; + +public class MaxIpGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new LongBytesRefTupleBlockSourceOperator( + blockFactory, + IntStream.range(0, size) + .mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean()))))) + ); + } + + @Override + protected DataType acceptedDataType() { + return DataType.IP; + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MaxIpAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "max of ips"; + } + + @Override + protected void assertSimpleGroup(List input, Block result, int position, Long group) { + Optional max = input.stream().flatMap(p -> allBytesRefs(p, group)).max(Comparator.naturalOrder()); + if (max.isEmpty()) { + assertThat(result.isNull(position), equalTo(true)); + return; + } + assertThat(result.isNull(position), equalTo(false)); + assertThat(BlockUtils.toJavaObject(result, position), equalTo(max.get())); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinIpAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinIpAggregatorFunctionTests.java new file mode 100644 index 0000000000000..17e9812d2e4e8 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinIpAggregatorFunctionTests.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.operator.SequenceBytesRefBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; + +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.equalTo; + +public class MinIpAggregatorFunctionTests extends AggregatorFunctionTestCase { + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceBytesRefBlockSourceOperator( + blockFactory, + IntStream.range(0, size).mapToObj(l -> new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean())))) + ); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MinIpAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "min of ips"; + } + + @Override + public void assertSimpleOutput(List input, Block result) { + Optional max = input.stream().flatMap(b -> allBytesRefs(b)).min(Comparator.naturalOrder()); + if (max.isEmpty()) { + assertThat(result.isNull(0), equalTo(true)); + return; + } + assertThat(result.isNull(0), equalTo(false)); + assertThat(BlockUtils.toJavaObject(result, 0), equalTo(max.get())); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinIpGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinIpGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..f51662ffee352 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinIpGroupingAggregatorFunctionTests.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongBytesRefTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.xpack.esql.core.type.DataType; + +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.equalTo; + +public class MinIpGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new LongBytesRefTupleBlockSourceOperator( + blockFactory, + IntStream.range(0, size) + .mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean()))))) + ); + } + + @Override + protected DataType acceptedDataType() { + return DataType.IP; + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MinIpAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "min of ips"; + } + + @Override + protected void assertSimpleGroup(List input, Block result, int position, Long group) { + Optional max = input.stream().flatMap(p -> allBytesRefs(p, group)).min(Comparator.naturalOrder()); + if (max.isEmpty()) { + assertThat(result.isNull(position), equalTo(true)); + return; + } + assertThat(result.isNull(position), equalTo(false)); + assertThat(BlockUtils.toJavaObject(result, position), equalTo(max.get())); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopBooleanAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopBooleanAggregatorFunctionTests.java new file mode 100644 index 0000000000000..662b963d32473 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopBooleanAggregatorFunctionTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.operator.SequenceBooleanBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; + +import java.util.List; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.contains; + +public class TopBooleanAggregatorFunctionTests extends AggregatorFunctionTestCase { + private static final int LIMIT = 100; + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceBooleanBlockSourceOperator(blockFactory, IntStream.range(0, size).mapToObj(l -> randomBoolean()).toList()); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new TopBooleanAggregatorFunctionSupplier(inputChannels, LIMIT, true); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "top of booleans"; + } + + @Override + public void assertSimpleOutput(List input, Block result) { + Object[] values = input.stream().flatMap(b -> allBooleans(b)).sorted().limit(LIMIT).toArray(Object[]::new); + assertThat((List) BlockUtils.toJavaObject(result, 0), contains(values)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopIpAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopIpAggregatorFunctionTests.java new file mode 100644 index 0000000000000..1594f66ed9fe2 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopIpAggregatorFunctionTests.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.operator.SequenceBytesRefBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; + +import java.util.List; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.contains; + +public class TopIpAggregatorFunctionTests extends AggregatorFunctionTestCase { + private static final int LIMIT = 100; + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceBytesRefBlockSourceOperator( + blockFactory, + IntStream.range(0, size).mapToObj(l -> new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean())))) + ); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new TopIpAggregatorFunctionSupplier(inputChannels, LIMIT, true); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "top of ips"; + } + + @Override + public void assertSimpleOutput(List input, Block result) { + Object[] values = input.stream().flatMap(b -> allBytesRefs(b)).sorted().limit(LIMIT).toArray(Object[]::new); + assertThat((List) BlockUtils.toJavaObject(result, 0), contains(values)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..da55ff2d7aab3 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunctionTests.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongBytesRefTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.xpack.esql.core.type.DataType; + +import java.util.List; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; + +public class TopIpGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + private static final int LIMIT = 100; + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new LongBytesRefTupleBlockSourceOperator( + blockFactory, + IntStream.range(0, size) + .mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean()))))) + ); + } + + @Override + protected DataType acceptedDataType() { + return DataType.IP; + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new TopIpAggregatorFunctionSupplier(inputChannels, LIMIT, true); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "top of ips"; + } + + @Override + protected void assertSimpleGroup(List input, Block result, int position, Long group) { + Object[] values = input.stream().flatMap(b -> allBytesRefs(b, group)).sorted().limit(LIMIT).toArray(Object[]::new); + if (values.length == 0) { + assertThat(result.isNull(position), equalTo(true)); + } else if (values.length == 1) { + assertThat(BlockUtils.toJavaObject(result, position), equalTo(values[0])); + } else { + assertThat((List) BlockUtils.toJavaObject(result, position), contains(values)); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BooleanBucketedSortTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BooleanBucketedSortTests.java new file mode 100644 index 0000000000000..fcece4b6480b8 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BooleanBucketedSortTests.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data.sort; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.search.sort.SortOrder; + +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; + +public class BooleanBucketedSortTests extends BucketedSortTestCase { + @Override + protected BooleanBucketedSort build(SortOrder sortOrder, int bucketSize) { + return new BooleanBucketedSort(bigArrays(), sortOrder, bucketSize); + } + + @Override + protected Boolean randomValue() { + return randomBoolean(); + } + + @Override + protected List threeSortedValues() { + return List.of(false, true, true); + } + + @Override + protected void collect(BooleanBucketedSort sort, Boolean value, int bucket) { + sort.collect(value, bucket); + } + + @Override + protected void merge(BooleanBucketedSort sort, int groupId, BooleanBucketedSort other, int otherGroupId) { + sort.merge(groupId, other, otherGroupId); + } + + @Override + protected Block toBlock(BooleanBucketedSort sort, BlockFactory blockFactory, IntVector selected) { + return sort.toBlock(blockFactory, selected); + } + + @Override + protected void assertBlockTypeAndValues(Block block, List values) { + assertThat(block.elementType(), equalTo(ElementType.BOOLEAN)); + var typedBlock = (BooleanBlock) block; + for (int i = 0; i < values.size(); i++) { + assertThat(typedBlock.getBoolean(i), equalTo(values.get(i))); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java index 9e1bc145ad4ca..f857f50b2d30f 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java @@ -25,113 +25,119 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.List; import static org.hamcrest.Matchers.equalTo; -public abstract class BucketedSortTestCase extends ESTestCase { +public abstract class BucketedSortTestCase> extends ESTestCase { /** * Build a {@link T} to test. Sorts built by this method shouldn't need scores. */ protected abstract T build(SortOrder sortOrder, int bucketSize); /** - * Build the expected correctly typed value for a value. + * A random value for testing, with the appropriate precision for the type we're testing. */ - protected abstract Object expectedValue(double v); + protected abstract V randomValue(); /** - * A random value for testing, with the appropriate precision for the type we're testing. + * Returns a list of 3 values, in ascending order. */ - protected abstract double randomValue(); + protected abstract List threeSortedValues(); /** * Collect a value into the sort. * @param value value to collect, always sent as double just to have * a number to test. Subclasses should cast to their favorite types */ - protected abstract void collect(T sort, double value, int bucket); + protected abstract void collect(T sort, V value, int bucket); protected abstract void merge(T sort, int groupId, T other, int otherGroupId); protected abstract Block toBlock(T sort, BlockFactory blockFactory, IntVector selected); - protected abstract void assertBlockTypeAndValues(Block block, Object... values); + protected abstract void assertBlockTypeAndValues(Block block, List values); public final void testNeverCalled() { SortOrder order = randomFrom(SortOrder.values()); try (T sort = build(order, 1)) { - assertBlock(sort, randomNonNegativeInt()); + assertBlock(sort, randomNonNegativeInt(), List.of()); } } public final void testSingleDoc() { try (T sort = build(randomFrom(SortOrder.values()), 1)) { - collect(sort, 1, 0); + var values = threeSortedValues(); + + collect(sort, values.get(0), 0); - assertBlock(sort, 0, expectedValue(1)); + assertBlock(sort, 0, List.of(values.get(0))); } } public final void testNonCompetitive() { try (T sort = build(SortOrder.DESC, 1)) { - collect(sort, 2, 0); - collect(sort, 1, 0); + var values = threeSortedValues(); - assertBlock(sort, 0, expectedValue(2)); + collect(sort, values.get(1), 0); + collect(sort, values.get(0), 0); + + assertBlock(sort, 0, List.of(values.get(1))); } } public final void testCompetitive() { try (T sort = build(SortOrder.DESC, 1)) { - collect(sort, 1, 0); - collect(sort, 2, 0); + var values = threeSortedValues(); - assertBlock(sort, 0, expectedValue(2)); - } - } + collect(sort, values.get(0), 0); + collect(sort, values.get(1), 0); - public final void testNegativeValue() { - try (T sort = build(SortOrder.DESC, 1)) { - collect(sort, -1, 0); - assertBlock(sort, 0, expectedValue(-1)); + assertBlock(sort, 0, List.of(values.get(1))); } } public final void testSomeBuckets() { try (T sort = build(SortOrder.DESC, 1)) { - collect(sort, 2, 0); - collect(sort, 2, 1); - collect(sort, 2, 2); - collect(sort, 3, 0); - - assertBlock(sort, 0, expectedValue(3)); - assertBlock(sort, 1, expectedValue(2)); - assertBlock(sort, 2, expectedValue(2)); - assertBlock(sort, 3); + var values = threeSortedValues(); + + collect(sort, values.get(1), 0); + collect(sort, values.get(1), 1); + collect(sort, values.get(1), 2); + collect(sort, values.get(2), 0); + + assertBlock(sort, 0, List.of(values.get(2))); + assertBlock(sort, 1, List.of(values.get(1))); + assertBlock(sort, 2, List.of(values.get(1))); + assertBlock(sort, 3, List.of()); } } public final void testBucketGaps() { try (T sort = build(SortOrder.DESC, 1)) { - collect(sort, 2, 0); - collect(sort, 2, 2); + var values = threeSortedValues(); - assertBlock(sort, 0, expectedValue(2)); - assertBlock(sort, 1); - assertBlock(sort, 2, expectedValue(2)); - assertBlock(sort, 3); + collect(sort, values.get(1), 0); + collect(sort, values.get(1), 2); + + assertBlock(sort, 0, List.of(values.get(1))); + assertBlock(sort, 1, List.of()); + assertBlock(sort, 2, List.of(values.get(1))); + assertBlock(sort, 3, List.of()); } } public final void testBucketsOutOfOrder() { try (T sort = build(SortOrder.DESC, 1)) { - collect(sort, 2, 1); - collect(sort, 2, 0); + var values = threeSortedValues(); + + collect(sort, values.get(1), 1); + collect(sort, values.get(1), 0); - assertBlock(sort, 0, expectedValue(2.0)); - assertBlock(sort, 1, expectedValue(2.0)); - assertBlock(sort, 2); + assertBlock(sort, 0, List.of(values.get(1))); + assertBlock(sort, 1, List.of(values.get(1))); + assertBlock(sort, 2, List.of()); } } @@ -143,194 +149,207 @@ public final void testManyBuckets() { } Collections.shuffle(Arrays.asList(buckets), random()); - double[] maxes = new double[buckets.length]; + var values = threeSortedValues(); + List maxes = new ArrayList(Collections.nCopies(buckets.length, null)); try (T sort = build(SortOrder.DESC, 1)) { for (int b : buckets) { - maxes[b] = 2; - collect(sort, 2, b); + maxes.set(b, values.get(1)); + collect(sort, values.get(1), b); if (randomBoolean()) { - maxes[b] = 3; - collect(sort, 3, b); + maxes.set(b, values.get(2)); + collect(sort, values.get(2), b); } if (randomBoolean()) { - collect(sort, -1, b); + collect(sort, values.get(0), b); } } for (int b = 0; b < buckets.length; b++) { - assertBlock(sort, b, expectedValue(maxes[b])); + assertBlock(sort, b, List.of(maxes.get(b))); } - assertBlock(sort, buckets.length); + assertBlock(sort, buckets.length, List.of()); } } public final void testTwoHitsDesc() { try (T sort = build(SortOrder.DESC, 2)) { - collect(sort, 1, 0); - collect(sort, 2, 0); - collect(sort, 3, 0); + var values = threeSortedValues(); - assertBlock(sort, 0, expectedValue(3), expectedValue(2)); + collect(sort, values.get(0), 0); + collect(sort, values.get(1), 0); + collect(sort, values.get(2), 0); + + assertBlock(sort, 0, List.of(values.get(2), values.get(1))); } } public final void testTwoHitsAsc() { try (T sort = build(SortOrder.ASC, 2)) { - collect(sort, 1, 0); - collect(sort, 2, 0); - collect(sort, 3, 0); + var values = threeSortedValues(); + + collect(sort, values.get(0), 0); + collect(sort, values.get(1), 0); + collect(sort, values.get(2), 0); - assertBlock(sort, 0, expectedValue(1), expectedValue(2)); + assertBlock(sort, 0, List.of(values.get(0), values.get(1))); } } public final void testTwoHitsTwoBucket() { try (T sort = build(SortOrder.DESC, 2)) { - collect(sort, 1, 0); - collect(sort, 1, 1); - collect(sort, 2, 0); - collect(sort, 2, 1); - collect(sort, 3, 0); - collect(sort, 3, 1); - collect(sort, 4, 1); - - assertBlock(sort, 0, expectedValue(3), expectedValue(2)); - assertBlock(sort, 1, expectedValue(4), expectedValue(3)); + var values = threeSortedValues(); + + collect(sort, values.get(0), 0); + collect(sort, values.get(0), 1); + collect(sort, values.get(1), 0); + collect(sort, values.get(1), 1); + collect(sort, values.get(2), 0); + + assertBlock(sort, 0, List.of(values.get(2), values.get(1))); + assertBlock(sort, 1, List.of(values.get(1), values.get(0))); } } public final void testManyBucketsManyHits() { // Set the values in random order - double[] values = new double[10000]; - for (int v = 0; v < values.length; v++) { - values[v] = randomValue(); + List values = new ArrayList(); + for (int v = 0; v < 10000; v++) { + values.add(randomValue()); } - Collections.shuffle(Arrays.asList(values), random()); + Collections.shuffle(values, random()); int buckets = between(2, 100); int bucketSize = between(2, 100); try (T sort = build(SortOrder.DESC, bucketSize)) { BitArray[] bucketUsed = new BitArray[buckets]; - Arrays.setAll(bucketUsed, i -> new BitArray(values.length, bigArrays())); - for (int doc = 0; doc < values.length; doc++) { + Arrays.setAll(bucketUsed, i -> new BitArray(values.size(), bigArrays())); + for (int doc = 0; doc < values.size(); doc++) { for (int bucket = 0; bucket < buckets; bucket++) { if (randomBoolean()) { bucketUsed[bucket].set(doc); - collect(sort, values[doc], bucket); + collect(sort, values.get(doc), bucket); } } } for (int bucket = 0; bucket < buckets; bucket++) { - List bucketValues = new ArrayList<>(values.length); - for (int doc = 0; doc < values.length; doc++) { + List bucketValues = new ArrayList<>(values.size()); + for (int doc = 0; doc < values.size(); doc++) { if (bucketUsed[bucket].get(doc)) { - bucketValues.add(values[doc]); + bucketValues.add(values.get(doc)); } } bucketUsed[bucket].close(); - assertBlock( - sort, - bucket, - bucketValues.stream().sorted((lhs, rhs) -> rhs.compareTo(lhs)).limit(bucketSize).map(this::expectedValue).toArray() - ); + assertBlock(sort, bucket, bucketValues.stream().sorted(Comparator.reverseOrder()).limit(bucketSize).toList()); } - assertBlock(sort, buckets); + assertBlock(sort, buckets, List.of()); } } public final void testMergeHeapToHeap() { try (T sort = build(SortOrder.ASC, 3)) { - collect(sort, 1, 0); - collect(sort, 2, 0); - collect(sort, 3, 0); + var values = threeSortedValues(); + + collect(sort, values.get(0), 0); + collect(sort, values.get(1), 0); + collect(sort, values.get(2), 0); try (T other = build(SortOrder.ASC, 3)) { - collect(other, 1, 0); - collect(other, 2, 0); - collect(other, 3, 0); + collect(other, values.get(0), 0); + collect(other, values.get(1), 0); + collect(other, values.get(2), 0); merge(sort, 0, other, 0); } - assertBlock(sort, 0, expectedValue(1), expectedValue(1), expectedValue(2)); + assertBlock(sort, 0, List.of(values.get(0), values.get(0), values.get(1))); } } public final void testMergeNoHeapToNoHeap() { try (T sort = build(SortOrder.ASC, 3)) { - collect(sort, 1, 0); - collect(sort, 2, 0); + var values = threeSortedValues(); + + collect(sort, values.get(0), 0); + collect(sort, values.get(1), 0); try (T other = build(SortOrder.ASC, 3)) { - collect(other, 1, 0); - collect(other, 2, 0); + collect(other, values.get(0), 0); + collect(other, values.get(1), 0); merge(sort, 0, other, 0); } - assertBlock(sort, 0, expectedValue(1), expectedValue(1), expectedValue(2)); + assertBlock(sort, 0, List.of(values.get(0), values.get(0), values.get(1))); } } public final void testMergeHeapToNoHeap() { try (T sort = build(SortOrder.ASC, 3)) { - collect(sort, 1, 0); - collect(sort, 2, 0); + var values = threeSortedValues(); + + collect(sort, values.get(0), 0); + collect(sort, values.get(1), 0); try (T other = build(SortOrder.ASC, 3)) { - collect(other, 1, 0); - collect(other, 2, 0); - collect(other, 3, 0); + collect(other, values.get(0), 0); + collect(other, values.get(1), 0); + collect(other, values.get(2), 0); merge(sort, 0, other, 0); } - assertBlock(sort, 0, expectedValue(1), expectedValue(1), expectedValue(2)); + assertBlock(sort, 0, List.of(values.get(0), values.get(0), values.get(1))); } } public final void testMergeNoHeapToHeap() { try (T sort = build(SortOrder.ASC, 3)) { - collect(sort, 1, 0); - collect(sort, 2, 0); - collect(sort, 3, 0); + var values = threeSortedValues(); + + collect(sort, values.get(0), 0); + collect(sort, values.get(1), 0); + collect(sort, values.get(2), 0); try (T other = build(SortOrder.ASC, 3)) { - collect(sort, 1, 0); - collect(sort, 2, 0); + collect(sort, values.get(0), 0); + collect(sort, values.get(1), 0); merge(sort, 0, other, 0); } - assertBlock(sort, 0, expectedValue(1), expectedValue(1), expectedValue(2)); + assertBlock(sort, 0, List.of(values.get(0), values.get(0), values.get(1))); } } public final void testMergeHeapToEmpty() { try (T sort = build(SortOrder.ASC, 3)) { + var values = threeSortedValues(); + try (T other = build(SortOrder.ASC, 3)) { - collect(other, 1, 0); - collect(other, 2, 0); - collect(other, 3, 0); + collect(other, values.get(0), 0); + collect(other, values.get(1), 0); + collect(other, values.get(2), 0); merge(sort, 0, other, 0); } - assertBlock(sort, 0, expectedValue(1), expectedValue(2), expectedValue(3)); + assertBlock(sort, 0, List.of(values.get(0), values.get(1), values.get(2))); } } public final void testMergeEmptyToHeap() { try (T sort = build(SortOrder.ASC, 3)) { - collect(sort, 1, 0); - collect(sort, 2, 0); - collect(sort, 3, 0); + var values = threeSortedValues(); + + collect(sort, values.get(0), 0); + collect(sort, values.get(1), 0); + collect(sort, values.get(2), 0); try (T other = build(SortOrder.ASC, 3)) { merge(sort, 0, other, 0); } - assertBlock(sort, 0, expectedValue(1), expectedValue(2), expectedValue(3)); + assertBlock(sort, 0, List.of(values.get(0), values.get(1), values.get(2))); } } @@ -340,20 +359,20 @@ public final void testMergeEmptyToEmpty() { merge(sort, 0, other, randomNonNegativeInt()); } - assertBlock(sort, 0); + assertBlock(sort, 0, List.of()); } } - private void assertBlock(T sort, int groupId, Object... values) { + protected void assertBlock(T sort, int groupId, List values) { var blockFactory = TestBlockFactory.getNonBreakingInstance(); try (var intVector = blockFactory.newConstantIntVector(groupId, 1)) { var block = toBlock(sort, blockFactory, intVector); assertThat(block.getPositionCount(), equalTo(1)); - assertThat(block.getTotalValueCount(), equalTo(values.length)); + assertThat(block.getTotalValueCount(), equalTo(values.size())); - if (values.length == 0) { + if (values.isEmpty()) { assertThat(block.elementType(), equalTo(ElementType.NULL)); assertThat(block.isNull(0), equalTo(true)); } else { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/DoubleBucketedSortTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/DoubleBucketedSortTests.java index 43b5caa092b9a..e5cc03f7c59cc 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/DoubleBucketedSortTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/DoubleBucketedSortTests.java @@ -14,26 +14,28 @@ import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.search.sort.SortOrder; +import java.util.List; + import static org.hamcrest.Matchers.equalTo; -public class DoubleBucketedSortTests extends BucketedSortTestCase { +public class DoubleBucketedSortTests extends BucketedSortTestCase { @Override protected DoubleBucketedSort build(SortOrder sortOrder, int bucketSize) { return new DoubleBucketedSort(bigArrays(), sortOrder, bucketSize); } @Override - protected Object expectedValue(double v) { - return v; + protected Double randomValue() { + return randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true); } @Override - protected double randomValue() { - return randomDoubleBetween(Double.MIN_VALUE, Double.MAX_VALUE, true); + protected List threeSortedValues() { + return List.of(-Double.MAX_VALUE, randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true), Double.MAX_VALUE); } @Override - protected void collect(DoubleBucketedSort sort, double value, int bucket) { + protected void collect(DoubleBucketedSort sort, Double value, int bucket) { sort.collect(value, bucket); } @@ -48,11 +50,11 @@ protected Block toBlock(DoubleBucketedSort sort, BlockFactory blockFactory, IntV } @Override - protected void assertBlockTypeAndValues(Block block, Object... values) { + protected void assertBlockTypeAndValues(Block block, List values) { assertThat(block.elementType(), equalTo(ElementType.DOUBLE)); var typedBlock = (DoubleBlock) block; - for (int i = 0; i < values.length; i++) { - assertThat(typedBlock.getDouble(i), equalTo(values[i])); + for (int i = 0; i < values.size(); i++) { + assertThat(typedBlock.getDouble(i), equalTo(values.get(i))); } } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/FloatBucketedSortTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/FloatBucketedSortTests.java index 8b3d288339037..5378e92f688ec 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/FloatBucketedSortTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/FloatBucketedSortTests.java @@ -14,27 +14,29 @@ import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.search.sort.SortOrder; +import java.util.List; + import static org.hamcrest.Matchers.equalTo; -public class FloatBucketedSortTests extends BucketedSortTestCase { +public class FloatBucketedSortTests extends BucketedSortTestCase { @Override protected FloatBucketedSort build(SortOrder sortOrder, int bucketSize) { return new FloatBucketedSort(bigArrays(), sortOrder, bucketSize); } @Override - protected Object expectedValue(double v) { - return v; + protected Float randomValue() { + return randomFloatBetween(-Float.MAX_VALUE, Float.MAX_VALUE, true); } @Override - protected double randomValue() { - return randomFloatBetween(Float.MIN_VALUE, Float.MAX_VALUE, true); + protected List threeSortedValues() { + return List.of(-Float.MAX_VALUE, randomFloatBetween(-Float.MAX_VALUE, Float.MAX_VALUE, true), Float.MAX_VALUE); } @Override - protected void collect(FloatBucketedSort sort, double value, int bucket) { - sort.collect((float) value, bucket); + protected void collect(FloatBucketedSort sort, Float value, int bucket) { + sort.collect(value, bucket); } @Override @@ -48,11 +50,11 @@ protected Block toBlock(FloatBucketedSort sort, BlockFactory blockFactory, IntVe } @Override - protected void assertBlockTypeAndValues(Block block, Object... values) { + protected void assertBlockTypeAndValues(Block block, List values) { assertThat(block.elementType(), equalTo(ElementType.FLOAT)); var typedBlock = (FloatBlock) block; - for (int i = 0; i < values.length; i++) { - assertThat((double) typedBlock.getFloat(i), equalTo(values[i])); + for (int i = 0; i < values.size(); i++) { + assertThat(typedBlock.getFloat(i), equalTo(values.get(i))); } } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/IntBucketedSortTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/IntBucketedSortTests.java index 70d0a79ea7473..62bf3ea69ceac 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/IntBucketedSortTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/IntBucketedSortTests.java @@ -14,27 +14,29 @@ import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.search.sort.SortOrder; +import java.util.List; + import static org.hamcrest.Matchers.equalTo; -public class IntBucketedSortTests extends BucketedSortTestCase { +public class IntBucketedSortTests extends BucketedSortTestCase { @Override protected IntBucketedSort build(SortOrder sortOrder, int bucketSize) { return new IntBucketedSort(bigArrays(), sortOrder, bucketSize); } @Override - protected Object expectedValue(double v) { - return (int) v; + protected Integer randomValue() { + return randomInt(); } @Override - protected double randomValue() { - return randomInt(); + protected List threeSortedValues() { + return List.of(Integer.MIN_VALUE, randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE), Integer.MAX_VALUE); } @Override - protected void collect(IntBucketedSort sort, double value, int bucket) { - sort.collect((int) value, bucket); + protected void collect(IntBucketedSort sort, Integer value, int bucket) { + sort.collect(value, bucket); } @Override @@ -48,11 +50,11 @@ protected Block toBlock(IntBucketedSort sort, BlockFactory blockFactory, IntVect } @Override - protected void assertBlockTypeAndValues(Block block, Object... values) { + protected void assertBlockTypeAndValues(Block block, List values) { assertThat(block.elementType(), equalTo(ElementType.INT)); var typedBlock = (IntBlock) block; - for (int i = 0; i < values.length; i++) { - assertThat(typedBlock.getInt(i), equalTo(values[i])); + for (int i = 0; i < values.size(); i++) { + assertThat(typedBlock.getInt(i), equalTo(values.get(i))); } } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/IpBucketedSortTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/IpBucketedSortTests.java new file mode 100644 index 0000000000000..dad7af26a484d --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/IpBucketedSortTests.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data.sort; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.search.sort.SortOrder; + +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; + +public class IpBucketedSortTests extends BucketedSortTestCase { + @Override + protected IpBucketedSort build(SortOrder sortOrder, int bucketSize) { + return new IpBucketedSort(bigArrays(), sortOrder, bucketSize); + } + + @Override + protected BytesRef randomValue() { + return new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean()))); + } + + @Override + protected List threeSortedValues() { + return List.of( + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::"))), + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1"))), + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("9999::"))) + ); + } + + @Override + protected void collect(IpBucketedSort sort, BytesRef value, int bucket) { + sort.collect(value, bucket); + } + + @Override + protected void merge(IpBucketedSort sort, int groupId, IpBucketedSort other, int otherGroupId) { + sort.merge(groupId, other, otherGroupId); + } + + @Override + protected Block toBlock(IpBucketedSort sort, BlockFactory blockFactory, IntVector selected) { + return sort.toBlock(blockFactory, selected); + } + + @Override + protected void assertBlockTypeAndValues(Block block, List values) { + assertThat(block.elementType(), equalTo(ElementType.BYTES_REF)); + var typedBlock = (BytesRefBlock) block; + var scratch = new BytesRef(); + for (int i = 0; i < values.size(); i++) { + assertThat("expected value on block position " + i, typedBlock.getBytesRef(i, scratch), equalTo(values.get(i))); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/LongBucketedSortTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/LongBucketedSortTests.java index bceed3b1d95b5..2d60a160c4458 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/LongBucketedSortTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/LongBucketedSortTests.java @@ -14,28 +14,29 @@ import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.search.sort.SortOrder; +import java.util.List; + import static org.hamcrest.Matchers.equalTo; -public class LongBucketedSortTests extends BucketedSortTestCase { +public class LongBucketedSortTests extends BucketedSortTestCase { @Override protected LongBucketedSort build(SortOrder sortOrder, int bucketSize) { return new LongBucketedSort(bigArrays(), sortOrder, bucketSize); } @Override - protected Object expectedValue(double v) { - return (long) v; + protected Long randomValue() { + return randomLong(); } @Override - protected double randomValue() { - // 2L^50 fits in the mantisa of a double which the test sort of needs. - return randomLongBetween(-2L ^ 50, 2L ^ 50); + protected List threeSortedValues() { + return List.of(Long.MIN_VALUE, randomLongBetween(Long.MIN_VALUE, Long.MAX_VALUE), Long.MAX_VALUE); } @Override - protected void collect(LongBucketedSort sort, double value, int bucket) { - sort.collect((long) value, bucket); + protected void collect(LongBucketedSort sort, Long value, int bucket) { + sort.collect(value, bucket); } @Override @@ -49,11 +50,11 @@ protected Block toBlock(LongBucketedSort sort, BlockFactory blockFactory, IntVec } @Override - protected void assertBlockTypeAndValues(Block block, Object... values) { + protected void assertBlockTypeAndValues(Block block, List values) { assertThat(block.elementType(), equalTo(ElementType.LONG)); var typedBlock = (LongBlock) block; - for (int i = 0; i < values.length; i++) { - assertThat(typedBlock.getLong(i), equalTo(values[i])); + for (int i = 0; i < values.size(); i++) { + assertThat(typedBlock.getLong(i), equalTo(values.get(i))); } } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java index 09f63e9fa45bb..ccc3dea78adc8 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java @@ -62,6 +62,7 @@ import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Releasables; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; @@ -543,6 +544,11 @@ public String indexName() { return "test_index"; } + @Override + public IndexSettings indexSettings() { + throw new UnsupportedOperationException(); + } + @Override public MappedFieldType.FieldExtractPreference fieldExtractPreference() { return MappedFieldType.FieldExtractPreference.NONE; @@ -1543,7 +1549,11 @@ protected void start(Driver driver, ActionListener driverListener) { PlainActionFuture future = new PlainActionFuture<>(); try { driverRunner.runToCompletion(drivers, future); - future.actionGet(TimeValue.timeValueSeconds(30)); + /* + * We use a 3-minute timer because many of the cases can + * take 40 seconds in CI. Locally it's taking 9 seconds. + */ + future.actionGet(TimeValue.timeValueMinutes(3)); } finally { terminate(threadPool); } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java index f4c545142508c..848415c4490fa 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java @@ -51,6 +51,7 @@ import org.elasticsearch.compute.operator.PageConsumerOperator; import org.elasticsearch.compute.operator.SourceOperator; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; @@ -497,6 +498,11 @@ public String indexName() { return "test_index"; } + @Override + public IndexSettings indexSettings() { + throw new UnsupportedOperationException(); + } + @Override public MappedFieldType.FieldExtractPreference fieldExtractPreference() { return MappedFieldType.FieldExtractPreference.NONE; diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/SequenceBytesRefBlockSourceOperator.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/SequenceBytesRefBlockSourceOperator.java new file mode 100644 index 0000000000000..75e71ff697efb --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/SequenceBytesRefBlockSourceOperator.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.operator; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.Page; + +import java.util.List; +import java.util.stream.Stream; + +/** + * A source operator whose output is the given double values. This operator produces pages + * containing a single Block. The Block contains the double values from the given list, in order. + */ +public class SequenceBytesRefBlockSourceOperator extends AbstractBlockSourceOperator { + + static final int DEFAULT_MAX_PAGE_POSITIONS = 8 * 1024; + + private final BytesRef[] values; + + public SequenceBytesRefBlockSourceOperator(BlockFactory blockFactory, Stream values) { + this(blockFactory, values, DEFAULT_MAX_PAGE_POSITIONS); + } + + public SequenceBytesRefBlockSourceOperator(BlockFactory blockFactory, Stream values, int maxPagePositions) { + super(blockFactory, maxPagePositions); + this.values = values.toArray(BytesRef[]::new); + } + + public SequenceBytesRefBlockSourceOperator(BlockFactory blockFactory, List values) { + this(blockFactory, values, DEFAULT_MAX_PAGE_POSITIONS); + } + + public SequenceBytesRefBlockSourceOperator(BlockFactory blockFactory, List values, int maxPagePositions) { + super(blockFactory, maxPagePositions); + this.values = values.toArray(BytesRef[]::new); + } + + @Override + protected Page createPage(int positionOffset, int length) { + try (var builder = blockFactory.newBytesRefVectorBuilder(length)) { + for (int i = 0; i < length; i++) { + builder.appendBytesRef(values[positionOffset + i]); + } + currentPosition += length; + return new Page(builder.build().asBlock()); + } + } + + protected int remaining() { + return values.length - currentPosition; + } +} diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java index faa2eb9bd82b0..ca477d4256527 100644 --- a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java +++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java @@ -138,19 +138,24 @@ public void testUnauthorizedIndices() throws IOException { } public void testInsufficientPrivilege() { - Exception error = expectThrows( - Exception.class, - () -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2 | STATS sum=sum(value)") - ); + Exception error = expectThrows(Exception.class, () -> runESQLCommand("metadata1_read2", "FROM index-user1 | STATS sum=sum(value)")); logger.info("error", error); + assertThat(error.getMessage(), containsString("Unknown index [index-user1]")); + } + + public void testLimitedPrivilege() throws Exception { + Response resp = runESQLCommand("metadata1_read2", """ + FROM index-user1,index-user2 METADATA _index + | STATS sum=sum(value), index=VALUES(_index) + """); + assertOK(resp); + Map respMap = entityAsMap(resp); assertThat( - error.getMessage(), - containsString( - "unauthorized for user [test-admin] run as [metadata1_read2] " - + "with effective roles [metadata1_read2] on indices [index-user1], " - + "this action is granted by the index privileges [read,all]" - ) + respMap.get("columns"), + equalTo(List.of(Map.of("name", "sum", "type", "double"), Map.of("name", "index", "type", "keyword"))) ); + assertThat(respMap.get("values"), equalTo(List.of(List.of(72.0, "index-user2")))); + } public void testDocumentLevelSecurity() throws Exception { diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java index 8ab375dfd24c7..5676a8bce3ede 100644 --- a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java +++ b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java @@ -56,14 +56,22 @@ public static void cleanUp() { oldClusterTestFeatureService = null; } - public MixedClusterEsqlSpecIT(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase, Mode mode) { - super(fileName, groupName, testName, lineNumber, testCase, mode); + public MixedClusterEsqlSpecIT( + String fileName, + String groupName, + String testName, + Integer lineNumber, + CsvTestCase testCase, + String instructions, + Mode mode + ) { + super(fileName, groupName, testName, lineNumber, testCase, instructions, mode); } @Override protected void shouldSkipTest(String testName) throws IOException { super.shouldSkipTest(testName); - assumeTrue("Test " + testName + " is skipped on " + bwcVersion, isEnabled(testName, bwcVersion)); + assumeTrue("Test " + testName + " is skipped on " + bwcVersion, isEnabled(testName, instructions, bwcVersion)); if (mode == ASYNC) { assumeTrue("Async is not supported on " + bwcVersion, supportsAsync()); } diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle b/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle index 7008bd8b7aa01..d650503909d30 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle +++ b/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle @@ -19,9 +19,8 @@ dependencies { } def supportedVersion = bwcVersion -> { - // This test is less restricted than the actual CCS compatibility matrix that we are supporting. - // CCQ is available on 8.13 or later - return bwcVersion.onOrAfter(Version.fromString("8.13.0")); + // ESQL requires its own resolve_fields API + return bwcVersion.onOrAfter(Version.fromString("8.16.0")); } BuildParams.bwcVersions.withWireCompatible(supportedVersion) { bwcVersion, baseName -> diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java index e6c7a3c73f1fb..ece80c15e87e5 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java @@ -88,16 +88,29 @@ public static List readScriptSpec() throws Exception { return testcases; } - public MultiClusterSpecIT(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase, Mode mode) { - super(fileName, groupName, testName, lineNumber, convertToRemoteIndices(testCase), mode); + public MultiClusterSpecIT( + String fileName, + String groupName, + String testName, + Integer lineNumber, + CsvTestCase testCase, + String instructions, + Mode mode + ) { + super(fileName, groupName, testName, lineNumber, convertToRemoteIndices(testCase), instructions, mode); } @Override protected void shouldSkipTest(String testName) throws IOException { super.shouldSkipTest(testName); checkCapabilities(remoteClusterClient(), remoteFeaturesService(), testName, testCase); + assumeTrue("CCS requires its own resolve_fields API", remoteFeaturesService().clusterHasFeature("esql.resolve_fields_api")); assumeFalse("can't test with _index metadata", hasIndexMetadata(testCase.query)); - assumeTrue("Test " + testName + " is skipped on " + Clusters.oldVersion(), isEnabled(testName, Clusters.oldVersion())); + assumeTrue( + "Test " + testName + " is skipped on " + Clusters.oldVersion(), + isEnabled(testName, instructions, Clusters.oldVersion()) + ); + assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains("inlinestats")); } private TestFeatureService remoteFeaturesService() throws IOException { diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java index 9dae850d6f349..d5c8926d93b84 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -61,6 +62,7 @@ record Doc(int id, String color, long data) { @Before public void setUpIndices() throws Exception { + assumeTrue("CCS requires its own resolve_fields API", remoteFeaturesService().clusterHasFeature("esql.resolve_fields_api")); final String mapping = """ "properties": { "data": { "type": "long" }, @@ -201,4 +203,18 @@ private RestClient remoteClusterClient() throws IOException { var clusterHosts = parseClusterHosts(remoteCluster.getHttpAddresses()); return buildClient(restClientSettings(), clusterHosts.toArray(new HttpHost[0])); } + + private TestFeatureService remoteFeaturesService() throws IOException { + if (remoteFeaturesService == null) { + try (RestClient remoteClient = remoteClusterClient()) { + var remoteNodeVersions = readVersionsFromNodesInfo(remoteClient); + var semanticNodeVersions = remoteNodeVersions.stream() + .map(ESRestTestCase::parseLegacyVersion) + .flatMap(Optional::stream) + .collect(Collectors.toSet()); + remoteFeaturesService = createTestFeatureService(getClusterStateFeatures(remoteClient), semanticNodeVersions); + } + } + return remoteFeaturesService; + } } diff --git a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java index 15407804a56f2..93385ec9efd89 100644 --- a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java @@ -21,8 +21,16 @@ protected String getTestRestCluster() { return cluster.getHttpAddresses(); } - public EsqlSpecIT(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase, Mode mode) { - super(fileName, groupName, testName, lineNumber, testCase, mode); + public EsqlSpecIT( + String fileName, + String groupName, + String testName, + Integer lineNumber, + CsvTestCase testCase, + String instructions, + Mode mode + ) { + super(fileName, groupName, testName, lineNumber, testCase, instructions, mode); } @Override diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java index 6494695a484d4..15d55e0258110 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java @@ -25,8 +25,16 @@ protected String getTestRestCluster() { return cluster.getHttpAddresses(); } - public EsqlSpecIT(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase, Mode mode) { - super(fileName, groupName, testName, lineNumber, testCase, mode); + public EsqlSpecIT( + String fileName, + String groupName, + String testName, + Integer lineNumber, + CsvTestCase testCase, + String instructions, + Mode mode + ) { + super(fileName, groupName, testName, lineNumber, testCase, instructions, mode); } @Override diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index bf54dcbfa96f6..af872715c2fea 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -10,15 +10,19 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.apache.http.util.EntityUtils; +import org.apache.lucene.search.DocIdSetIterator; import org.elasticsearch.Build; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ListMatcher; +import org.elasticsearch.test.MapMatcher; import org.elasticsearch.test.TestClustersThreadFilter; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.LogType; +import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase; import org.hamcrest.Matchers; import org.junit.Assert; @@ -27,14 +31,23 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Locale; import java.util.Map; +import static org.elasticsearch.test.ListMatcher.matchesList; +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.hamcrest.Matchers.any; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.startsWith; import static org.hamcrest.core.Is.is; @ThreadLeakFilters(filters = TestClustersThreadFilter.class) @@ -49,7 +62,7 @@ protected String getTestRestCluster() { return cluster.getHttpAddresses(); } - @ParametersFactory + @ParametersFactory(argumentFormatting = "%1s") public static List modes() { return Arrays.stream(Mode.values()).map(m -> new Object[] { m }).toList(); } @@ -59,19 +72,7 @@ public RestEsqlIT(Mode mode) { } public void testBasicEsql() throws IOException { - StringBuilder b = new StringBuilder(); - for (int i = 0; i < 1000; i++) { - b.append(String.format(Locale.ROOT, """ - {"create":{"_index":"%s"}} - {"@timestamp":"2020-12-12","test":"value%s","value":%d} - """, testIndexName(), i, i)); - } - Request bulk = new Request("POST", "/_bulk"); - bulk.addParameter("refresh", "true"); - bulk.addParameter("filter_path", "errors"); - bulk.setJsonEntity(b.toString()); - Response response = client().performRequest(bulk); - Assert.assertEquals("{\"errors\":false}", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)); + indexTestData(); RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | stats avg(value)"); if (Build.current().isSnapshot()) { @@ -273,6 +274,247 @@ public void testTableDuplicateNames() throws IOException { assertThat(re.getMessage(), containsString("[6:10] Duplicate field 'a'")); } + /** + * INLINESTATS can group on {@code NOW()}. It's a little silly, but + * doing something like {@code DATE_TRUNC(1 YEAR, NOW() - 1970-01-01T00:00:00Z)} is + * much more sensible. But just grouping on {@code NOW()} is enough to test this. + *

    + * This works because {@code NOW()} locks it's value at the start of the entire + * query. It's part of the "configuration" of the query. + *

    + */ + public void testInlineStatsNow() throws IOException { + indexTestData(); + + RequestObjectBuilder builder = requestObjectBuilder().query( + fromIndex() + " | EVAL now=NOW() | INLINESTATS AVG(value) BY now | SORT value ASC" + ); + Map result = runEsql(builder); + ListMatcher values = matchesList(); + for (int i = 0; i < 1000; i++) { + values = values.item( + matchesList().item("2020-12-12T00:00:00.000Z") + .item("value" + i) + .item("value" + i) + .item(i) + .item(any(String.class)) + .item(499.5) + ); + } + assertMap( + result, + matchesMap().entry( + "columns", + matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date")) + .item(matchesMap().entry("name", "test").entry("type", "text")) + .item(matchesMap().entry("name", "test.keyword").entry("type", "keyword")) + .item(matchesMap().entry("name", "value").entry("type", "long")) + .item(matchesMap().entry("name", "now").entry("type", "date")) + .item(matchesMap().entry("name", "AVG(value)").entry("type", "double")) + ).entry("values", values) + ); + } + + public void testProfile() throws IOException { + indexTestData(); + + RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | STATS AVG(value)"); + builder.profile(true); + if (Build.current().isSnapshot()) { + // Lock to shard level partitioning, so we get consistent profile output + builder.pragmas(Settings.builder().put("data_partitioning", "shard").build()); + } + Map result = runEsql(builder); + assertMap( + result, + matchesMap().entry("columns", matchesList().item(matchesMap().entry("name", "AVG(value)").entry("type", "double"))) + .entry("values", List.of(List.of(499.5d))) + .entry("profile", matchesMap().entry("drivers", instanceOf(List.class))) + ); + + MapMatcher commonProfile = matchesMap().entry("iterations", greaterThan(0)) + .entry("cpu_nanos", greaterThan(0)) + .entry("took_nanos", greaterThan(0)) + .entry("operators", instanceOf(List.class)); + List> signatures = new ArrayList<>(); + @SuppressWarnings("unchecked") + List> profiles = (List>) ((Map) result.get("profile")).get("drivers"); + for (Map p : profiles) { + assertThat(p, commonProfile); + List sig = new ArrayList<>(); + @SuppressWarnings("unchecked") + List> operators = (List>) p.get("operators"); + for (Map o : operators) { + sig.add(checkOperatorProfile(o)); + } + signatures.add(sig); + } + assertThat( + signatures, + containsInAnyOrder( + matchesList().item("LuceneSourceOperator") + .item("ValuesSourceReaderOperator") + .item("AggregationOperator") + .item("ExchangeSinkOperator"), + matchesList().item("ExchangeSourceOperator").item("ExchangeSinkOperator"), + matchesList().item("ExchangeSourceOperator") + .item("AggregationOperator") + .item("ProjectOperator") + .item("LimitOperator") + .item("EvalOperator") + .item("ProjectOperator") + .item("OutputOperator") + ) + ); + } + + public void testInlineStatsProfile() throws IOException { + indexTestData(); + + RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | INLINESTATS AVG(value) | SORT value ASC"); + builder.profile(true); + if (Build.current().isSnapshot()) { + // Lock to shard level partitioning, so we get consistent profile output + builder.pragmas(Settings.builder().put("data_partitioning", "shard").build()); + } + Map result = runEsql(builder); + ListMatcher values = matchesList(); + for (int i = 0; i < 1000; i++) { + values = values.item(matchesList().item("2020-12-12T00:00:00.000Z").item("value" + i).item("value" + i).item(i).item(499.5)); + } + assertMap( + result, + matchesMap().entry( + "columns", + matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date")) + .item(matchesMap().entry("name", "test").entry("type", "text")) + .item(matchesMap().entry("name", "test.keyword").entry("type", "keyword")) + .item(matchesMap().entry("name", "value").entry("type", "long")) + .item(matchesMap().entry("name", "AVG(value)").entry("type", "double")) + ).entry("values", values).entry("profile", matchesMap().entry("drivers", instanceOf(List.class))) + ); + + MapMatcher commonProfile = matchesMap().entry("iterations", greaterThan(0)) + .entry("cpu_nanos", greaterThan(0)) + .entry("took_nanos", greaterThan(0)) + .entry("operators", instanceOf(List.class)); + List> signatures = new ArrayList<>(); + @SuppressWarnings("unchecked") + List> profiles = (List>) ((Map) result.get("profile")).get("drivers"); + for (Map p : profiles) { + assertThat(p, commonProfile); + List sig = new ArrayList<>(); + @SuppressWarnings("unchecked") + List> operators = (List>) p.get("operators"); + for (Map o : operators) { + sig.add(checkOperatorProfile(o)); + } + signatures.add(sig); + } + assertThat( + signatures, + containsInAnyOrder( + // First pass read and start agg + matchesList().item("LuceneSourceOperator") + .item("ValuesSourceReaderOperator") + .item("AggregationOperator") + .item("ExchangeSinkOperator"), + // First pass node level reduce + matchesList().item("ExchangeSourceOperator").item("ExchangeSinkOperator"), + // First pass finish agg + matchesList().item("ExchangeSourceOperator") + .item("AggregationOperator") + .item("ProjectOperator") + .item("EvalOperator") + .item("ProjectOperator") + .item("OutputOperator"), + // Second pass read and join via eval + matchesList().item("LuceneSourceOperator") + .item("EvalOperator") + .item("ValuesSourceReaderOperator") + .item("TopNOperator") + .item("ValuesSourceReaderOperator") + .item("ProjectOperator") + .item("ExchangeSinkOperator"), + // Second pass node level reduce + matchesList().item("ExchangeSourceOperator").item("ExchangeSinkOperator"), + // Second pass finish + matchesList().item("ExchangeSourceOperator").item("TopNOperator").item("OutputOperator") + ) + ); + } + + private String checkOperatorProfile(Map o) { + String name = (String) o.get("operator"); + name = name.replaceAll("\\[.+", ""); + MapMatcher status = switch (name) { + case "LuceneSourceOperator" -> matchesMap().entry("processed_slices", greaterThan(0)) + .entry("processed_shards", List.of(testIndexName() + ":0")) + .entry("total_slices", greaterThan(0)) + .entry("slice_index", 0) + .entry("slice_max", 0) + .entry("slice_min", 0) + .entry("current", DocIdSetIterator.NO_MORE_DOCS) + .entry("pages_emitted", greaterThan(0)) + .entry("processing_nanos", greaterThan(0)) + .entry("processed_queries", List.of("*:*")); + case "ValuesSourceReaderOperator" -> basicProfile().entry("readers_built", matchesMap().extraOk()); + case "AggregationOperator" -> matchesMap().entry("pages_processed", greaterThan(0)).entry("aggregation_nanos", greaterThan(0)); + case "ExchangeSinkOperator" -> matchesMap().entry("pages_accepted", greaterThan(0)); + case "ExchangeSourceOperator" -> matchesMap().entry("pages_emitted", greaterThan(0)).entry("pages_waiting", 0); + case "ProjectOperator", "EvalOperator" -> basicProfile(); + case "LimitOperator" -> matchesMap().entry("pages_processed", greaterThan(0)) + .entry("limit", 1000) + .entry("limit_remaining", 999); + case "OutputOperator" -> null; + case "TopNOperator" -> matchesMap().entry("occupied_rows", 0) + .entry("ram_used", instanceOf(String.class)) + .entry("ram_bytes_used", greaterThan(0)); + default -> throw new AssertionError("unexpected status: " + o); + }; + MapMatcher expectedOp = matchesMap().entry("operator", startsWith(name)); + if (status != null) { + expectedOp = expectedOp.entry("status", status); + } + assertMap(o, expectedOp); + return name; + } + + private MapMatcher basicProfile() { + return matchesMap().entry("pages_processed", greaterThan(0)).entry("process_nanos", greaterThan(0)); + } + + private void indexTestData() throws IOException { + Request createIndex = new Request("PUT", testIndexName()); + createIndex.setJsonEntity(""" + { + "settings": { + "index": { + "number_of_shards": 1 + } + } + }"""); + Response response = client().performRequest(createIndex); + assertThat( + entityToMap(response.getEntity(), XContentType.JSON), + matchesMap().entry("shards_acknowledged", true).entry("index", testIndexName()).entry("acknowledged", true) + ); + + StringBuilder b = new StringBuilder(); + for (int i = 0; i < 1000; i++) { + b.append(String.format(Locale.ROOT, """ + {"create":{"_index":"%s"}} + {"@timestamp":"2020-12-12","test":"value%s","value":%d} + """, testIndexName(), i, i)); + } + Request bulk = new Request("POST", "/_bulk"); + bulk.addParameter("refresh", "true"); + bulk.addParameter("filter_path", "errors"); + bulk.setJsonEntity(b.toString()); + response = client().performRequest(bulk); + Assert.assertEquals("{\"errors\":false}", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)); + } + private void assertException(String query, String... errorMessages) throws IOException { ResponseException re = expectThrows(ResponseException.class, () -> runEsqlSync(requestObjectBuilder().query(query))); assertThat(re.getResponse().getStatusLine().getStatusCode(), equalTo(400)); diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java index e650f0815f964..a9074073f1f19 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java @@ -82,6 +82,7 @@ public abstract class EsqlSpecTestCase extends ESRestTestCase { private final String testName; private final Integer lineNumber; protected final CsvTestCase testCase; + protected final String instructions; protected final Mode mode; public enum Mode { @@ -89,7 +90,7 @@ public enum Mode { ASYNC } - @ParametersFactory(argumentFormatting = "%2$s.%3$s %6$s") + @ParametersFactory(argumentFormatting = "%2$s.%3$s %7$s") public static List readScriptSpec() throws Exception { List urls = classpathResources("/*.csv-spec"); assertTrue("Not enough specs found " + urls, urls.size() > 0); @@ -108,12 +109,21 @@ public static List readScriptSpec() throws Exception { return testcases; } - protected EsqlSpecTestCase(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase, Mode mode) { + protected EsqlSpecTestCase( + String fileName, + String groupName, + String testName, + Integer lineNumber, + CsvTestCase testCase, + String instructions, + Mode mode + ) { this.fileName = fileName; this.groupName = groupName; this.testName = testName; this.lineNumber = lineNumber; this.testCase = testCase; + this.instructions = instructions; this.mode = mode; } @@ -155,7 +165,7 @@ public final void test() throws Throwable { protected void shouldSkipTest(String testName) throws IOException { checkCapabilities(adminClient(), testFeatureService, testName, testCase); - assumeTrue("Test " + testName + " is not enabled", isEnabled(testName, Version.CURRENT)); + assumeTrue("Test " + testName + " is not enabled", isEnabled(testName, instructions, Version.CURRENT)); } protected static void checkCapabilities(RestClient client, TestFeatureService testFeatureService, String testName, CsvTestCase testCase) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java index a672e4a50612c..82b7459066586 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java @@ -119,6 +119,8 @@ public static class RequestObjectBuilder { private Boolean keepOnCompletion = null; + private Boolean profile = null; + public RequestObjectBuilder() throws IOException { this(randomFrom(XContentType.values())); } @@ -180,6 +182,11 @@ public RequestObjectBuilder pragmas(Settings pragmas) throws IOException { return this; } + public RequestObjectBuilder profile(boolean profile) { + this.profile = profile; + return this; + } + public RequestObjectBuilder build() throws IOException { if (isBuilt == false) { if (tables != null) { @@ -195,6 +202,9 @@ public RequestObjectBuilder build() throws IOException { } builder.endObject(); } + if (profile != null) { + builder.field("profile", profile); + } builder.endObject(); isBuilt = true; } @@ -567,6 +577,32 @@ public void testErrorMessageForArrayValuesInParams() throws IOException { assertThat(EntityUtils.toString(re.getResponse().getEntity()), containsString("n1=[5, 6, 7] is not supported as a parameter")); } + public void testComplexFieldNames() throws IOException { + bulkLoadTestData(1); + // catch verification exception, field names not found + int fieldNumber = 5000; + String q1 = fromIndex() + queryWithComplexFieldNames(fieldNumber); + ResponseException e = expectThrows(ResponseException.class, () -> runEsql(requestObjectBuilder().query(q1))); + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + assertThat(e.getMessage(), containsString("verification_exception")); + + // catch automaton's TooComplexToDeterminizeException + fieldNumber = 6000; + final String q2 = fromIndex() + queryWithComplexFieldNames(fieldNumber); + e = expectThrows(ResponseException.class, () -> runEsql(requestObjectBuilder().query(q2))); + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + assertThat(e.getMessage(), containsString("The field names are too complex to process")); + } + + private static String queryWithComplexFieldNames(int field) { + StringBuilder query = new StringBuilder(); + query.append(" | keep ").append(randomAlphaOfLength(10)).append(1); + for (int i = 2; i <= field; i++) { + query.append(", ").append(randomAlphaOfLength(10)).append(i); + } + return query.toString(); + } + private static String expectedTextBody(String format, int count, @Nullable Character csvDelimiter) { StringBuilder sb = new StringBuilder(); switch (format) { @@ -730,7 +766,7 @@ static Map removeAsyncProperties(Map map) { return Collections.unmodifiableMap(copy); } - static Map entityToMap(HttpEntity entity, XContentType expectedContentType) throws IOException { + protected static Map entityToMap(HttpEntity entity, XContentType expectedContentType) throws IOException { try (InputStream content = entity.getContent()) { XContentType xContentType = XContentType.fromMediaType(entity.getContentType().getValue()); assertEquals(expectedContentType, xContentType); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java index 3b3e12978ae04..bfa9c11cbcd53 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java @@ -69,21 +69,21 @@ public final class CsvTestUtils { private CsvTestUtils() {} - public static boolean isEnabled(String testName, Version version) { + public static boolean isEnabled(String testName, String instructions, Version version) { if (testName.endsWith("-Ignore")) { return false; } - Tuple skipRange = skipVersionRange(testName); + Tuple skipRange = skipVersionRange(testName, instructions); if (skipRange != null && version.onOrAfter(skipRange.v1()) && version.onOrBefore(skipRange.v2())) { return false; } return true; } - private static final Pattern INSTRUCTION_PATTERN = Pattern.compile("#\\[(.*?)]"); + private static final Pattern INSTRUCTION_PATTERN = Pattern.compile("\\[(.*?)]"); - public static Map extractInstructions(String testName) { - Matcher matcher = INSTRUCTION_PATTERN.matcher(testName); + public static Map parseInstructions(String instructions) { + Matcher matcher = INSTRUCTION_PATTERN.matcher(instructions); Map pairs = new HashMap<>(); if (matcher.find()) { String[] groups = matcher.group(1).split(","); @@ -98,8 +98,8 @@ public static Map extractInstructions(String testName) { return pairs; } - public static Tuple skipVersionRange(String testName) { - Map pairs = extractInstructions(testName); + public static Tuple skipVersionRange(String testName, String instructions) { + Map pairs = parseInstructions(instructions); String versionRange = pairs.get("skip"); if (versionRange != null) { String[] skipVersions = versionRange.split("-", Integer.MAX_VALUE); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 530b2bc01b3d6..f9b768d67d574 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -96,8 +96,8 @@ public class CsvTestsDataLoader { "cartesian_multipolygons.csv" ); private static final TestsDataset DISTANCES = new TestsDataset("distances", "mapping-distances.json", "distances.csv"); - private static final TestsDataset K8S = new TestsDataset("k8s", "k8s-mappings.json", "k8s.csv", "k8s-settings.json", true); + private static final TestsDataset ADDRESSES = new TestsDataset("addresses", "mapping-addresses.json", "addresses.csv", null, true); public static final Map CSV_DATASET_MAP = Map.ofEntries( Map.entry(EMPLOYEES.indexName, EMPLOYEES), @@ -121,7 +121,8 @@ public class CsvTestsDataLoader { Map.entry(AIRPORT_CITY_BOUNDARIES.indexName, AIRPORT_CITY_BOUNDARIES), Map.entry(CARTESIAN_MULTIPOLYGONS.indexName, CARTESIAN_MULTIPOLYGONS), Map.entry(K8S.indexName, K8S), - Map.entry(DISTANCES.indexName, DISTANCES) + Map.entry(DISTANCES.indexName, DISTANCES), + Map.entry(ADDRESSES.indexName, ADDRESSES) ); private static final EnrichConfig LANGUAGES_ENRICH = new EnrichConfig("languages_policy", "enrich-policy-languages.json"); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index 2bf3baf845010..4e69dffa13bc8 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -31,6 +31,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.Range; import org.elasticsearch.xpack.esql.core.index.EsIndex; import org.elasticsearch.xpack.esql.core.session.Configuration; @@ -169,6 +170,10 @@ public static Literal of(Source source, Object value) { return new Literal(source, value, DataType.fromJava(value)); } + public static ReferenceAttribute referenceAttribute(String name, DataType type) { + return new ReferenceAttribute(EMPTY, name, type); + } + public static Range rangeOf(Expression value, Expression lower, boolean includeLower, Expression upper, boolean includeUpper) { return new Range(EMPTY, value, lower, includeLower, upper, includeUpper, randomZone()); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/addresses.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/addresses.csv new file mode 100644 index 0000000000000..0eea102400d60 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/addresses.csv @@ -0,0 +1,4 @@ +street:keyword,number:keyword,zip_code:keyword,city.name:keyword,city.country.name:keyword,city.country.continent.name:keyword,city.country.continent.planet.name:keyword,city.country.continent.planet.galaxy:keyword +Keizersgracht,281,1016 ED,Amsterdam,Netherlands,Europe,Earth,Milky Way +Kearny St,88,CA 94108,San Francisco,United States of America,North America,Earth,Milky Way +Marunouchi,2-7-2,100-7014,Tokyo,Japan,Asia,Earth,Milky Way diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index 776cc2f95f465..f9d83641ab4bd 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -1171,3 +1171,27 @@ from employees a:datetime null ; + +ImplicitCastingEqual +required_capability: rangequery_for_datetime +from employees +| where birth_date == "1957-05-23T00:00:00Z" +| keep emp_no, birth_date +; + +emp_no:integer | birth_date:datetime +10007 | 1957-05-23T00:00:00Z +; + +ImplicitCastingIn +required_capability: rangequery_for_datetime +from employees +| where birth_date IN ("1957-05-23T00:00:00Z", "1958-02-19T00:00:00Z") +| keep emp_no, birth_date +| sort emp_no +; + +emp_no:integer | birth_date:datetime +10007 | 1957-05-23T00:00:00Z +10008 | 1958-02-19T00:00:00Z +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec index 812198c324217..38f09d2e3c56e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec @@ -26,6 +26,19 @@ first_name:keyword | left:keyword | full_name:keyword | right:keyword | last_nam Georgi | left | Georgi Facello | right | Facello ; +shadowingSubfields +FROM addresses +| KEEP city.country.continent.planet.name, city.country.name, city.name +| DISSECT city.name "%{city.country.continent.planet.name} %{?}" +| SORT city.name +; + +city.country.name:keyword | city.name:keyword | city.country.continent.planet.name:keyword +Netherlands | Amsterdam | null +United States of America | San Francisco | San +Japan | Tokyo | null +; + shadowingSelf FROM employees | KEEP first_name, last_name @@ -50,6 +63,51 @@ last_name:keyword | left:keyword | foo:keyword | middle:keyword | ri Facello | left | Georgi1 Georgi2 Facello | middle | right | Georgi1 | Georgi2 | Facello ; +shadowingInternal +FROM employees +| KEEP first_name, last_name +| WHERE last_name == "Facello" +| EVAL name = concat(first_name, "1 ", last_name) +| DISSECT name "%{foo} %{foo}" +; + +first_name:keyword | last_name:keyword | name:keyword | foo:keyword +Georgi | Facello | Georgi1 Facello | Facello +; + +shadowingWhenPushedDownPastRename +required_capability: fixed_pushdown_past_project +ROW city = "Zürich", long_city_name = "Zurich, the largest city in Switzerland" +| RENAME city AS c +| DISSECT long_city_name "Zurich, the %{city} city in Switzerland" +; + +c:keyword | long_city_name:keyword | city:keyword +Zürich | Zurich, the largest city in Switzerland | largest +; + +shadowingWhenPushedDownPastRename2 +required_capability: fixed_pushdown_past_project +ROW city = "Zürich", long_city_name = "Zurich, the largest city in Switzerland" +| RENAME city AS c +| DISSECT long_city_name "Zurich, the %{city} city in %{foo}" +; + +c:keyword | long_city_name:keyword | city:keyword | foo:keyword +Zürich | Zurich, the largest city in Switzerland | largest | Switzerland +; + +shadowingWhenPushedDownPastRename3 +required_capability: fixed_pushdown_past_project +ROW city = "Zürich", long_city_name = "Zurich, the largest city in Switzerland" +| RENAME long_city_name AS c +| DISSECT c "Zurich, the %{long_city_name} city in Switzerland" +; + +city:keyword | c:keyword | long_city_name:keyword +Zürich | Zurich, the largest city in Switzerland | largest +; + complexPattern ROW a = "1953-01-23T12:15:00Z - some text - 127.0.0.1;" diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec index d34620a9e118d..15fe6853ae491 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec @@ -436,6 +436,23 @@ ROW a = "1.2.3.4 [2023-01-23T12:15:00.000Z] Connected" // end::grokWithEscape-result[] ; +grokWithDuplicateFieldNames +// tag::grokWithDuplicateFieldNames[] +FROM addresses +| KEEP city.name, zip_code +| GROK zip_code "%{WORD:zip_parts} %{WORD:zip_parts}" +// end::grokWithDuplicateFieldNames[] +| SORT city.name +; + +// tag::grokWithDuplicateFieldNames-result[] +city.name:keyword | zip_code:keyword | zip_parts:keyword +Amsterdam | 1016 ED | ["1016", "ED"] +San Francisco | CA 94108 | ["CA", "94108"] +Tokyo | 100-7014 | null +// end::grokWithDuplicateFieldNames-result[] +; + basicDissect // tag::basicDissect[] ROW a = "2023-01-23T12:15:00.000Z - some text - 127.0.0.1" diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec index 35530cf6fdb8e..9886d6cce0ca2 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec @@ -122,3 +122,53 @@ FROM employees | STATS COUNT(*), MIN(salary * 10), MAX(languages)| DROP `COUNT( MIN(salary * 10):i | MAX(languages):i 253240 | 5 ; + +// Not really shadowing, but let's keep the name consistent with the other command's tests +shadowingInternal +FROM employees +| SORT emp_no ASC +| KEEP emp_no, first_name, last_name +| DROP last_name, last_name +| LIMIT 2 +; + +emp_no:integer | first_name:keyword + 10001 | Georgi + 10002 | Bezalel +; + +shadowingInternalWildcard +FROM employees +| SORT emp_no ASC +| KEEP emp_no, first_name, last_name +| DROP last*name, last*name, last*, last_name +| LIMIT 2 +; + +emp_no:integer | first_name:keyword + 10001 | Georgi + 10002 | Bezalel +; + +subfields +FROM addresses +| DROP city.country.continent.planet.name, city.country.continent.name, city.country.name, number, street, zip_code, city.country.continent.planet.name +| SORT city.name +; + +city.country.continent.planet.galaxy:keyword | city.name:keyword +Milky Way | Amsterdam +Milky Way | San Francisco +Milky Way | Tokyo +; + +subfieldsWildcard +FROM addresses +| DROP *.name, number, street, zip_code, *ame +; + +city.country.continent.planet.galaxy:keyword +Milky Way +Milky Way +Milky Way +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec index fc8c48afdf8cc..925c08f317125 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec @@ -69,6 +69,33 @@ ROW left = "left", foo = "foo", client_ip = "172.21.0.5", env = "env", right = " left:keyword | client_ip:keyword | env:keyword | right:keyword | foo:keyword ; +shadowingSubfields#[skip:-8.13.99, reason:ENRICH extended in 8.14.0] +required_capability: enrich_load +FROM addresses +| KEEP city.country.continent.planet.name, city.country.name, city.name +| EVAL city.name = REPLACE(city.name, "San Francisco", "South San Francisco") +| ENRICH city_names ON city.name WITH city.country.continent.planet.name = airport +| SORT city.name +; + +city.country.name:keyword | city.name:keyword | city.country.continent.planet.name:text +Netherlands | Amsterdam | null +United States of America | South San Francisco | San Francisco Int'l +Japan | Tokyo | null +; + +shadowingSubfieldsLimit0#[skip:-8.13.99, reason:ENRICH extended in 8.14.0] +FROM addresses +| KEEP city.country.continent.planet.name, city.country.name, city.name +| EVAL city.name = REPLACE(city.name, "San Francisco", "South San Francisco") +| ENRICH city_names ON city.name WITH city.country.continent.planet.name = airport +| SORT city.name +| LIMIT 0 +; + +city.country.name:keyword | city.name:keyword | city.country.continent.planet.name:text +; + shadowingSelf required_capability: enrich_load ROW left = "left", client_ip = "172.21.0.5", env = "env", right = "right" @@ -107,6 +134,82 @@ ROW left = "left", airport = "Zurich Airport ZRH", city = "Zürich", middle = "m left:keyword | city:keyword | middle:keyword | right:keyword | airport:text | region:text | city_boundary:geo_shape ; +shadowingInternal#[skip:-8.13.99, reason:ENRICH extended in 8.14.0] +required_capability: enrich_load +ROW city = "Zürich" +| ENRICH city_names ON city WITH x = airport, x = region +; + +city:keyword | x:text +Zürich | Bezirk Zürich +; + +shadowingInternalImplicit#[skip:-8.13.99, reason:ENRICH extended in 8.14.0] +required_capability: enrich_load +ROW city = "Zürich" +| ENRICH city_names ON city WITH airport = region +; + +city:keyword | airport:text +Zürich | Bezirk Zürich +; + +shadowingInternalImplicit2#[skip:-8.13.99, reason:ENRICH extended in 8.14.0] +required_capability: enrich_load +ROW city = "Zürich" +| ENRICH city_names ON city WITH airport, airport = region +; + +city:keyword | airport:text +Zürich | Bezirk Zürich +; + +shadowingInternalImplicit3#[skip:-8.13.99, reason:ENRICH extended in 8.14.0] +required_capability: enrich_load +ROW city = "Zürich" +| ENRICH city_names ON city WITH airport = region, airport +; + +city:keyword | airport:text +Zürich | Zurich Int'l +; + +shadowingWhenPushedDownPastRename +required_capability: enrich_load +required_capability: fixed_pushdown_past_project +ROW city = "Zürich", airport = "ZRH" +| RENAME airport AS a +| ENRICH city_names ON city WITH airport +; + +city:keyword | a:keyword | airport:text +Zürich | ZRH | Zurich Int'l +; + +shadowingWhenPushedDownPastRename2 +required_capability: enrich_load +required_capability: fixed_pushdown_past_project +ROW city = "Zürich", airport = "ZRH" +| RENAME airport AS a +| ENRICH city_names ON city WITH airport, region +; + +city:keyword | a:keyword | airport:text | region:text +Zürich | ZRH | Zurich Int'l | Bezirk Zürich +; + +shadowingWhenPushedDownPastRename3 +required_capability: enrich_load +required_capability: fixed_pushdown_past_project +ROW city = "Zürich", airport = "ZRH" +| RENAME city as c +| ENRICH city_names ON c WITH city = airport +; + +c:keyword | airport:keyword | city:text +Zürich | ZRH | Zurich Int'l +; + simple required_capability: enrich_load diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec index 3df3b85e5e3af..61a0ccd4af0c5 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec @@ -15,6 +15,19 @@ left:keyword | right:keyword | x:integer left | right | 1 ; +shadowingSubfields#[skip:-8.13.3,reason:fixed in 8.13] +FROM addresses +| KEEP city.country.continent.planet.name, city.country.name, city.name +| EVAL city.country.continent.planet.name = to_upper(city.country.continent.planet.name) +| SORT city.name +; + +city.country.name:keyword | city.name:keyword | city.country.continent.planet.name:keyword +Netherlands | Amsterdam | EARTH +United States of America | San Francisco | EARTH +Japan | Tokyo | EARTH +; + shadowingSelf ROW left = "left", x = 10000 , right = "right" | EVAL x = x + 1 @@ -33,6 +46,55 @@ left:keyword | middle:keyword | right:keyword | x:integer | y:integer left | middle | right | 9 | 10 ; +shadowingInternal +ROW x = 10000 +| EVAL x = x + 1, x = x - 2 +; + +x:integer +9999 +; + +shadowingWhenPushedDownPastRename +required_capability: fixed_pushdown_past_project +FROM employees +| WHERE emp_no < 10002 +| KEEP emp_no, languages +| RENAME emp_no AS z +| EVAL emp_no = 3 +; + +z:integer | languages:integer | emp_no:integer + 10001 | 2 | 3 +; + +shadowingWhenPushedDownPastRename2 +required_capability: fixed_pushdown_past_project +FROM employees +| WHERE emp_no < 10002 +| KEEP emp_no, languages +| RENAME emp_no AS z +| EVAL emp_no = z + 1, emp_no = emp_no + languages, a = 0, languages = -1 +; + +z:integer | emp_no:integer | a:integer | languages:integer + 10001 | 10004 | 0 | -1 +; + +shadowingWhenPushedDownPastRename3 +required_capability: fixed_pushdown_past_project +FROM employees +| WHERE emp_no < 10002 +| KEEP emp_no, languages +| RENAME emp_no AS z +| EVAL emp_no = z + 1 +; + +z:integer | languages:integer | emp_no:integer + 10001 | 2 | 10002 +; + + withMath row a = 1 | eval b = 2 + 3; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec index 2ee7f783b7e97..537b69547c6be 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec @@ -1,4 +1,11 @@ // Floating point types-specific tests +parseLargeMagnitudeValues +required_capability: fix_parsing_large_negative_numbers +row a = 92233720368547758090, b = -9223372036854775809; + +a:double | b:double +9.223372036854776E+19 | -9.223372036854776E+18 +; inDouble from employees | keep emp_no, height, height.float, height.half_float, height.scaled_float | where height in (2.03, 2.0299999713897705, 2.029296875, 2.0300000000000002) | sort emp_no; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec index 9d574eed7be6b..98c88d06caa75 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec @@ -26,6 +26,19 @@ first_name:keyword | left:keyword | full_name:keyword | right:keyword | last_nam Georgi | left | Georgi Facello | right | Facello ; +shadowingSubfields +FROM addresses +| KEEP city.country.continent.planet.name, city.country.name, city.name +| GROK city.name "%{WORD:city.country.continent.planet.name} %{WORD}" +| SORT city.name +; + +city.country.name:keyword | city.name:keyword | city.country.continent.planet.name:keyword +Netherlands | Amsterdam | null +United States of America | San Francisco | San +Japan | Tokyo | null +; + shadowingSelf FROM employees | KEEP first_name, last_name @@ -50,6 +63,51 @@ last_name:keyword | left:keyword | foo:keyword | middle:keyword | ri Facello | left | Georgi1 Georgi2 Facello | middle | right | Georgi1 | Georgi2 | Facello ; +shadowingInternal +FROM addresses +| KEEP city.name, zip_code +| GROK zip_code "%{WORD:zip_parts} %{WORD:zip_parts}" +| SORT city.name +; + +city.name:keyword | zip_code:keyword | zip_parts:keyword +Amsterdam | 1016 ED | ["1016", "ED"] +San Francisco | CA 94108 | ["CA", "94108"] +Tokyo | 100-7014 | null +; + +shadowingWhenPushedDownPastRename +required_capability: fixed_pushdown_past_project +ROW city = "Zürich", long_city_name = "Zürich, the largest city in Switzerland" +| RENAME city AS c +| GROK long_city_name "Zürich, the %{WORD:city} %{WORD:city} %{WORD:city} %{WORD:city}" +; + +c:keyword | long_city_name:keyword | city:keyword +Zürich | Zürich, the largest city in Switzerland | ["largest", "city", "in", "Switzerland"] +; + +shadowingWhenPushedDownPastRename2 +required_capability: fixed_pushdown_past_project +ROW city = "Zürich", long_city_name = "Zürich, the largest city in Switzerland" +| RENAME city AS c +| GROK long_city_name "Zürich, the %{WORD:city} %{WORD:foo} %{WORD:city} %{WORD:foo}" +; + +c:keyword | long_city_name:keyword | city:keyword | foo:keyword +Zürich | Zürich, the largest city in Switzerland | ["largest", "in"] | ["city", "Switzerland"] +; + +shadowingWhenPushedDownPastRename3 +required_capability: fixed_pushdown_past_project +ROW city = "Zürich", long_city_name = "Zürich, the largest city in Switzerland" +| RENAME long_city_name AS c +| GROK c "Zürich, the %{WORD:long_city_name} %{WORD:long_city_name} %{WORD:long_city_name} %{WORD:long_city_name}" +; + +city:keyword | c:keyword | long_city_name:keyword +Zürich | Zürich, the largest city in Switzerland | ["largest", "city", "in", "Switzerland"] +; complexPattern ROW a = "1953-01-23T12:15:00Z 127.0.0.1 some.email@foo.com 42" diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec new file mode 100644 index 0000000000000..90d5bbd514c81 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec @@ -0,0 +1,503 @@ +maxOfInt +required_capability: inlinestats + +// tag::max-languages[] +FROM employees +| KEEP emp_no, languages +| INLINESTATS max_lang = MAX(languages) +| WHERE max_lang == languages +| SORT emp_no ASC +| LIMIT 5 +// end::max-languages[] +; + +// tag::max-languages-result[] +emp_no:integer | languages:integer | max_lang:integer + 10002 | 5 | 5 + 10004 | 5 | 5 + 10011 | 5 | 5 + 10012 | 5 | 5 + 10014 | 5 | 5 +// end::max-languages-result[] +; + +maxOfIntByKeyword +required_capability: inlinestats + +FROM employees +| KEEP emp_no, languages, gender +| INLINESTATS max_lang = MAX(languages) BY gender +| WHERE max_lang == languages +| SORT emp_no ASC +| LIMIT 5; + +emp_no:integer | languages:integer | gender:keyword | max_lang:integer + 10002 | 5 | F | 5 + 10004 | 5 | M | 5 + 10011 | 5 | null | 5 + 10012 | 5 | null | 5 + 10014 | 5 | null | 5 +; + +maxOfLongByKeyword +required_capability: inlinestats + +FROM employees +| KEEP emp_no, avg_worked_seconds, gender +| INLINESTATS max_avg_worked_seconds = MAX(avg_worked_seconds) BY gender +| WHERE max_avg_worked_seconds == avg_worked_seconds +| SORT emp_no ASC; + +emp_no:integer | avg_worked_seconds:long | gender:keyword | max_avg_worked_seconds:long + 10007 | 393084805 | F | 393084805 + 10015 | 390266432 | null | 390266432 + 10030 | 394597613 | M | 394597613 +; + +maxOfLong +required_capability: inlinestats + +FROM employees +| KEEP emp_no, avg_worked_seconds, gender +| INLINESTATS max_avg_worked_seconds = MAX(avg_worked_seconds) +| WHERE max_avg_worked_seconds == avg_worked_seconds +| SORT emp_no ASC; + +emp_no:integer | avg_worked_seconds:long | gender:keyword | max_avg_worked_seconds:long + 10030 | 394597613 | M | 394597613 +; + +// TODO allow inline calculation like BY l = SUBSTRING( +maxOfLongByCalculatedKeyword +required_capability: inlinestats + +// tag::longest-tenured-by-first[] +FROM employees +| EVAL l = SUBSTRING(last_name, 0, 1) +| KEEP emp_no, avg_worked_seconds, l +| INLINESTATS max_avg_worked_seconds = MAX(avg_worked_seconds) BY l +| WHERE max_avg_worked_seconds == avg_worked_seconds +| SORT l ASC +| LIMIT 5 +// end::longest-tenured-by-first[] +; + +// tag::longest-tenured-by-first-result[] +emp_no:integer | avg_worked_seconds:long | l:keyword | max_avg_worked_seconds:long + 10065 | 372660279 | A | 372660279 + 10074 | 382397583 | B | 382397583 + 10044 | 387408356 | C | 387408356 + 10030 | 394597613 | D | 394597613 + 10087 | 305782871 | E | 305782871 +// end::longest-tenured-by-first-result[] +; + +maxOfLongByInt +required_capability: inlinestats + +FROM employees +| KEEP emp_no, avg_worked_seconds, languages +| INLINESTATS max_avg_worked_seconds = MAX(avg_worked_seconds) BY languages +| WHERE max_avg_worked_seconds == avg_worked_seconds +| SORT languages ASC; + +emp_no:integer | avg_worked_seconds:long | languages:integer | max_avg_worked_seconds:long + 10044 | 387408356 | 1 | 387408356 + 10099 | 377713748 | 2 | 377713748 + 10030 | 394597613 | 3 | 394597613 + 10007 | 393084805 | 4 | 393084805 + 10015 | 390266432 | 5 | 390266432 + 10027 | 374037782 | null | 374037782 +; + +maxOfLongByIntDouble +required_capability: inlinestats + +FROM employees +| KEEP emp_no, avg_worked_seconds, languages, height +| EVAL height=ROUND(height, 1) +| INLINESTATS max_avg_worked_seconds = MAX(avg_worked_seconds) BY languages, height +| WHERE max_avg_worked_seconds == avg_worked_seconds +| SORT languages, height ASC +| LIMIT 4; + +emp_no:integer | avg_worked_seconds:long | languages:integer | height:double | max_avg_worked_seconds:long + 10083 | 331236443 | 1 | 1.4 | 331236443 + 10084 | 359067056 | 1 | 1.5 | 359067056 + 10033 | 208374744 | 1 | 1.6 | 208374744 + 10086 | 328580163 | 1 | 1.7 | 328580163 +; + + +two +required_capability: inlinestats + +FROM employees +| KEEP emp_no, languages, avg_worked_seconds, gender +| INLINESTATS avg_avg_worked_seconds = AVG(avg_worked_seconds) BY languages +| WHERE avg_worked_seconds > avg_avg_worked_seconds +| INLINESTATS max_languages = MAX(languages) BY gender +| SORT emp_no ASC +| LIMIT 3; + +emp_no:integer | languages:integer | avg_worked_seconds:long | gender:keyword | avg_avg_worked_seconds:double | max_languages:integer + 10002 | 5 | 328922887 | F | 3.133013149047619E8 | 5 + 10006 | 3 | 372957040 | F | 2.978159518235294E8 | 5 + 10007 | 4 | 393084805 | F | 2.863684210555556E8 | 5 +; + +byMultivaluedSimple +required_capability: inlinestats + +// tag::mv-group[] +FROM airports +| INLINESTATS min_scalerank=MIN(scalerank) BY type +| EVAL type=MV_SORT(type), min_scalerank=MV_SORT(min_scalerank) +| KEEP abbrev, type, scalerank, min_scalerank +| WHERE abbrev == "GWL" +// end::mv-group[] +; + +// tag::mv-group-result[] +abbrev:keyword | type:keyword | scalerank:integer | min_scalerank:integer + GWL | [mid, military] | 9 | [2, 4] +// end::mv-group-result[] +; + +byMultivaluedMvExpand +required_capability: inlinestats + +// tag::mv-expand[] +FROM airports +| KEEP abbrev, type, scalerank +| MV_EXPAND type +| INLINESTATS min_scalerank=MIN(scalerank) BY type +| SORT min_scalerank ASC +| WHERE abbrev == "GWL" +// end::mv-expand[] +; + +// tag::mv-expand-result[] +abbrev:keyword | type:keyword | scalerank:integer | min_scalerank:integer + GWL | mid | 9 | 2 + GWL | military | 9 | 4 +// end::mv-expand-result[] +; + +byMvExpand +required_capability: inlinestats + +// tag::extreme-airports[] +FROM airports +| MV_EXPAND type +| EVAL lat = ST_Y(location) +| INLINESTATS most_northern=MAX(lat), most_southern=MIN(lat) BY type +| WHERE lat == most_northern OR lat == most_southern +| SORT lat DESC +| KEEP type, name, location +// end::extreme-airports[] +; + +// tag::extreme-airports-result[] + type:keyword | name:text | location:geo_point + mid | Svalbard Longyear | POINT (15.495229 78.246717) + major | Tromsø Langnes | POINT (18.9072624292132 69.6796790473478) + military | Severomorsk-3 (Murmansk N.E.) | POINT (33.2903527616285 69.0168711826804) + spaceport | Baikonur Cosmodrome | POINT (63.307354423875 45.9635739403124) + small | Dhamial | POINT (73.0320498392002 33.5614146278861) + small | Sahnewal | POINT (75.9570722403652 30.8503598561702) + spaceport | Centre Spatial Guyanais | POINT (-52.7684296893452 5.23941001258035) + military | Santos Air Force Base | POINT (-46.3052704931003 -23.9237590410637) + major | Christchurch Int'l | POINT (172.538675565223 -43.4885486784104) + mid | Hermes Quijada Int'l | POINT (-67.7530268462675 -53.7814746058316) +// end::extreme-airports-result[] +; + +brokenwhy-Ignore +required_capability: inlinestats + +FROM airports +| INLINESTATS min_scalerank=MIN(scalerank) BY type +| MV_EXPAND type +| WHERE scalerank == MV_MIN(scalerank); + +abbrev:keyword | type:keyword | scalerank:integer | min_scalerank:integer + GWL | [mid, military] | 9 | [2, 4] +; + +afterStats +required_capability: inlinestats + +FROM airports +| STATS count=COUNT(*) BY country +| INLINESTATS avg=AVG(count) +| WHERE count > avg * 3 +| SORT count DESC, country ASC +; + +count:long | country:keyword | avg:double + 129 | United States | 4.455 + 50 | India | 4.455 + 45 | Mexico | 4.455 + 41 | China | 4.455 + 37 | Canada | 4.455 + 31 | Brazil | 4.455 + 26 | Russia | 4.455 + 19 | null | 4.455 + 17 | Australia | 4.455 + 17 | United Kingdom | 4.455 +; + +afterWhere +required_capability: inlinestats + +FROM airports +| WHERE country != "United States" +| INLINESTATS count=COUNT(*) BY country +| SORT count DESC, abbrev ASC +| KEEP abbrev, country, count +| LIMIT 4 +; + +abbrev:keyword | country:keyword | count:long + AGR | India | 50 + AMD | India | 50 + BBI | India | 50 + BDQ | India | 50 +; + +afterLookup +required_capability: inlinestats + +FROM airports +| RENAME scalerank AS int +| LOOKUP int_number_names ON int +| RENAME name as scalerank +| DROP int +| INLINESTATS count=COUNT(*) BY scalerank +| SORT abbrev ASC +| KEEP abbrev, scalerank +| LIMIT 4 +; + +abbrev:keyword | scalerank:keyword + ABJ | four + ABQ | six + ABV | five + ACA | four +; + +afterEnrich +required_capability: inlinestats +required_capability: enrich_load + +FROM airports +| KEEP abbrev, city +| WHERE abbrev NOT IN ("ADJ", "ATQ") // Skip airports in regions with right-to-left text which the test file isn't good with +| ENRICH city_names ON city WITH region +| WHERE MV_COUNT(region) == 1 +| INLINESTATS COUNT(*) BY region +| SORT abbrev ASC +| WHERE `COUNT(*)` > 1 +| LIMIT 3 +; + +abbrev:keyword | city:keyword | region:text | "COUNT(*)":long + ALA | Almaty | Жетісу ауданы | 2 + BXJ | Almaty | Жетісу ауданы | 2 + FUK | Fukuoka | 中央区 | 2 +; + +beforeStats +required_capability: inlinestats + +FROM airports +| EVAL lat = ST_Y(location) +| INLINESTATS avg_lat=AVG(lat) +| STATS northern=COUNT(lat > avg_lat OR NULL), southern=COUNT(lat < avg_lat OR NULL) +; + +northern:long | southern:long + 520 | 371 +; + +beforeKeepSort +required_capability: inlinestats + +FROM employees +| INLINESTATS max_salary = MAX(salary) by languages +| KEEP emp_no, languages, max_salary +| SORT emp_no ASC +| LIMIT 3; + +emp_no:integer | languages:integer | max_salary:integer + 10001 | 2 | 73578 + 10002 | 5 | 66817 + 10003 | 4 | 74572 +; + +beforeKeep +required_capability: inlinestats + +FROM employees +| INLINESTATS max_salary = MAX(salary) by languages +| KEEP emp_no, languages, max_salary +| LIMIT 3; + +ignoreOrder:true +emp_no:integer | languages:integer | max_salary:integer + 10001 | 2 | 73578 + 10002 | 5 | 66817 + 10003 | 4 | 74572 +; + +beforeEnrich +required_capability: inlinestats +required_capability: enrich_load + +FROM airports +| KEEP abbrev, type, city +| INLINESTATS COUNT(*) BY type +| ENRICH city_names ON city WITH region +| WHERE MV_COUNT(region) == 1 +| SORT abbrev ASC +| LIMIT 3 +; + +abbrev:keyword | type:keyword | city:keyword | "COUNT(*)":long | region:text + ABJ | mid | Abidjan | 499 | Abidjan + ABV | major | Abuja | 385 | Municipal Area Council + ACA | major | Acapulco de Juárez | 385 | Acapulco de Juárez +; + +beforeAndAfterEnrich +required_capability: inlinestats +required_capability: enrich_load + +FROM airports +| KEEP abbrev, type, city +| INLINESTATS COUNT(*) BY type +| ENRICH city_names ON city WITH region +| WHERE MV_COUNT(region) == 1 +| INLINESTATS count_region=COUNT(*) BY region +| SORT abbrev ASC +| LIMIT 3 +; + +abbrev:keyword | type:keyword | city:keyword | "COUNT(*)":long | region:text | count_region:long + ABJ | mid | Abidjan | 499 | Abidjan | 1 + ABV | major | Abuja | 385 | Municipal Area Council | 1 + ACA | major | Acapulco de Juárez | 385 | Acapulco de Juárez | 1 +; + + +shadowing +required_capability: inlinestats + +ROW left = "left", client_ip = "172.21.0.5", env = "env", right = "right" +| INLINESTATS env=VALUES(right) BY client_ip +; + +left:keyword | client_ip:keyword | right:keyword | env:keyword +left | 172.21.0.5 | right | right +; + +shadowingMulti +required_capability: inlinestats + +ROW left = "left", airport = "Zurich Airport ZRH", city = "Zürich", middle = "middle", region = "North-East Switzerland", right = "right" +| INLINESTATS airport=VALUES(left), region=VALUES(left), city_boundary=VALUES(left) BY city +; + +left:keyword | city:keyword | middle:keyword | right:keyword | airport:keyword | region:keyword | city_boundary:keyword +left | Zürich | middle | right | left | left | left +; + +shadowingSelf +required_capability: inlinestats + +ROW city="Raleigh" +| INLINESTATS city=COUNT(city) +; + +city:long +1 +; + +shadowingSelfBySelf-Ignore +required_capability: inlinestats + +ROW city="Raleigh" +| INLINESTATS city=COUNT(city) BY city +; + +city:long +1 +; + +shadowingInternal-Ignore +required_capability: inlinestats + +ROW city = "Zürich" +| INLINESTATS x=VALUES(city), x=VALUES(city) +; + +city:keyword | x:keyword +Zürich | Zürich +; + +byConstant-Ignore +required_capability: inlinestats + +FROM employees +| KEEP emp_no, languages +| INLINESTATS max_lang = MAX(languages) BY y=1 +| WHERE max_lang == languages +| SORT emp_no ASC +| LIMIT 5 +; + +emp_no:integer | languages:integer | max_lang:integer | y:integer + 10002 | 5 | 5 | 1 + 10004 | 5 | 5 | 1 + 10011 | 5 | 5 | 1 + 10012 | 5 | 5 | 1 + 10014 | 5 | 5 | 1 +; + +aggConstant +required_capability: inlinestats + +FROM employees +| KEEP emp_no +| INLINESTATS one = MAX(1) BY emp_no +| SORT emp_no ASC +| LIMIT 5 +; + +emp_no:integer | one:integer + 10001 | 1 + 10002 | 1 + 10003 | 1 + 10004 | 1 + 10005 | 1 +; + +percentile +required_capability: inlinestats + +FROM employees +| KEEP emp_no, salary +| INLINESTATS ninety_fifth_salary = PERCENTILE(salary, 95) +| WHERE salary > ninety_fifth_salary +| SORT emp_no ASC +| LIMIT 5 +; + +emp_no:integer | salary:integer | ninety_fifth_salary:double + 10007 | 74572 | 73584.95 + 10019 | 73717 | 73584.95 + 10027 | 73851 | 73584.95 + 10029 | 74999 | 73584.95 + 10045 | 74970 | 73584.95 +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/keep.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/keep.csv-spec index 14a3807b8729c..6bc534a9fd918 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/keep.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/keep.csv-spec @@ -539,3 +539,63 @@ c:i 1 1 ; + +shadowingInternal#[skip:-8.13.3,reason:fixed in 8.13] +FROM employees +| SORT emp_no ASC +| KEEP last_name, emp_no, last_name +| LIMIT 2 +; + +emp_no:integer | last_name:keyword + 10001 | Facello + 10002 | Simmel +; + +shadowingInternalWildcard#[skip:-8.13.3,reason:fixed in 8.13] +FROM employees +| SORT emp_no ASC +| KEEP last*name, emp_no, last*name, first_name, last*, gender, last* +| LIMIT 2 +; + +emp_no:integer | first_name:keyword | gender:keyword | last_name:keyword + 10001 | Georgi | M | Facello + 10002 | Bezalel | F | Simmel +; + +shadowingInternalWildcardAndExplicit#[skip:-8.13.3,reason:fixed in 8.13] +FROM employees +| SORT emp_no ASC +| KEEP last*name, emp_no, last_name, first_name, last*, languages, last_name, gender, last*name +| LIMIT 2 +; + +emp_no:integer | first_name:keyword | languages:integer | last_name:keyword | gender:keyword + 10001 | Georgi | 2 | Facello | M + 10002 | Bezalel | 5 | Simmel | F +; + +shadowingSubfields#[skip:-8.13.3,reason:fixed in 8.13] +FROM addresses +| KEEP city.country.continent.planet.name, city.country.continent.name, city.country.name, city.name, city.country.continent.planet.name +| SORT city.name +; + +city.country.continent.name:keyword | city.country.name:keyword | city.name:keyword | city.country.continent.planet.name:keyword +Europe | Netherlands | Amsterdam | Earth +North America | United States of America | San Francisco | Earth +Asia | Japan | Tokyo | Earth +; + +shadowingSubfieldsWildcard#[skip:-8.13.3,reason:fixed in 8.13] +FROM addresses +| KEEP *name, city.country.continent.planet.name +| SORT city.name +; + +city.country.continent.name:keyword | city.country.name:keyword | city.name:keyword | city.country.continent.planet.name:keyword +Europe | Netherlands | Amsterdam | Earth +North America | United States of America | San Francisco | Earth +Asia | Japan | Tokyo | Earth +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-addresses.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-addresses.json new file mode 100644 index 0000000000000..679efb3c8d38b --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-addresses.json @@ -0,0 +1,44 @@ +{ + "properties" : { + "street" : { + "type": "keyword" + }, + "number" : { + "type": "keyword" + }, + "zip_code": { + "type": "keyword" + }, + "city" : { + "properties": { + "name": { + "type": "keyword" + }, + "country": { + "properties": { + "name": { + "type": "keyword" + }, + "continent": { + "properties": { + "name": { + "type": "keyword" + }, + "planet": { + "properties": { + "name": { + "type": "keyword" + }, + "galaxy": { + "type": "keyword" + } + } + } + } + } + } + } + } + } + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec index 8337af42df5ea..b00bb5143726c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec @@ -190,6 +190,19 @@ s:double 25976.0 ; +exp#[skip:-8.15.99,reason:new scalar function added in 8.16] +// tag::exp[] +ROW d = 5.0 +| EVAL s = EXP(d) +// end::exp[] +; + +// tag::exp-result[] +d: double | s:double +5.0 | 148.413159102576603 +// end::exp-result[] +; + powHeightSquared from employees | sort height asc | limit 20 | eval s = round(pow(height, 2) - 2, 2) | keep height, s | sort s desc | limit 4; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec index e7fa027ff1d6e..1e23df1ab9107 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec @@ -27,6 +27,7 @@ synopsis:keyword "date date_trunc(interval:date_period|time_duration, date:date)" double e() "boolean ends_with(str:keyword|text, suffix:keyword|text)" +"double exp(number:double|integer|long|unsigned_long)" "double|integer|long|unsigned_long floor(number:double|integer|long|unsigned_long)" "keyword from_base64(string:keyword|text)" "boolean|double|integer|ip|keyword|long|text|version greatest(first:boolean|double|integer|ip|keyword|long|text|version, ?rest...:boolean|double|integer|ip|keyword|long|text|version)" @@ -38,10 +39,10 @@ double e() "double log(?base:integer|unsigned_long|long|double, number:integer|unsigned_long|long|double)" "double log10(number:double|integer|long|unsigned_long)" "keyword|text ltrim(string:keyword|text)" -"boolean|double|integer|long|date max(field:boolean|double|integer|long|date)" +"boolean|double|integer|long|date|ip max(field:boolean|double|integer|long|date|ip)" "double|integer|long median(number:double|integer|long)" "double|integer|long median_absolute_deviation(number:double|integer|long)" -"boolean|double|integer|long|date min(field:boolean|double|integer|long|date)" +"boolean|double|integer|long|date|ip min(field:boolean|double|integer|long|date|ip)" "boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version mv_append(field1:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version, field2:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version)" "double mv_avg(number:double|integer|long|unsigned_long)" "keyword mv_concat(string:text|keyword, delim:text|keyword)" @@ -57,7 +58,7 @@ double e() "double|integer|long|unsigned_long mv_sum(number:double|integer|long|unsigned_long)" "keyword mv_zip(string1:keyword|text, string2:keyword|text, ?delim:keyword|text)" date now() -"double|integer|long percentile(number:double|integer|long, percentile:double|integer|long)" +"double percentile(number:double|integer|long, percentile:double|integer|long)" double pi() "double pow(base:double|integer|long|unsigned_long, exponent:double|integer|long|unsigned_long)" "keyword repeat(string:keyword|text, number:integer)" @@ -80,7 +81,7 @@ double pi() "double st_y(point:geo_point|cartesian_point)" "boolean starts_with(str:keyword|text, prefix:keyword|text)" "keyword substring(string:keyword|text, start:integer, ?length:integer)" -"long sum(number:double|integer|long)" +"long|double sum(number:double|integer|long)" "double tan(angle:double|integer|long|unsigned_long)" "double tanh(angle:double|integer|long|unsigned_long)" double tau() @@ -110,7 +111,7 @@ double tau() "keyword|text to_upper(str:keyword|text)" "version to_ver(field:keyword|text|version)" "version to_version(field:keyword|text|version)" -"double|integer|long|date top(field:double|integer|long|date, limit:integer, order:keyword)" +"boolean|double|integer|long|date|ip top(field:boolean|double|integer|long|date|ip, limit:integer, order:keyword)" "keyword|text trim(string:keyword|text)" "boolean|date|double|integer|ip|keyword|long|text|version values(field:boolean|date|double|integer|ip|keyword|long|text|version)" "double weighted_avg(number:double|integer|long, weight:double|integer|long)" @@ -147,6 +148,7 @@ date_parse |[datePattern, dateString] |["keyword|text", "keyword|te date_trunc |[interval, date] |["date_period|time_duration", date] |[Interval; expressed using the timespan literal syntax., Date expression] e |null |null |null ends_with |[str, suffix] |["keyword|text", "keyword|text"] |[String expression. If `null`\, the function returns `null`., String expression. If `null`\, the function returns `null`.] +exp |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. floor |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. from_base64 |string |"keyword|text" |A base64 string. greatest |first |"boolean|double|integer|ip|keyword|long|text|version" |First of the columns to evaluate. @@ -158,10 +160,10 @@ locate |[string, substring, start] |["keyword|text", "keyword|te log |[base, number] |["integer|unsigned_long|long|double", "integer|unsigned_long|long|double"] |["Base of logarithm. If `null`\, the function returns `null`. If not provided\, this function returns the natural logarithm (base e) of a value.", "Numeric expression. If `null`\, the function returns `null`."] log10 |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. ltrim |string |"keyword|text" |String expression. If `null`, the function returns `null`. -max |field |"boolean|double|integer|long|date" |[""] +max |field |"boolean|double|integer|long|date|ip" |[""] median |number |"double|integer|long" |[""] median_absolut|number |"double|integer|long" |[""] -min |field |"boolean|double|integer|long|date" |[""] +min |field |"boolean|double|integer|long|date|ip" |[""] mv_append |[field1, field2] |["boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version", "boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version"] | ["", ""] mv_avg |number |"double|integer|long|unsigned_long" |Multivalue expression. mv_concat |[string, delim] |["text|keyword", "text|keyword"] |[Multivalue expression., Delimiter.] @@ -230,7 +232,7 @@ to_unsigned_lo|field |"boolean|date|keyword|text|d to_upper |str |"keyword|text" |String expression. If `null`, the function returns `null`. to_ver |field |"keyword|text|version" |Input value. The input can be a single- or multi-valued column or an expression. to_version |field |"keyword|text|version" |Input value. The input can be a single- or multi-valued column or an expression. -top |[field, limit, order] |["double|integer|long|date", integer, keyword] |[The field to collect the top values for.,The maximum number of values to collect.,The order to calculate the top values. Either `asc` or `desc`.] +top |[field, limit, order] |["boolean|double|integer|long|date|ip", integer, keyword] |[The field to collect the top values for.,The maximum number of values to collect.,The order to calculate the top values. Either `asc` or `desc`.] trim |string |"keyword|text" |String expression. If `null`, the function returns `null`. values |field |"boolean|date|double|integer|ip|keyword|long|text|version" |[""] weighted_avg |[number, weight] |["double|integer|long", "double|integer|long"] |[A numeric value., A numeric weight.] @@ -268,6 +270,7 @@ date_parse |Returns a date by parsing the second argument using the format sp date_trunc |Rounds down a date to the closest interval. e |Returns {wikipedia}/E_(mathematical_constant)[Euler's number]. ends_with |Returns a boolean that indicates whether a keyword string ends with another string. +exp |Returns the value of e raised to the power of the given number. floor |Round a number down to the nearest integer. from_base64 |Decode a base64 string. greatest |Returns the maximum value from multiple columns. This is similar to <> except it is intended to run on multiple columns at once. @@ -275,7 +278,7 @@ ip_prefix |Truncates an IP to a given prefix length. least |Returns the minimum value from multiple columns. This is similar to <> except it is intended to run on multiple columns at once. left |Returns the substring that extracts 'length' chars from 'string' starting from the left. length |Returns the character length of a string. -locate |Returns an integer that indicates the position of a keyword substring within another string +locate |Returns an integer that indicates the position of a keyword substring within another string. log |Returns the logarithm of a value to a base. The input can be any numeric value, the return value is always a double. Logs of zero, negative numbers, and base of one return `null` as well as a warning. log10 |Returns the logarithm of a value to base 10. The input can be any numeric value, the return value is always a double. Logs of 0 and negative numbers return `null` as well as a warning. ltrim |Removes leading whitespaces from a string. @@ -298,7 +301,7 @@ mv_sort |Sorts a multivalued field in lexicographical order. mv_sum |Converts a multivalued field into a single valued field containing the sum of all of the values. mv_zip |Combines the values from two multivalued fields with a delimiter that joins them together. now |Returns current date and time. -percentile |The value at which a certain percentage of observed values occur. +percentile |Returns the value at which a certain percentage of observed values occur. For example, the 95th percentile is the value which is greater than 95% of the observed values and the 50th percentile is the `MEDIAN`. pi |Returns {wikipedia}/Pi[Pi], the ratio of a circle's circumference to its diameter. pow |Returns the value of `base` raised to the power of `exponent`. repeat |Returns a string constructed by concatenating `string` with itself the specified `number` of times. @@ -320,8 +323,8 @@ st_within |Returns whether the first geometry is within the second geometry. st_x |Extracts the `x` coordinate from the supplied point. If the points is of type `geo_point` this is equivalent to extracting the `longitude` value. st_y |Extracts the `y` coordinate from the supplied point. If the points is of type `geo_point` this is equivalent to extracting the `latitude` value. starts_with |Returns a boolean that indicates whether a keyword string starts with another string. -substring |Returns a substring of a string, specified by a start position and an optional length -sum |The sum of a numeric field. +substring |Returns a substring of a string, specified by a start position and an optional length. +sum |The sum of a numeric expression. tan |Returns the {wikipedia}/Sine_and_cosine[Tangent] trigonometric function of an angle. tanh |Returns the {wikipedia}/Hyperbolic_functions[Tangent] hyperbolic function of an angle. tau |Returns the https://tauday.com/tau-manifesto[ratio] of a circle's circumference to its radius. @@ -351,7 +354,7 @@ to_unsigned_lo|Converts an input value to an unsigned long value. If the input p to_upper |Returns a new string representing the input string converted to upper case. to_ver |Converts an input string to a version value. to_version |Converts an input string to a version value. -top |Collects the top values for a field. Includes repeated values. +top |Collects the top values for a field. Includes repeated values. trim |Removes leading and trailing whitespaces from a string. values |Collect values for a field. weighted_avg |The weighted average of a numeric field. @@ -390,6 +393,7 @@ date_parse |date date_trunc |date |[false, false] |false |false e |double |null |false |false ends_with |boolean |[false, false] |false |false +exp |double |false |false |false floor |"double|integer|long|unsigned_long" |false |false |false from_base64 |keyword |false |false |false greatest |"boolean|double|integer|ip|keyword|long|text|version" |false |true |false @@ -401,10 +405,10 @@ locate |integer log |double |[true, false] |false |false log10 |double |false |false |false ltrim |"keyword|text" |false |false |false -max |"boolean|double|integer|long|date" |false |false |true +max |"boolean|double|integer|long|date|ip" |false |false |true median |"double|integer|long" |false |false |true median_absolut|"double|integer|long" |false |false |true -min |"boolean|double|integer|long|date" |false |false |true +min |"boolean|double|integer|long|date|ip" |false |false |true mv_append |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version" |[false, false] |false |false mv_avg |double |false |false |false mv_concat |keyword |[false, false] |false |false @@ -420,7 +424,7 @@ mv_sort |"boolean|date|double|integer|ip|keyword|long|text|version" mv_sum |"double|integer|long|unsigned_long" |false |false |false mv_zip |keyword |[false, false, true] |false |false now |date |null |false |false -percentile |"double|integer|long" |[false, false] |false |true +percentile |double |[false, false] |false |true pi |double |null |false |false pow |double |[false, false] |false |false repeat |keyword |[false, false] |false |false @@ -443,7 +447,7 @@ st_x |double st_y |double |false |false |false starts_with |boolean |[false, false] |false |false substring |keyword |[false, false, true] |false |false -sum |long |false |false |true +sum |"long|double" |false |false |true tan |double |false |false |false tanh |double |false |false |false tau |double |null |false |false @@ -473,7 +477,7 @@ to_unsigned_lo|unsigned_long to_upper |"keyword|text" |false |false |false to_ver |version |false |false |false to_version |version |false |false |false -top |"double|integer|long|date" |[false, false, false] |false |true +top |"boolean|double|integer|long|date|ip" |[false, false, false] |false |true trim |"keyword|text" |false |false |false values |"boolean|date|double|integer|ip|keyword|long|text|version" |false |false |true weighted_avg |"double" |[false, false] |false |true @@ -493,5 +497,5 @@ countFunctions#[skip:-8.15.99] meta functions | stats a = count(*), b = count(*), c = count(*) | mv_expand c; a:long | b:long | c:long -112 | 112 | 112 +113 | 113 | 113 ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec index 1e830486cc7c7..ca4c627cae749 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec @@ -174,3 +174,42 @@ avg_worked_seconds:l | birth_date:date | emp_no:i | first_n 341158890 | 1961-10-15T00:00:00.000Z | 10060 | Breannda | M | 1.42 | 1.4199999570846558 | 1.419921875 | 1.42 | 1987-11-02T00:00:00.000Z | [false, false, false, true]| [Business Analyst, Data Scientist, Senior Team Lead] | 2 | 2 | 2 | 2 | Billingsley | 29175 | [-1.76, -0.85] | [-1, 0] | [-0.85, -1.76] | [-1, 0] | true | 29175 246355863 | null | 10042 | Magy | F | 1.44 | 1.440000057220459 | 1.4404296875 | 1.44 | 1993-03-21T00:00:00.000Z | null | [Architect, Business Analyst, Internship, Junior Developer] | 3 | 3 | 3 | 3 | Stamatiou | 30404 | [-9.28, 9.42] | [-9, 9] | [-9.28, 9.42] | [-9, 9] | true | 30404 ; + +shadowing +FROM employees +| SORT emp_no ASC +| KEEP emp_no, first_name, last_name +| RENAME emp_no AS last_name +| LIMIT 2 +; + +last_name:integer | first_name:keyword + 10001 | Georgi + 10002 | Bezalel +; + +shadowingSubfields +FROM addresses +| KEEP city.country.continent.planet.name, city.country.continent.name, city.country.name, city.name +| RENAME city.name AS city.country.continent.planet.name, city.country.name AS city.country.continent.name +| SORT city.country.continent.planet.name +; + +city.country.continent.name:keyword | city.country.continent.planet.name:keyword +Netherlands | Amsterdam +United States of America | San Francisco +Japan | Tokyo +; + +shadowingInternal +FROM employees +| SORT emp_no ASC +| KEEP emp_no, last_name +| RENAME emp_no AS x, last_name AS x +| LIMIT 2 +; + +x:keyword +Facello +Simmel +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/row.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/row.csv-spec index bb1cf7358ca74..da640b6306299 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/row.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/row.csv-spec @@ -36,6 +36,24 @@ a:integer // end::multivalue-result[] ; +shadowingInternal +required_capability: unique_names +ROW a = 1, a = 2; + +a:integer + 2 +; + +shadowingInternalSubfields +required_capability: unique_names +// Fun fact: "Sissi" is an actual exoplanet name, after the character from the movie with the same name. A.k.a. HAT-P-14 b. +ROW city.country.continent.planet.name = "Earth", city.country.continent.name = "Netherlands", city.country.continent.planet.name = "Sissi" +; + +city.country.continent.name:keyword | city.country.continent.planet.name:keyword +Netherlands | Sissi +; + unsignedLongLiteral ROW long_max = 9223372036854775807, ul_start = 9223372036854775808, ul_end = 18446744073709551615, double=18446744073709551616; @@ -70,10 +88,11 @@ a:integer | b:integer | c:null | z:integer ; evalRowWithNull2 +required_capability: unique_names row a = 1, null, b = 2, c = null, null | eval z = a+b; -a:integer | null:null | b:integer | c:null | null:null | z:integer -1 | null | 2 | null | null | 3 +a:integer | b:integer | c:null | null:null | z:integer + 1 | 2 | null | null | 3 ; evalRowWithNull3 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index 2d306cd8fd2a0..7b0dda9eb3669 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -50,6 +50,32 @@ a:boolean | b:boolean | c:boolean | d:boolean true | true | false | true ; +maxOfIp +required_capability: agg_max_min_ip_support +from hosts +| eval x = ip0 +| where host > "alpha" +| stats max(ip0), a = max(ip0), b = max(x), c = max(case(host == "beta", ip0, ip1)); + +max(ip0):ip | a:ip | b:ip | c:ip +fe82::cae2:65ff:fece:fec0 | fe82::cae2:65ff:fece:fec0 | fe82::cae2:65ff:fece:fec0 | fe82::cae2:65ff:fece:fec0 +; + + +maxOfIpGrouping +required_capability: agg_max_min_ip_support +from hosts +| eval x = ip0 +| where host > "alpha" +| stats max(ip0), a = max(ip0), b = max(x), c = max(case(host == "beta", ip0, ip1)) by host +| sort host asc; + +max(ip0):ip | a:ip | b:ip | c:ip | host:keyword +127.0.0.1 | 127.0.0.1 | 127.0.0.1 | 127.0.0.1 | beta +fe82::cae2:65ff:fece:fec0 | fe82::cae2:65ff:fece:fec0 | fe82::cae2:65ff:fece:fec0 | fe82::cae2:65ff:fece:fec0 | epsilon +fe80::cae2:65ff:fece:feb9 | fe80::cae2:65ff:fece:feb9 | fe80::cae2:65ff:fece:feb9 | fe81::cae2:65ff:fece:feb9 | gamma +; + minOfBooleanExpression required_capability: agg_max_min_boolean_support from employees @@ -69,6 +95,32 @@ s:boolean false ; +minOfIp +required_capability: agg_max_min_ip_support +from hosts +| eval x = ip0 +| where host > "alpha" +| stats min(ip0), a = min(ip0), b = min(x), c = min(case(host == "beta", ip0, ip1)); + +min(ip0):ip | a:ip | b:ip | c:ip +127.0.0.1 | 127.0.0.1 | 127.0.0.1 | 127.0.0.1 +; + + +minOfIpGrouping +required_capability: agg_max_min_ip_support +from hosts +| eval x = ip0 +| where host > "alpha" +| stats min(ip0), a = min(ip0), b = min(x), c = min(case(host == "beta", ip0, ip1)) by host +| sort host asc; + +min(ip0):ip | a:ip | b:ip | c:ip | host:keyword +127.0.0.1 | 127.0.0.1 | 127.0.0.1 | 127.0.0.1 | beta +fe80::cae2:65ff:fece:feb9 | fe80::cae2:65ff:fece:feb9 | fe80::cae2:65ff:fece:feb9 | 127.0.0.1 | epsilon +fe80::cae2:65ff:fece:feb9 | fe80::cae2:65ff:fece:feb9 | fe80::cae2:65ff:fece:feb9 | 127.0.0.3 | gamma +; + maxOfShort // short becomes int until https://github.com/elastic/elasticsearch-internal/issues/724 from employees | stats l = max(languages.short); @@ -613,6 +665,65 @@ ca:l | cx:l | l:i 1 | 1 | null ; +/////////////////////////////////////////////////////////////// +// Test edge case interaction with push down past a rename +// https://github.com/elastic/elasticsearch/issues/108008 +/////////////////////////////////////////////////////////////// + +countSameFieldWithEval +required_capability: fixed_pushdown_past_project +from employees | stats b = count(gender), c = count(gender) by gender | eval b = gender | sort c asc +; + +c:l | gender:s | b:s +0 | null | null +33 | F | F +57 | M | M +; + +countSameFieldWithDissect +required_capability: fixed_pushdown_past_project +from employees | stats b = count(gender), c = count(gender) by gender | dissect gender "%{b}" | sort c asc +; + +c:l | gender:s | b:s +0 | null | null +33 | F | F +57 | M | M +; + +countSameFieldWithGrok +required_capability: fixed_pushdown_past_project +from employees | stats b = count(gender), c = count(gender) by gender | grok gender "%{USERNAME:b}" | sort c asc +; + +c:l | gender:s | b:s +0 | null | null +33 | F | F +57 | M | M +; + +countSameFieldWithEnrich +required_capability: fixed_pushdown_past_project +required_capability: enrich_load +from employees | stats b = count(gender), c = count(gender) by gender | enrich languages_policy on gender with b = language_name | sort c asc +; + +c:l | gender:s | b:s +0 | null | null +33 | F | null +57 | M | null +; + +countSameFieldWithEnrichLimit0 +required_capability: fixed_pushdown_past_project +from employees | stats b = count(gender), c = count(gender) by gender | enrich languages_policy on gender with b = language_name | sort c asc | limit 0 +; + +c:l | gender:s | b:s +; +/////////////////////////////////////////////////////////////// + aggsWithoutStats from employees | stats by gender | sort gender; @@ -1158,6 +1269,34 @@ word_count:long // end::docsCountWithExpression-result[] ; +count_where#[skip:-8.12.1,reason:implemented in 8.12] +// tag::count-where[] +ROW n=1 +| WHERE n < 0 +| STATS COUNT(n) +// end::count-where[] +; + +// tag::count-where-result[] +COUNT(n):long + 0 +// end::count-where-result[] +; + + +count_or_null#[skip:-8.14.1,reason:implemented in 8.14] +// tag::count-or-null[] +ROW n=1 +| STATS COUNT(n > 0 OR NULL), COUNT(n < 0 OR NULL) +// end::count-or-null[] +; + +// tag::count-or-null-result[] +COUNT(n > 0 OR NULL):long | COUNT(n < 0 OR NULL):long + 1 | 0 +// end::count-or-null-result[] +; + countMultiValuesRow ROW keyword_field = ["foo", "bar"], int_field = [1, 2, 3] | STATS ck = COUNT(keyword_field), ci = COUNT(int_field), c = COUNT(*); @@ -1857,3 +1996,93 @@ warning:Line 3:17: java.lang.ArithmeticException: / by zero w_avg:double null ; + +shadowingInternal +FROM employees +| STATS x = MAX(emp_no), x = MIN(emp_no) +; + +x:integer +10001 +; + +shadowingInternalWithGroup#[skip:-8.14.1,reason:implemented in 8.14] +FROM employees +| STATS x = MAX(emp_no), x = MIN(emp_no) BY x = gender +| SORT x ASC +; + +x:keyword +F +M +null +; + +shadowingTheGroup +FROM employees +| STATS gender = MAX(emp_no), gender = MIN(emp_no) BY gender +| SORT gender ASC +; + +gender:keyword +F +M +null +; + +docsStatsMvGroup +// tag::mv-group[] +ROW i=1, a=["a", "b"] | STATS MIN(i) BY a | SORT a ASC +// end::mv-group[] +; + +// tag::mv-group-result[] +MIN(i):integer | a:keyword + 1 | a + 1 | b +// end::mv-group-result[] +; + +docsStatsMultiMvGroup +// tag::multi-mv-group[] +ROW i=1, a=["a", "b"], b=[2, 3] | STATS MIN(i) BY a, b | SORT a ASC, b ASC +// end::multi-mv-group[] +; + +// tag::multi-mv-group-result[] +MIN(i):integer | a:keyword | b:integer + 1 | a | 2 + 1 | a | 3 + 1 | b | 2 + 1 | b | 3 +// end::multi-mv-group-result[] +; + +statsByConstant#[skip:-8.14.1,reason:implemented in 8.14] +from employees +| stats m = max(salary), a = round(avg(salary)) by 0 +; + +m:integer |a:double |0:integer +74999 |48249.0 |0 +; + +statsByConstantFromStats#[skip:-8.12.1,reason:implemented in 8.12] +from employees +| stats c = count(languages) +| stats a = count(*) by c +; + +a:long |c:long +1 |90 +; + +statsByConstantFromEval#[skip:-8.14.1,reason:implemented in 8.14] +from employees +| eval x = 0 +| stats m = max(salary), a = round(avg(salary)) by x +; + +m:integer |a:double |x:integer +74999 |48249.0 |0 +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_top.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_top.csv-spec index d03bdb3c3dfd7..86f91adf506d1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_top.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_top.csv-spec @@ -106,8 +106,8 @@ FROM employees long = TOP(salary_change.long, 1, "asc") ; -date:date | double:double | integer:integer | long:long -1985-02-18T00:00:00.000Z | -9.81 | 25324 | -9 +date:date | double:double | integer:integer | long:long +1985-02-18T00:00:00.000Z | -9.81 | 25324 | -9 ; topAllTypesMax @@ -120,8 +120,8 @@ FROM employees long = TOP(salary_change.long, 1, "desc") ; -date:date | double:double | integer:integer | long:long -1999-04-30T00:00:00.000Z | 14.74 | 74999 | 14 +date:date | double:double | integer:integer | long:long +1999-04-30T00:00:00.000Z | 14.74 | 74999 | 14 ; topAscDesc @@ -154,3 +154,73 @@ FROM employees integer:integer [5, 5] ; + +topBooleans +required_capability: agg_top +required_capability: agg_top_boolean_support +FROM employees +| eval x = salary is not null +| where emp_no > 10050 +| STATS + top_asc = TOP(still_hired, 2, "asc"), + min = TOP(still_hired, 1, "asc"), + top_desc = TOP(still_hired, 2, "desc"), + max = TOP(still_hired, 1, "desc"), + a = TOP(salary is not null, 2, "asc"), + b = TOP(x, 2, "asc"), + c = TOP(case(salary is null, true, false), 2, "asc"), + d = TOP(is_rehired, 2, "asc") +; + +top_asc:boolean | min:boolean | top_desc:boolean | max:boolean | a:boolean | b:boolean | c:boolean | d:boolean +[false, false] | false | [true, true] | true | [true, true] | [true, true] | [false, false] | [false, false] +; + +topBooleansRow +required_capability: agg_top +required_capability: agg_top_boolean_support +ROW constant = true, mv = [true, false] +| STATS + constant_asc = TOP(constant, 2, "asc"), + constant_desc = TOP(constant, 2, "desc"), + mv_asc = TOP(mv, 2, "asc"), + mv_desc = TOP(mv, 2, "desc") +; + +constant_asc:boolean | constant_desc:boolean | mv_asc:boolean | mv_desc:boolean +true | true | [false, true] | [true, false] +; + +topIps +required_capability: agg_top +required_capability: agg_top_ip_support +from hosts +| eval x = ip0 +| where host > "alpha" +| stats + a = TOP(ip0, 2, "desc"), + b = TOP(x, 2, "desc"), + c = TOP(case(host == "beta", ip0, ip1), 2, "desc"); + +a:ip | b:ip | c:ip +[fe82::cae2:65ff:fece:fec0, fe81::cae2:65ff:fece:feb9] | [fe82::cae2:65ff:fece:fec0, fe81::cae2:65ff:fece:feb9] | [fe82::cae2:65ff:fece:fec0, fe81::cae2:65ff:fece:feb9] +; + +topIpsGrouping +required_capability: agg_top +required_capability: agg_top_ip_support +from hosts +| eval x = ip0 +| where host > "alpha" +| stats + a = TOP(ip0, 2, "desc"), + b = TOP(x, 2, "desc"), + c = TOP(case(host == "beta", ip0, ip1), 2, "desc") + by host +| sort host asc; + +a:ip | b:ip | c:ip | host:keyword +[127.0.0.1, 127.0.0.1] | [127.0.0.1, 127.0.0.1] | [127.0.0.1, 127.0.0.1] | beta +[fe82::cae2:65ff:fece:fec0, fe81::cae2:65ff:fece:feb9] | [fe82::cae2:65ff:fece:fec0, fe81::cae2:65ff:fece:feb9] | [fe82::cae2:65ff:fece:fec0, fe81::cae2:65ff:fece:feb9] | epsilon +[fe80::cae2:65ff:fece:feb9, fe80::cae2:65ff:fece:feb9] | [fe80::cae2:65ff:fece:feb9, fe80::cae2:65ff:fece:feb9] | [fe81::cae2:65ff:fece:feb9, 127.0.0.3] | gamma +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index 349f968666132..e1aa411702420 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -97,6 +97,7 @@ multiIndexIpString required_capability: union_types required_capability: metadata_fields required_capability: casting_operator +required_capability: union_types_remove_fields FROM sample_data, sample_data_str METADATA _index | EVAL client_ip = client_ip::ip @@ -125,6 +126,7 @@ multiIndexIpStringRename required_capability: union_types required_capability: metadata_fields required_capability: casting_operator +required_capability: union_types_remove_fields FROM sample_data, sample_data_str METADATA _index | EVAL host_ip = client_ip::ip @@ -152,6 +154,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 multiIndexIpStringRenameToString required_capability: union_types required_capability: metadata_fields +required_capability: union_types_remove_fields FROM sample_data, sample_data_str METADATA _index | EVAL host_ip = TO_STRING(TO_IP(client_ip)) @@ -179,6 +182,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 multiIndexWhereIpString required_capability: union_types required_capability: metadata_fields +required_capability: union_types_remove_fields FROM sample_data, sample_data_str METADATA _index | WHERE STARTS_WITH(TO_STRING(client_ip), "172.21.2") @@ -196,6 +200,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 3450233 | Connected multiIndexWhereIpStringLike required_capability: union_types required_capability: metadata_fields +required_capability: union_types_remove_fields FROM sample_data, sample_data_str METADATA _index | WHERE TO_STRING(client_ip) LIKE "172.21.2.*" @@ -210,9 +215,39 @@ sample_data_str | 2023-10-23T12:27:28.948Z | 2764889 | Connected sample_data_str | 2023-10-23T12:15:03.360Z | 3450233 | Connected to 10.1.0.3 ; +multiIndexSortIpString +required_capability: union_types +required_capability: casting_operator +required_capability: union_types_remove_fields + +FROM sample_data, sample_data_str +| SORT client_ip::ip +| LIMIT 1 +; + +@timestamp:date | client_ip:null | event_duration:long | message:keyword +2023-10-23T13:33:34.937Z | null | 1232382 | Disconnected +; + +multiIndexSortIpStringEval +required_capability: union_types +required_capability: casting_operator +required_capability: union_types_remove_fields + +FROM sample_data, sample_data_str +| SORT client_ip::ip, @timestamp ASC +| EVAL client_ip_as_ip = client_ip::ip +| LIMIT 1 +; + +@timestamp:date | client_ip:null | event_duration:long | message:keyword | client_ip_as_ip:ip +2023-10-23T13:33:34.937Z | null | 1232382 | Disconnected | 172.21.0.5 +; + multiIndexIpStringStats required_capability: union_types required_capability: casting_operator +required_capability: union_types_remove_fields FROM sample_data, sample_data_str | EVAL client_ip = client_ip::ip @@ -231,6 +266,7 @@ count:long | client_ip:ip multiIndexIpStringRenameStats required_capability: union_types required_capability: casting_operator +required_capability: union_types_remove_fields FROM sample_data, sample_data_str | EVAL host_ip = client_ip::ip @@ -248,6 +284,7 @@ count:long | host_ip:ip multiIndexIpStringRenameToStringStats required_capability: union_types +required_capability: union_types_remove_fields FROM sample_data, sample_data_str | EVAL host_ip = TO_STRING(TO_IP(client_ip)) @@ -298,6 +335,27 @@ count:long | client_ip:ip 2 | 172.21.2.162 ; +statsUnionAggInline +required_capability: union_types +required_capability: inlinestats + +FROM sample_data, sample_data_str +| STATS + count = COUNT(CIDR_MATCH(TO_IP(client_ip), "172.21.0.0/24") OR NULL) + BY + @timestamp = DATE_TRUNC(10 minutes, @timestamp) +| SORT count DESC, @timestamp ASC +| LIMIT 4 +; + +count:long | @timestamp:date + 2 | 2023-10-23T13:30:00.000Z + 0 | 2023-10-23T12:10:00.000Z + 0 | 2023-10-23T12:20:00.000Z + 0 | 2023-10-23T13:50:00.000Z +; + + multiIndexIpStringStatsInline2 required_capability: union_types required_capability: union_types_agg_cast @@ -333,6 +391,7 @@ mc:l | count:l multiIndexWhereIpStringStats required_capability: union_types +required_capability: union_types_remove_fields FROM sample_data, sample_data_str | WHERE STARTS_WITH(TO_STRING(client_ip), "172.21.2") @@ -349,6 +408,7 @@ count:long | message:keyword multiIndexTsLong required_capability: union_types required_capability: metadata_fields +required_capability: union_types_remove_fields FROM sample_data, sample_data_ts_long METADATA _index | EVAL @timestamp = TO_DATETIME(@timestamp) @@ -376,6 +436,7 @@ sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 multiIndexTsLongRename required_capability: union_types required_capability: metadata_fields +required_capability: union_types_remove_fields FROM sample_data, sample_data_ts_long METADATA _index | EVAL ts = TO_DATETIME(@timestamp) @@ -403,6 +464,7 @@ sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 multiIndexTsLongRenameToString required_capability: union_types required_capability: metadata_fields +required_capability: union_types_remove_fields FROM sample_data, sample_data_ts_long METADATA _index | EVAL ts = TO_STRING(TO_DATETIME(@timestamp)) @@ -430,6 +492,7 @@ sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 multiIndexWhereTsLong required_capability: union_types required_capability: metadata_fields +required_capability: union_types_remove_fields FROM sample_data, sample_data_ts_long METADATA _index | WHERE TO_LONG(@timestamp) < 1698068014937 @@ -446,6 +509,7 @@ sample_data_ts_long | 172.21.2.162 | 3450233 | Connected to 10. multiIndexTsLongStats required_capability: union_types +required_capability: union_types_remove_fields FROM sample_data, sample_data_ts_long | EVAL @timestamp = DATE_TRUNC(1 hour, TO_DATETIME(@timestamp)) @@ -517,6 +581,7 @@ mc:l | count:l multiIndexTsLongStatsStats required_capability: union_types required_capability: union_types_agg_cast +required_capability: union_types_remove_fields FROM sample_data, sample_data_ts_long | EVAL ts = TO_STRING(@timestamp) @@ -531,6 +596,7 @@ mc:l | count:l multiIndexTsLongRenameStats required_capability: union_types +required_capability: union_types_remove_fields FROM sample_data, sample_data_ts_long | EVAL hour = DATE_TRUNC(1 hour, TO_DATETIME(@timestamp)) @@ -546,6 +612,7 @@ count:long | hour:date multiIndexTsLongRenameToDatetimeToStringStats required_capability: union_types +required_capability: union_types_remove_fields FROM sample_data, sample_data_ts_long | EVAL hour = LEFT(TO_STRING(TO_DATETIME(@timestamp)), 13) @@ -561,6 +628,7 @@ count:long | hour:keyword multiIndexTsLongRenameToStringStats required_capability: union_types +required_capability: union_types_remove_fields FROM sample_data, sample_data_ts_long | EVAL mess = LEFT(TO_STRING(@timestamp), 7) @@ -579,6 +647,7 @@ count:long | mess:keyword multiIndexTsLongStatsInline required_capability: union_types +required_capability: union_types_remove_fields FROM sample_data, sample_data_ts_long | STATS count=COUNT(*), max=MAX(TO_DATETIME(@timestamp)) @@ -603,6 +672,7 @@ count:long multiIndexWhereTsLongStats required_capability: union_types +required_capability: union_types_remove_fields FROM sample_data, sample_data_ts_long | WHERE TO_LONG(@timestamp) < 1698068014937 @@ -619,6 +689,7 @@ count:long | message:keyword multiIndexIpStringTsLong required_capability: union_types required_capability: metadata_fields +required_capability: union_types_remove_fields FROM sample_data* METADATA _index | EVAL @timestamp = TO_DATETIME(@timestamp), client_ip = TO_IP(client_ip) @@ -687,6 +758,7 @@ sample_data_ts_long | 8268153 | Connection error multiIndexIpStringTsLongRename required_capability: union_types required_capability: metadata_fields +required_capability: union_types_remove_fields FROM sample_data* METADATA _index | EVAL ts = TO_DATETIME(@timestamp), host_ip = TO_IP(client_ip) @@ -755,6 +827,7 @@ sample_data_ts_long | 8268153 | Connection error multiIndexIpStringTsLongRenameToString required_capability: union_types required_capability: metadata_fields +required_capability: union_types_remove_fields FROM sample_data* METADATA _index | EVAL ts = TO_STRING(TO_DATETIME(@timestamp)), host_ip = TO_STRING(TO_IP(client_ip)) @@ -789,6 +862,7 @@ sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 multiIndexWhereIpStringTsLong required_capability: union_types required_capability: metadata_fields +required_capability: union_types_remove_fields FROM sample_data* METADATA _index | WHERE TO_LONG(@timestamp) < 1698068014937 AND TO_STRING(client_ip) == "172.21.2.162" @@ -804,6 +878,7 @@ sample_data_ts_long | 3450233 | Connected to 10.1.0.3 multiIndexWhereIpStringTsLongStats required_capability: union_types +required_capability: union_types_remove_fields FROM sample_data* | WHERE TO_LONG(@timestamp) < 1698068014937 AND TO_STRING(client_ip) == "172.21.2.162" @@ -819,6 +894,7 @@ count:long | message:keyword multiIndexWhereIpStringLikeTsLong required_capability: union_types required_capability: metadata_fields +required_capability: union_types_remove_fields FROM sample_data* METADATA _index | WHERE TO_LONG(@timestamp) < 1698068014937 AND TO_STRING(client_ip) LIKE "172.21.2.16?" @@ -834,6 +910,7 @@ sample_data_ts_long | 3450233 | Connected to 10.1.0.3 multiIndexWhereIpStringLikeTsLongStats required_capability: union_types +required_capability: union_types_remove_fields FROM sample_data* | WHERE TO_LONG(@timestamp) < 1698068014937 AND TO_STRING(client_ip) LIKE "172.21.2.16?" @@ -849,6 +926,7 @@ count:long | message:keyword multiIndexMultiColumnTypesRename required_capability: union_types required_capability: metadata_fields +required_capability: union_types_remove_fields FROM sample_data* METADATA _index | WHERE event_duration > 8000000 @@ -865,6 +943,7 @@ null | null | 8268153 | Connection error | samp multiIndexMultiColumnTypesRenameAndKeep required_capability: union_types required_capability: metadata_fields +required_capability: union_types_remove_fields FROM sample_data* METADATA _index | WHERE event_duration > 8000000 @@ -882,6 +961,7 @@ sample_data_ts_long | 2023-10-23T13:52:55.015Z | 1698069175015 | 16 multiIndexMultiColumnTypesRenameAndDrop required_capability: union_types required_capability: metadata_fields +required_capability: union_types_remove_fields FROM sample_data* METADATA _index | WHERE event_duration > 8000000 @@ -895,3 +975,43 @@ event_duration:long | _index:keyword | ts:date | ts_str:k 8268153 | sample_data_str | 2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015Z | 1698069175015 | 172.21.3.15 | 172.21.3.15 8268153 | sample_data_ts_long | 2023-10-23T13:52:55.015Z | 1698069175015 | 1698069175015 | 172.21.3.15 | 172.21.3.15 ; + + +inlineStatsUnionGroup +required_capability: union_types +required_capability: inlinestats + +FROM sample_data, sample_data_ts_long +| EVAL @timestamp = SUBSTRING(TO_STRING(@timestamp), 0, 7) +| INLINESTATS count = COUNT(*) BY @timestamp +| SORT client_ip ASC, @timestamp ASC +| LIMIT 4 +; + +client_ip:ip | event_duration:long | message:keyword | @timestamp:keyword | count:long + 172.21.0.5 | 1232382 | Disconnected | 1698068 | 1 + 172.21.0.5 | 1232382 | Disconnected | 2023-10 | 7 +172.21.2.113 | 2764889 | Connected to 10.1.0.2 | 1698064 | 1 +172.21.2.113 | 2764889 | Connected to 10.1.0.2 | 2023-10 | 7 + +; + +inlineStatsUnionGroupTogether +required_capability: union_types +required_capability: inlinestats + +FROM sample_data, sample_data_ts_long +| EVAL @timestamp = TO_STRING(TO_DATETIME(@timestamp)) +| INLINESTATS count = COUNT(*) BY @timestamp +| SORT client_ip ASC, @timestamp ASC +| LIMIT 4 +; + +client_ip:ip | event_duration:long | message:keyword | @timestamp:keyword | count:long + 172.21.0.5 | 1232382 | Disconnected | 2023-10-23T13:33:34.937Z | 2 + 172.21.0.5 | 1232382 | Disconnected | 2023-10-23T13:33:34.937Z | 2 +172.21.2.113 | 2764889 | Connected to 10.1.0.2 | 2023-10-23T12:27:28.948Z | 2 +172.21.2.113 | 2764889 | Connected to 10.1.0.2 | 2023-10-23T12:27:28.948Z | 2 +; + +# Once INLINESTATS supports expressions in agg functions and groups, convert the group in the inlinestats diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java index 93f8c75ddb088..77726ca9fdcce 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.junit.Before; import java.time.ZoneOffset; @@ -743,4 +744,47 @@ record RateKey(String cluster, String host) { assertThat((double) values.get(0).get(0), closeTo(rates.stream().mapToDouble(d -> 20. * d + 10.0 * Math.floor(d)).sum(), 0.1)); } } + + public void testIndexMode() { + createIndex("events"); + int numDocs = between(1, 10); + for (int i = 0; i < numDocs; i++) { + index("events", Integer.toString(i), Map.of("v", i)); + } + refresh("events"); + List columns = List.of( + new ColumnInfoImpl("_index", DataType.KEYWORD), + new ColumnInfoImpl("_index_mode", DataType.KEYWORD) + ); + try (EsqlQueryResponse resp = run(""" + FROM events,hosts METADATA _index_mode, _index + | WHERE _index_mode == "time_series" + | STATS BY _index, _index_mode + """)) { + assertThat(resp.columns(), equalTo(columns)); + List> values = EsqlTestUtils.getValuesList(resp); + assertThat(values, hasSize(1)); + assertThat(values, equalTo(List.of(List.of("hosts", "time_series")))); + } + try (EsqlQueryResponse resp = run(""" + FROM events,hosts METADATA _index_mode, _index + | WHERE _index_mode == "standard" + | STATS BY _index, _index_mode + """)) { + assertThat(resp.columns(), equalTo(columns)); + List> values = EsqlTestUtils.getValuesList(resp); + assertThat(values, hasSize(1)); + assertThat(values, equalTo(List.of(List.of("events", "standard")))); + } + try (EsqlQueryResponse resp = run(""" + FROM events,hosts METADATA _index_mode, _index + | STATS BY _index, _index_mode + | SORT _index + """)) { + assertThat(resp.columns(), equalTo(columns)); + List> values = EsqlTestUtils.getValuesList(resp); + assertThat(values, hasSize(2)); + assertThat(values, equalTo(List.of(List.of("events", "standard"), List.of("hosts", "time_series")))); + } + } } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownCartesianPointIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownCartesianPointIT.java new file mode 100644 index 0000000000000..1a31ba01d3e05 --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownCartesianPointIT.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.spatial; + +import org.elasticsearch.geo.ShapeTestUtils; +import org.elasticsearch.geometry.Geometry; + +public class SpatialPushDownCartesianPointIT extends SpatialPushDownTestCase { + + @Override + protected String fieldType() { + return "point"; + } + + @Override + protected Geometry getIndexGeometry() { + return ShapeTestUtils.randomPoint(); + } + + @Override + protected Geometry getQueryGeometry() { + return ShapeTestUtils.randomGeometry(false); + } + + @Override + protected String castingFunction() { + return "TO_CARTESIANSHAPE"; + } +} diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownCartesianShapeIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownCartesianShapeIT.java new file mode 100644 index 0000000000000..3ab7a0d516ed9 --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownCartesianShapeIT.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.spatial; + +import org.elasticsearch.geo.ShapeTestUtils; +import org.elasticsearch.geometry.Geometry; + +public class SpatialPushDownCartesianShapeIT extends SpatialPushDownTestCase { + + @Override + protected String fieldType() { + return "shape"; + } + + @Override + protected Geometry getIndexGeometry() { + return ShapeTestUtils.randomGeometryWithoutCircle(false); + } + + @Override + protected Geometry getQueryGeometry() { + return ShapeTestUtils.randomGeometry(false); + } + + @Override + protected String castingFunction() { + return "TO_CARTESIANSHAPE"; + } +} diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownGeoPointIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownGeoPointIT.java new file mode 100644 index 0000000000000..c4b22e0b55585 --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownGeoPointIT.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.spatial; + +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Geometry; + +public class SpatialPushDownGeoPointIT extends SpatialPushDownTestCase { + + @Override + protected String fieldType() { + return "geo_point"; + } + + @Override + protected Geometry getIndexGeometry() { + return GeometryTestUtils.randomPoint(); + } + + @Override + protected Geometry getQueryGeometry() { + return GeometryTestUtils.randomGeometry(false); + } + + @Override + protected String castingFunction() { + return "TO_GEOSHAPE"; + } +} diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownGeoShapeIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownGeoShapeIT.java new file mode 100644 index 0000000000000..3fa0385ea3681 --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownGeoShapeIT.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.spatial; + +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Geometry; + +public class SpatialPushDownGeoShapeIT extends SpatialPushDownTestCase { + + @Override + protected String fieldType() { + return "geo_shape"; + } + + @Override + protected Geometry getIndexGeometry() { + return GeometryTestUtils.randomGeometryWithoutCircle(0, false); + } + + @Override + protected Geometry getQueryGeometry() { + return GeometryTestUtils.randomGeometry(false); + } + + @Override + protected String castingFunction() { + return "TO_GEOSHAPE"; + } +} diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownTestCase.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownTestCase.java new file mode 100644 index 0000000000000..ead9d76f0b74d --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/spatial/SpatialPushDownTestCase.java @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.spatial; + +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.ShapeType; +import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xpack.core.esql.action.EsqlQueryRequestBuilder; +import org.elasticsearch.xpack.core.esql.action.EsqlQueryResponse; +import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; +import org.elasticsearch.xpack.spatial.SpatialPlugin; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Locale; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; + +/** + * Base class to check that a query than can be pushed down gives the same result + * if it is actually pushed down and when it is executed by the compute engine, + * + * For doing that we create two indices, one fully indexed and another with index + * and doc values disabled. Then we index the same data in both indices and we check + * that the same ES|QL queries produce the same results in both. + */ +public abstract class SpatialPushDownTestCase extends ESIntegTestCase { + + protected Collection> nodePlugins() { + return List.of(EsqlPlugin.class, SpatialPlugin.class); + } + + /** + * Elasticsearch field type + */ + protected abstract String fieldType(); + + /** + * A random {@link Geometry} to be indexed. + */ + protected abstract Geometry getIndexGeometry(); + + /** + * A random {@link Geometry} to be used for querying. + */ + protected abstract Geometry getQueryGeometry(); + + /** + * Necessary to build a ES|QL query. It should be "TO_GEOSHAPE" for geo + * fields and "TO_CARTESIANSHAPE" for cartesian fields. + */ + protected abstract String castingFunction(); + + public void testPushedDownQueriesSingleValue() throws RuntimeException { + assertPushedDownQueries(false); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/110830") + public void testPushedDownQueriesMultiValue() throws RuntimeException { + assertPushedDownQueries(true); + } + + private void assertPushedDownQueries(boolean multiValue) throws RuntimeException { + assertAcked(prepareCreate("indexed").setMapping(String.format(Locale.ROOT, """ + { + "properties" : { + "location": { "type" : "%s" } + } + } + """, fieldType()))); + + assertAcked(prepareCreate("not-indexed").setMapping(String.format(Locale.ROOT, """ + { + "properties" : { + "location": { "type" : "%s", "index" : false, "doc_values" : false } + } + } + """, fieldType()))); + for (int i = 0; i < random().nextInt(50, 100); i++) { + if (multiValue) { + final String[] values = new String[randomIntBetween(1, 5)]; + for (int j = 0; j < values.length; j++) { + values[j] = "\"" + WellKnownText.toWKT(getIndexGeometry()) + "\""; + } + index("indexed", i + "", "{\"location\" : " + Arrays.toString(values) + " }"); + index("not-indexed", i + "", "{\"location\" : " + Arrays.toString(values) + " }"); + } else { + final String value = WellKnownText.toWKT(getIndexGeometry()); + index("indexed", i + "", "{\"location\" : \"" + value + "\" }"); + index("not-indexed", i + "", "{\"location\" : \"" + value + "\" }"); + } + } + + refresh("indexed", "not-indexed"); + + for (int i = 0; i < 10; i++) { + final Geometry geometry = getQueryGeometry(); + final String wkt = WellKnownText.toWKT(geometry); + assertFunction("ST_INTERSECTS", wkt); + assertFunction("ST_DISJOINT", wkt); + assertFunction("ST_CONTAINS", wkt); + // within and lines are not globally supported so we avoid it here + if (containsLine(geometry) == false) { + assertFunction("ST_WITHIN", wkt); + } + } + } + + private void assertFunction(String spatialFunction, String wkt) { + final String query1 = String.format(Locale.ROOT, """ + FROM indexed | WHERE %s(location, %s("%s")) | STATS COUNT(*) + """, spatialFunction, castingFunction(), wkt); + final String query2 = String.format(Locale.ROOT, """ + FROM not-indexed | WHERE %s(location, %s("%s")) | STATS COUNT(*) + """, spatialFunction, castingFunction(), wkt); + try ( + EsqlQueryResponse response1 = EsqlQueryRequestBuilder.newRequestBuilder(client()).query(query1).get(); + EsqlQueryResponse response2 = EsqlQueryRequestBuilder.newRequestBuilder(client()).query(query2).get(); + ) { + assertEquals(response1.response().column(0).iterator().next(), response2.response().column(0).iterator().next()); + } + } + + private static boolean containsLine(Geometry geometry) { + if (geometry instanceof GeometryCollection collection) { + for (Geometry g : collection) { + if (containsLine(g)) { + return true; + } + } + return false; + } else { + return geometry.type() == ShapeType.LINESTRING || geometry.type() == ShapeType.MULTILINESTRING; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InBooleanEvaluator.java b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InBooleanEvaluator.java new file mode 100644 index 0000000000000..ad5a79f8f2176 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InBooleanEvaluator.java @@ -0,0 +1,194 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +import java.util.Arrays; +import java.util.BitSet; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link In}. + * This class is generated. Edit {@code InEvaluator.java.st} instead. + */ +public class InBooleanEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator[] rhs; + + private final DriverContext driverContext; + + public InBooleanEvaluator( + Source source, + EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator[] rhs, + DriverContext driverContext + ) { + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try (BooleanBlock lhsBlock = (BooleanBlock) lhs.eval(page)) { + BooleanBlock[] rhsBlocks = new BooleanBlock[rhs.length]; + try (Releasable rhsRelease = Releasables.wrap(rhsBlocks)) { + for (int i = 0; i < rhsBlocks.length; i++) { + rhsBlocks[i] = (BooleanBlock) rhs[i].eval(page); + } + BooleanVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + BooleanVector[] rhsVectors = new BooleanVector[rhs.length]; + for (int i = 0; i < rhsBlocks.length; i++) { + rhsVectors[i] = rhsBlocks[i].asVector(); + if (rhsVectors[i] == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + } + return eval(page.getPositionCount(), lhsVector, rhsVectors); + } + } + } + + private BooleanBlock eval(int positionCount, BooleanBlock lhsBlock, BooleanBlock[] rhsBlocks) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + boolean hasTrue = false; + boolean hasFalse = false; + BitSet nulls = new BitSet(rhs.length); + BitSet mvs = new BitSet(rhs.length); + boolean foundMatch; + for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue; + } + // unpack rhsBlocks into rhsValues + nulls.clear(); + mvs.clear(); + hasTrue = false; + hasFalse = false; + for (int i = 0; i < rhsBlocks.length; i++) { + if (rhsBlocks[i].isNull(p)) { + nulls.set(i); + continue; + } + if (rhsBlocks[i].getValueCount(p) > 1) { + mvs.set(i); + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + continue; + } + if (hasTrue && hasFalse) { + continue; + } + int o = rhsBlocks[i].getFirstValueIndex(p); + if (rhsBlocks[i].getBoolean(o)) { + hasTrue = true; + } else { + hasFalse = true; + } + } + if (nulls.cardinality() == rhsBlocks.length || mvs.cardinality() == rhsBlocks.length) { + result.appendNull(); + continue; + } + foundMatch = lhsBlock.getBoolean(lhsBlock.getFirstValueIndex(p)) ? hasTrue : hasFalse; + if (foundMatch) { + result.appendBoolean(true); + } else { + if (nulls.cardinality() > 0) { + result.appendNull(); + } else { + result.appendBoolean(false); + } + } + } + return result.build(); + } + } + + private BooleanBlock eval(int positionCount, BooleanVector lhsVector, BooleanVector[] rhsVectors) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + boolean hasTrue = false; + boolean hasFalse = false; + for (int p = 0; p < positionCount; p++) { + // unpack rhsVectors into rhsValues + hasTrue = false; + hasFalse = false; + for (int i = 0; i < rhsVectors.length; i++) { + if (hasTrue && hasFalse) { + continue; + } + if (rhsVectors[i].getBoolean(p)) { + hasTrue = true; + } else { + hasFalse = true; + } + } + result.appendBoolean(lhsVector.getBoolean(p) ? hasTrue : hasFalse); + } + return result.build(); + } + } + + @Override + public String toString() { + return "InBooleanEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, () -> Releasables.close(rhs)); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + private final EvalOperator.ExpressionEvaluator.Factory lhs; + private final EvalOperator.ExpressionEvaluator.Factory[] rhs; + + Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, EvalOperator.ExpressionEvaluator.Factory[] rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public InBooleanEvaluator get(DriverContext context) { + EvalOperator.ExpressionEvaluator[] rhs = Arrays.stream(this.rhs) + .map(a -> a.get(context)) + .toArray(EvalOperator.ExpressionEvaluator[]::new); + return new InBooleanEvaluator(source, lhs.get(context), rhs, context); + } + + @Override + public String toString() { + return "InBooleanEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InBytesRefEvaluator.java b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InBytesRefEvaluator.java new file mode 100644 index 0000000000000..f47a6e9cc5693 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InBytesRefEvaluator.java @@ -0,0 +1,186 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +import java.util.Arrays; +import java.util.BitSet; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link In}. + * This class is generated. Edit {@code InEvaluator.java.st} instead. + */ +public class InBytesRefEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator[] rhs; + + private final DriverContext driverContext; + + public InBytesRefEvaluator( + Source source, + EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator[] rhs, + DriverContext driverContext + ) { + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try (BytesRefBlock lhsBlock = (BytesRefBlock) lhs.eval(page)) { + BytesRefBlock[] rhsBlocks = new BytesRefBlock[rhs.length]; + try (Releasable rhsRelease = Releasables.wrap(rhsBlocks)) { + for (int i = 0; i < rhsBlocks.length; i++) { + rhsBlocks[i] = (BytesRefBlock) rhs[i].eval(page); + } + BytesRefVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + BytesRefVector[] rhsVectors = new BytesRefVector[rhs.length]; + for (int i = 0; i < rhsBlocks.length; i++) { + rhsVectors[i] = rhsBlocks[i].asVector(); + if (rhsVectors[i] == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + } + return eval(page.getPositionCount(), lhsVector, rhsVectors); + } + } + } + + private BooleanBlock eval(int positionCount, BytesRefBlock lhsBlock, BytesRefBlock[] rhsBlocks) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + BytesRef[] rhsValues = new BytesRef[rhs.length]; + BytesRef lhsScratch = new BytesRef(); + BytesRef[] rhsScratch = new BytesRef[rhs.length]; + for (int i = 0; i < rhs.length; i++) { + rhsScratch[i] = new BytesRef(); + } + BitSet nulls = new BitSet(rhs.length); + BitSet mvs = new BitSet(rhs.length); + boolean foundMatch; + for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue; + } + // unpack rhsBlocks into rhsValues + nulls.clear(); + mvs.clear(); + for (int i = 0; i < rhsBlocks.length; i++) { + if (rhsBlocks[i].isNull(p)) { + nulls.set(i); + continue; + } + if (rhsBlocks[i].getValueCount(p) > 1) { + mvs.set(i); + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + continue; + } + int o = rhsBlocks[i].getFirstValueIndex(p); + rhsValues[i] = rhsBlocks[i].getBytesRef(o, rhsScratch[i]); + } + if (nulls.cardinality() == rhsBlocks.length || mvs.cardinality() == rhsBlocks.length) { + result.appendNull(); + continue; + } + foundMatch = In.process(nulls, mvs, lhsBlock.getBytesRef(lhsBlock.getFirstValueIndex(p), lhsScratch), rhsValues); + if (foundMatch) { + result.appendBoolean(true); + } else { + if (nulls.cardinality() > 0) { + result.appendNull(); + } else { + result.appendBoolean(false); + } + } + } + return result.build(); + } + } + + private BooleanBlock eval(int positionCount, BytesRefVector lhsVector, BytesRefVector[] rhsVectors) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + BytesRef[] rhsValues = new BytesRef[rhs.length]; + BytesRef lhsScratch = new BytesRef(); + BytesRef[] rhsScratch = new BytesRef[rhs.length]; + for (int i = 0; i < rhs.length; i++) { + rhsScratch[i] = new BytesRef(); + } + for (int p = 0; p < positionCount; p++) { + // unpack rhsVectors into rhsValues + for (int i = 0; i < rhsVectors.length; i++) { + rhsValues[i] = rhsVectors[i].getBytesRef(p, rhsScratch[i]); + } + result.appendBoolean(In.process(null, null, lhsVector.getBytesRef(p, lhsScratch), rhsValues)); + } + return result.build(); + } + } + + @Override + public String toString() { + return "InBytesRefEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, () -> Releasables.close(rhs)); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + private final EvalOperator.ExpressionEvaluator.Factory lhs; + private final EvalOperator.ExpressionEvaluator.Factory[] rhs; + + Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, EvalOperator.ExpressionEvaluator.Factory[] rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public InBytesRefEvaluator get(DriverContext context) { + EvalOperator.ExpressionEvaluator[] rhs = Arrays.stream(this.rhs) + .map(a -> a.get(context)) + .toArray(EvalOperator.ExpressionEvaluator[]::new); + return new InBytesRefEvaluator(source, lhs.get(context), rhs, context); + } + + @Override + public String toString() { + return "InBytesRefEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InDoubleEvaluator.java new file mode 100644 index 0000000000000..f03dbb4a15a4c --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InDoubleEvaluator.java @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.DoubleVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +import java.util.Arrays; +import java.util.BitSet; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link In}. + * This class is generated. Edit {@code InEvaluator.java.st} instead. + */ +public class InDoubleEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator[] rhs; + + private final DriverContext driverContext; + + public InDoubleEvaluator( + Source source, + EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator[] rhs, + DriverContext driverContext + ) { + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try (DoubleBlock lhsBlock = (DoubleBlock) lhs.eval(page)) { + DoubleBlock[] rhsBlocks = new DoubleBlock[rhs.length]; + try (Releasable rhsRelease = Releasables.wrap(rhsBlocks)) { + for (int i = 0; i < rhsBlocks.length; i++) { + rhsBlocks[i] = (DoubleBlock) rhs[i].eval(page); + } + DoubleVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + DoubleVector[] rhsVectors = new DoubleVector[rhs.length]; + for (int i = 0; i < rhsBlocks.length; i++) { + rhsVectors[i] = rhsBlocks[i].asVector(); + if (rhsVectors[i] == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + } + return eval(page.getPositionCount(), lhsVector, rhsVectors); + } + } + } + + private BooleanBlock eval(int positionCount, DoubleBlock lhsBlock, DoubleBlock[] rhsBlocks) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + double[] rhsValues = new double[rhs.length]; + BitSet nulls = new BitSet(rhs.length); + BitSet mvs = new BitSet(rhs.length); + boolean foundMatch; + for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue; + } + // unpack rhsBlocks into rhsValues + nulls.clear(); + mvs.clear(); + for (int i = 0; i < rhsBlocks.length; i++) { + if (rhsBlocks[i].isNull(p)) { + nulls.set(i); + continue; + } + if (rhsBlocks[i].getValueCount(p) > 1) { + mvs.set(i); + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + continue; + } + int o = rhsBlocks[i].getFirstValueIndex(p); + rhsValues[i] = rhsBlocks[i].getDouble(o); + } + if (nulls.cardinality() == rhsBlocks.length || mvs.cardinality() == rhsBlocks.length) { + result.appendNull(); + continue; + } + foundMatch = In.process(nulls, mvs, lhsBlock.getDouble(lhsBlock.getFirstValueIndex(p)), rhsValues); + if (foundMatch) { + result.appendBoolean(true); + } else { + if (nulls.cardinality() > 0) { + result.appendNull(); + } else { + result.appendBoolean(false); + } + } + } + return result.build(); + } + } + + private BooleanBlock eval(int positionCount, DoubleVector lhsVector, DoubleVector[] rhsVectors) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + double[] rhsValues = new double[rhs.length]; + for (int p = 0; p < positionCount; p++) { + // unpack rhsVectors into rhsValues + for (int i = 0; i < rhsVectors.length; i++) { + rhsValues[i] = rhsVectors[i].getDouble(p); + } + result.appendBoolean(In.process(null, null, lhsVector.getDouble(p), rhsValues)); + } + return result.build(); + } + } + + @Override + public String toString() { + return "InDoubleEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, () -> Releasables.close(rhs)); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + private final EvalOperator.ExpressionEvaluator.Factory lhs; + private final EvalOperator.ExpressionEvaluator.Factory[] rhs; + + Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, EvalOperator.ExpressionEvaluator.Factory[] rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public InDoubleEvaluator get(DriverContext context) { + EvalOperator.ExpressionEvaluator[] rhs = Arrays.stream(this.rhs) + .map(a -> a.get(context)) + .toArray(EvalOperator.ExpressionEvaluator[]::new); + return new InDoubleEvaluator(source, lhs.get(context), rhs, context); + } + + @Override + public String toString() { + return "InDoubleEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InIntEvaluator.java b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InIntEvaluator.java new file mode 100644 index 0000000000000..73f4a02d8dcd0 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InIntEvaluator.java @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +import java.util.Arrays; +import java.util.BitSet; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link In}. + * This class is generated. Edit {@code InEvaluator.java.st} instead. + */ +public class InIntEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator[] rhs; + + private final DriverContext driverContext; + + public InIntEvaluator( + Source source, + EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator[] rhs, + DriverContext driverContext + ) { + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try (IntBlock lhsBlock = (IntBlock) lhs.eval(page)) { + IntBlock[] rhsBlocks = new IntBlock[rhs.length]; + try (Releasable rhsRelease = Releasables.wrap(rhsBlocks)) { + for (int i = 0; i < rhsBlocks.length; i++) { + rhsBlocks[i] = (IntBlock) rhs[i].eval(page); + } + IntVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + IntVector[] rhsVectors = new IntVector[rhs.length]; + for (int i = 0; i < rhsBlocks.length; i++) { + rhsVectors[i] = rhsBlocks[i].asVector(); + if (rhsVectors[i] == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + } + return eval(page.getPositionCount(), lhsVector, rhsVectors); + } + } + } + + private BooleanBlock eval(int positionCount, IntBlock lhsBlock, IntBlock[] rhsBlocks) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + int[] rhsValues = new int[rhs.length]; + BitSet nulls = new BitSet(rhs.length); + BitSet mvs = new BitSet(rhs.length); + boolean foundMatch; + for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue; + } + // unpack rhsBlocks into rhsValues + nulls.clear(); + mvs.clear(); + for (int i = 0; i < rhsBlocks.length; i++) { + if (rhsBlocks[i].isNull(p)) { + nulls.set(i); + continue; + } + if (rhsBlocks[i].getValueCount(p) > 1) { + mvs.set(i); + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + continue; + } + int o = rhsBlocks[i].getFirstValueIndex(p); + rhsValues[i] = rhsBlocks[i].getInt(o); + } + if (nulls.cardinality() == rhsBlocks.length || mvs.cardinality() == rhsBlocks.length) { + result.appendNull(); + continue; + } + foundMatch = In.process(nulls, mvs, lhsBlock.getInt(lhsBlock.getFirstValueIndex(p)), rhsValues); + if (foundMatch) { + result.appendBoolean(true); + } else { + if (nulls.cardinality() > 0) { + result.appendNull(); + } else { + result.appendBoolean(false); + } + } + } + return result.build(); + } + } + + private BooleanBlock eval(int positionCount, IntVector lhsVector, IntVector[] rhsVectors) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + int[] rhsValues = new int[rhs.length]; + for (int p = 0; p < positionCount; p++) { + // unpack rhsVectors into rhsValues + for (int i = 0; i < rhsVectors.length; i++) { + rhsValues[i] = rhsVectors[i].getInt(p); + } + result.appendBoolean(In.process(null, null, lhsVector.getInt(p), rhsValues)); + } + return result.build(); + } + } + + @Override + public String toString() { + return "InIntEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, () -> Releasables.close(rhs)); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + private final EvalOperator.ExpressionEvaluator.Factory lhs; + private final EvalOperator.ExpressionEvaluator.Factory[] rhs; + + Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, EvalOperator.ExpressionEvaluator.Factory[] rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public InIntEvaluator get(DriverContext context) { + EvalOperator.ExpressionEvaluator[] rhs = Arrays.stream(this.rhs) + .map(a -> a.get(context)) + .toArray(EvalOperator.ExpressionEvaluator[]::new); + return new InIntEvaluator(source, lhs.get(context), rhs, context); + } + + @Override + public String toString() { + return "InIntEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InLongEvaluator.java b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InLongEvaluator.java new file mode 100644 index 0000000000000..966ebe7541411 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InLongEvaluator.java @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +import java.util.Arrays; +import java.util.BitSet; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link In}. + * This class is generated. Edit {@code InEvaluator.java.st} instead. + */ +public class InLongEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator[] rhs; + + private final DriverContext driverContext; + + public InLongEvaluator( + Source source, + EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator[] rhs, + DriverContext driverContext + ) { + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + LongBlock[] rhsBlocks = new LongBlock[rhs.length]; + try (Releasable rhsRelease = Releasables.wrap(rhsBlocks)) { + for (int i = 0; i < rhsBlocks.length; i++) { + rhsBlocks[i] = (LongBlock) rhs[i].eval(page); + } + LongVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + LongVector[] rhsVectors = new LongVector[rhs.length]; + for (int i = 0; i < rhsBlocks.length; i++) { + rhsVectors[i] = rhsBlocks[i].asVector(); + if (rhsVectors[i] == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + } + return eval(page.getPositionCount(), lhsVector, rhsVectors); + } + } + } + + private BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock[] rhsBlocks) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + long[] rhsValues = new long[rhs.length]; + BitSet nulls = new BitSet(rhs.length); + BitSet mvs = new BitSet(rhs.length); + boolean foundMatch; + for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue; + } + // unpack rhsBlocks into rhsValues + nulls.clear(); + mvs.clear(); + for (int i = 0; i < rhsBlocks.length; i++) { + if (rhsBlocks[i].isNull(p)) { + nulls.set(i); + continue; + } + if (rhsBlocks[i].getValueCount(p) > 1) { + mvs.set(i); + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + continue; + } + int o = rhsBlocks[i].getFirstValueIndex(p); + rhsValues[i] = rhsBlocks[i].getLong(o); + } + if (nulls.cardinality() == rhsBlocks.length || mvs.cardinality() == rhsBlocks.length) { + result.appendNull(); + continue; + } + foundMatch = In.process(nulls, mvs, lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsValues); + if (foundMatch) { + result.appendBoolean(true); + } else { + if (nulls.cardinality() > 0) { + result.appendNull(); + } else { + result.appendBoolean(false); + } + } + } + return result.build(); + } + } + + private BooleanBlock eval(int positionCount, LongVector lhsVector, LongVector[] rhsVectors) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + long[] rhsValues = new long[rhs.length]; + for (int p = 0; p < positionCount; p++) { + // unpack rhsVectors into rhsValues + for (int i = 0; i < rhsVectors.length; i++) { + rhsValues[i] = rhsVectors[i].getLong(p); + } + result.appendBoolean(In.process(null, null, lhsVector.getLong(p), rhsValues)); + } + return result.build(); + } + } + + @Override + public String toString() { + return "InLongEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, () -> Releasables.close(rhs)); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + private final EvalOperator.ExpressionEvaluator.Factory lhs; + private final EvalOperator.ExpressionEvaluator.Factory[] rhs; + + Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, EvalOperator.ExpressionEvaluator.Factory[] rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public InLongEvaluator get(DriverContext context) { + EvalOperator.ExpressionEvaluator[] rhs = Arrays.stream(this.rhs) + .map(a -> a.get(context)) + .toArray(EvalOperator.ExpressionEvaluator[]::new); + return new InLongEvaluator(source, lhs.get(context), rhs, context); + } + + @Override + public String toString() { + return "InLongEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/Column.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/Column.java index a19dafba1559b..6287bf54ce5b0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/Column.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/Column.java @@ -15,7 +15,6 @@ import org.elasticsearch.core.Releasables; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.planner.PlannerUtils; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; @@ -28,7 +27,7 @@ public record Column(DataType type, Block values) implements Releasable, Writeab } public Column(BlockStreamInput in) throws IOException { - this(EsqlDataTypes.fromTypeName(in.readString()), in.readNamedWriteable(Block.class)); + this(DataType.fromTypeName(in.readString()), in.readNamedWriteable(Block.class)); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 3353a9352a4bb..e6142e8161d44 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -44,6 +44,11 @@ public enum Cap { */ FN_SUBSTRING_EMPTY_NULL, + /** + * Support for the {@code INLINESTATS} syntax. + */ + INLINESTATS(true), + /** * Support for aggregation function {@code TOP}. */ @@ -54,6 +59,21 @@ public enum Cap { */ AGG_MAX_MIN_BOOLEAN_SUPPORT, + /** + * Support for ips in aggregations {@code MAX} and {@code MIN}. + */ + AGG_MAX_MIN_IP_SUPPORT, + + /** + * Support for booleans in {@code TOP} aggregation. + */ + AGG_TOP_BOOLEAN_SUPPORT, + + /** + * Support for ips in {@code TOP} aggregation. + */ + AGG_TOP_IP_SUPPORT, + /** * Optimization for ST_CENTROID changed some results in cartesian data. #108713 */ @@ -127,7 +147,41 @@ public enum Cap { /** * Fix for union-types when aggregating over an inline conversion with conversion function. Done in #110652. */ - UNION_TYPES_INLINE_FIX; + UNION_TYPES_INLINE_FIX, + + /** + * Fix for union-types when sorting a type-casted field. We changed how we remove synthetic union-types fields. + */ + UNION_TYPES_REMOVE_FIELDS, + + /** + * Fix a parsing issue where numbers below Long.MIN_VALUE threw an exception instead of parsing as doubles. + * see Parsing large numbers is inconsistent #104323 + */ + FIX_PARSING_LARGE_NEGATIVE_NUMBERS, + + /** + * Fix the status code returned when trying to run count_distinct on the _source type (which is not supported). + * see count_distinct(_source) returns a 500 response + */ + FIX_COUNT_DISTINCT_SOURCE_ERROR, + + /** + * Use RangeQuery for BinaryComparison on DateTime fields. + */ + RANGEQUERY_FOR_DATETIME, + + /** + * Fix for non-unique attribute names in ROW and logical plans. + * https://github.com/elastic/elasticsearch/issues/110541 + */ + UNIQUE_NAMES, + + /** + * Make attributes of GROK/DISSECT adjustable and fix a shadowing bug when pushing them down past PROJECT. + * https://github.com/elastic/elasticsearch/issues/108008 + */ + FIXED_PUSHDOWN_PAST_PROJECT; private final boolean snapshotOnly; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsAction.java new file mode 100644 index 0000000000000..6bb52fc8c0aaf --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsAction.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.RemoteClusterActionType; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.action.fieldcaps.TransportFieldCapabilitiesAction; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; + +/** + * A fork of the field-caps API for ES|QL. This fork allows us to gradually introduce features and optimizations to this internal + * API without risking breaking the external field-caps API. For now, this API delegates to the field-caps API, but gradually, + * we will decouple this API completely from the field-caps. + */ +public class EsqlResolveFieldsAction extends HandledTransportAction { + public static final String NAME = "indices:data/read/esql/resolve_fields"; + public static final ActionType TYPE = new ActionType<>(NAME); + public static final RemoteClusterActionType REMOTE_TYPE = new RemoteClusterActionType<>( + NAME, + FieldCapabilitiesResponse::new + ); + + private final TransportFieldCapabilitiesAction fieldCapsAction; + + @Inject + public EsqlResolveFieldsAction( + TransportService transportService, + ActionFilters actionFilters, + TransportFieldCapabilitiesAction fieldCapsAction + ) { + // TODO replace DIRECT_EXECUTOR_SERVICE when removing workaround for https://github.com/elastic/elasticsearch/issues/97916 + super(NAME, transportService, actionFilters, FieldCapabilitiesRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); + this.fieldCapsAction = fieldCapsAction; + } + + @Override + protected void doExecute(Task task, FieldCapabilitiesRequest request, final ActionListener listener) { + fieldCapsAction.executeRequest(task, request, REMOTE_TYPE, listener); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java index 290a816275a29..67c6e1f48a47a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java @@ -9,11 +9,9 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.compute.data.Block; -import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.DoubleBlock; @@ -21,30 +19,21 @@ import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.lucene.UnsupportedValueSource; -import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; -import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.planner.PlannerUtils; import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.Map; import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToLong; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.longToUnsignedLong; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.spatialToString; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToIP; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToVersion; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.versionToString; /** @@ -155,55 +144,4 @@ private static Object valueAt(DataType dataType, Block block, int offset, BytesR NULL, PARTIAL_AGG -> throw EsqlIllegalArgumentException.illegalDataType(dataType); }; } - - /** - * Converts a list of values to Pages so that we can parse from xcontent. It's not - * super efficient, but it doesn't really have to be. - */ - static Page valuesToPage(BlockFactory blockFactory, List columns, List> values) { - List dataTypes = columns.stream().map(ColumnInfoImpl::type).toList(); - List results = dataTypes.stream() - .map(c -> PlannerUtils.toElementType(c).newBlockBuilder(values.size(), blockFactory)) - .toList(); - - for (List row : values) { - for (int c = 0; c < row.size(); c++) { - var builder = results.get(c); - var value = row.get(c); - switch (dataTypes.get(c)) { - case UNSIGNED_LONG -> ((LongBlock.Builder) builder).appendLong(longToUnsignedLong(((Number) value).longValue(), true)); - case LONG, COUNTER_LONG -> ((LongBlock.Builder) builder).appendLong(((Number) value).longValue()); - case INTEGER, COUNTER_INTEGER -> ((IntBlock.Builder) builder).appendInt(((Number) value).intValue()); - case DOUBLE, COUNTER_DOUBLE -> ((DoubleBlock.Builder) builder).appendDouble(((Number) value).doubleValue()); - case KEYWORD, TEXT, UNSUPPORTED -> ((BytesRefBlock.Builder) builder).appendBytesRef(new BytesRef(value.toString())); - case IP -> ((BytesRefBlock.Builder) builder).appendBytesRef(stringToIP(value.toString())); - case DATETIME -> { - long longVal = dateTimeToLong(value.toString()); - ((LongBlock.Builder) builder).appendLong(longVal); - } - case BOOLEAN -> ((BooleanBlock.Builder) builder).appendBoolean(((Boolean) value)); - case NULL -> builder.appendNull(); - case VERSION -> ((BytesRefBlock.Builder) builder).appendBytesRef(stringToVersion(new BytesRef(value.toString()))); - case SOURCE -> { - @SuppressWarnings("unchecked") - Map o = (Map) value; - try { - try (XContentBuilder sourceBuilder = JsonXContent.contentBuilder()) { - sourceBuilder.map(o); - ((BytesRefBlock.Builder) builder).appendBytesRef(BytesReference.bytes(sourceBuilder).toBytesRef()); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - case GEO_POINT, GEO_SHAPE, CARTESIAN_POINT, CARTESIAN_SHAPE -> { - // This just converts WKT to WKB, so does not need CRS knowledge, we could merge GEO and CARTESIAN here - BytesRef wkb = stringToSpatial(value.toString()); - ((BytesRefBlock.Builder) builder).appendBytesRef(wkb); - } - } - } - } - return new Page(results.stream().map(Block.Builder::build).toArray(Block[]::new)); - } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index add1f74cc3f04..75e494fe9671e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.compute.data.Block; +import org.elasticsearch.logging.Logger; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.esql.Column; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; @@ -39,6 +40,7 @@ import org.elasticsearch.xpack.esql.core.plan.TableIdentifier; import org.elasticsearch.xpack.esql.core.rule.ParameterizedRule; import org.elasticsearch.xpack.esql.core.rule.ParameterizedRuleExecutor; +import org.elasticsearch.xpack.esql.core.rule.Rule; import org.elasticsearch.xpack.esql.core.rule.RuleExecutor; import org.elasticsearch.xpack.esql.core.session.Configuration; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -60,12 +62,11 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.DateTimeArithmeticOperation; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; +import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Drop; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; -import org.elasticsearch.xpack.esql.plan.logical.EsqlAggregate; -import org.elasticsearch.xpack.esql.plan.logical.EsqlUnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Keep; import org.elasticsearch.xpack.esql.plan.logical.Limit; @@ -74,12 +75,13 @@ import org.elasticsearch.xpack.esql.plan.logical.MvExpand; import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.Rename; +import org.elasticsearch.xpack.esql.plan.logical.Stats; +import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject; import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier; import org.elasticsearch.xpack.esql.stats.FeatureMetric; import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.esql.type.MultiTypeEsField; import java.util.ArrayList; @@ -114,9 +116,12 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.NESTED; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; +import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount; import static org.elasticsearch.xpack.esql.stats.FeatureMetric.LIMIT; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isTemporalAmount; +/** + * This class is part of the planner. Resolves references (such as variable and index names) and performs implicit casting. + */ public class Analyzer extends ParameterizedRuleExecutor { // marker list of attributes for plans that do not have any concrete fields to return, but have other computed columns to return // ie from test | stats c = count(*) @@ -140,7 +145,7 @@ public class Analyzer extends ParameterizedRuleExecutor("Finish Analysis", Limiter.ONCE, new AddImplicitLimit(), new UnresolveUnionTypes()); + var finish = new Batch<>("Finish Analysis", Limiter.ONCE, new AddImplicitLimit(), new UnionTypesCleanup()); rules = List.of(init, resolution, finish); } @@ -169,16 +174,17 @@ protected Iterable> batches() { return rules; } - private static class ResolveTable extends ParameterizedAnalyzerRule { + private static class ResolveTable extends ParameterizedAnalyzerRule { @Override - protected LogicalPlan rule(EsqlUnresolvedRelation plan, AnalyzerContext context) { + protected LogicalPlan rule(UnresolvedRelation plan, AnalyzerContext context) { if (context.indexResolution().isValid() == false) { return plan.unresolvedMessage().equals(context.indexResolution().toString()) ? plan - : new EsqlUnresolvedRelation( + : new UnresolvedRelation( plan.source(), plan.table(), + plan.frozen(), plan.metadataFields(), plan.indexMode(), context.indexResolution().toString() @@ -187,9 +193,10 @@ protected LogicalPlan rule(EsqlUnresolvedRelation plan, AnalyzerContext context) TableIdentifier table = plan.table(); if (context.indexResolution().matches(table.index()) == false) { // TODO: fix this (and tests), or drop check (seems SQL-inherited, where's also defective) - new EsqlUnresolvedRelation( + new UnresolvedRelation( plan.source(), plan.table(), + plan.frozen(), plan.metadataFields(), plan.indexMode(), "invalid [" + table + "] resolution to [" + context.indexResolution() + "]" @@ -216,13 +223,13 @@ private static List mappingAsAttributes(Source source, Map list, Source source, String parentName, Map mapping) { + private static void mappingAsAttributes(List list, Source source, FieldAttribute parent, Map mapping) { for (Map.Entry entry : mapping.entrySet()) { String name = entry.getKey(); EsField t = entry.getValue(); if (t != null) { - name = parentName == null ? name : parentName + "." + name; + name = parent == null ? name : parent.fieldName() + "." + name; var fieldProperties = t.getProperties(); var type = t.getDataType().widenSmallNumeric(); // due to a bug also copy the field since the Attribute hierarchy extracts the data type @@ -231,19 +238,16 @@ private static void mappingAsAttributes(List list, Source source, Str t = new EsField(t.getName(), type, t.getProperties(), t.isAggregatable(), t.isAlias()); } + FieldAttribute attribute = t instanceof UnsupportedEsField uef + ? new UnsupportedAttribute(source, name, uef) + : new FieldAttribute(source, parent, name, t); // primitive branch - if (EsqlDataTypes.isPrimitive(type)) { - Attribute attribute; - if (t instanceof UnsupportedEsField uef) { - attribute = new UnsupportedAttribute(source, name, uef); - } else { - attribute = new FieldAttribute(source, null, name, t); - } + if (DataType.isPrimitive(type)) { list.add(attribute); } // allow compound object even if they are unknown (but not NESTED) if (type != NESTED && fieldProperties.isEmpty() == false) { - mappingAsAttributes(list, source, name, fieldProperties); + mappingAsAttributes(list, source, attribute, fieldProperties); } } } @@ -383,7 +387,7 @@ private LocalRelation tableMapAsRelation(Source source, Map mapT } } - private static class ResolveRefs extends BaseAnalyzerRule { + public static class ResolveRefs extends BaseAnalyzerRule { @Override protected LogicalPlan doRule(LogicalPlan plan) { if (plan.childrenResolved() == false) { @@ -396,8 +400,8 @@ protected LogicalPlan doRule(LogicalPlan plan) { childrenOutput.addAll(output); } - if (plan instanceof Aggregate agg) { - return resolveAggregate(agg, childrenOutput); + if (plan instanceof Stats stats) { + return resolveStats(stats, childrenOutput); } if (plan instanceof Drop d) { @@ -431,11 +435,11 @@ protected LogicalPlan doRule(LogicalPlan plan) { return plan.transformExpressionsOnly(UnresolvedAttribute.class, ua -> maybeResolveAttribute(ua, childrenOutput)); } - private LogicalPlan resolveAggregate(Aggregate a, List childrenOutput) { + private LogicalPlan resolveStats(Stats stats, List childrenOutput) { // if the grouping is resolved but the aggs are not, use the former to resolve the latter // e.g. STATS a ... GROUP BY a = x + 1 Holder changed = new Holder<>(false); - List groupings = a.groupings(); + List groupings = stats.groupings(); // first resolve groupings since the aggs might refer to them // trying to globally resolve unresolved attributes will lead to some being marked as unresolvable if (Resolvables.resolved(groupings) == false) { @@ -449,12 +453,12 @@ private LogicalPlan resolveAggregate(Aggregate a, List childrenOutput } groupings = newGroupings; if (changed.get()) { - a = new EsqlAggregate(a.source(), a.child(), a.aggregateType(), newGroupings, a.aggregates()); + stats = stats.with(newGroupings, stats.aggregates()); changed.set(false); } } - if (a.expressionsResolved() == false) { + if (stats.expressionsResolved() == false) { AttributeMap resolved = new AttributeMap<>(); for (Expression e : groupings) { Attribute attr = Expressions.attribute(e); @@ -465,7 +469,7 @@ private LogicalPlan resolveAggregate(Aggregate a, List childrenOutput List resolvedList = NamedExpressions.mergeOutputAttributes(new ArrayList<>(resolved.keySet()), childrenOutput); List newAggregates = new ArrayList<>(); - for (NamedExpression aggregate : a.aggregates()) { + for (NamedExpression aggregate : stats.aggregates()) { var agg = (NamedExpression) aggregate.transformUp(UnresolvedAttribute.class, ua -> { Expression ne = ua; Attribute maybeResolved = maybeResolveAttribute(ua, resolvedList); @@ -478,10 +482,10 @@ private LogicalPlan resolveAggregate(Aggregate a, List childrenOutput newAggregates.add(agg); } - a = changed.get() ? new EsqlAggregate(a.source(), a.child(), a.aggregateType(), groupings, newAggregates) : a; + stats = changed.get() ? stats.with(groupings, newAggregates) : stats; } - return a; + return (LogicalPlan) stats; } private LogicalPlan resolveMvExpand(MvExpand p, List childrenOutput) { @@ -575,20 +579,28 @@ private LogicalPlan resolveLookup(Lookup l, List childrenOutput) { } private Attribute maybeResolveAttribute(UnresolvedAttribute ua, List childrenOutput) { + return maybeResolveAttribute(ua, childrenOutput, log); + } + + private static Attribute maybeResolveAttribute(UnresolvedAttribute ua, List childrenOutput, Logger logger) { if (ua.customMessage()) { return ua; } - return resolveAttribute(ua, childrenOutput); + return resolveAttribute(ua, childrenOutput, logger); } private Attribute resolveAttribute(UnresolvedAttribute ua, List childrenOutput) { + return resolveAttribute(ua, childrenOutput, log); + } + + private static Attribute resolveAttribute(UnresolvedAttribute ua, List childrenOutput, Logger logger) { Attribute resolved = ua; var named = resolveAgainstList(ua, childrenOutput); // if resolved, return it; otherwise keep it in place to be resolved later if (named.size() == 1) { resolved = named.get(0); - if (log.isTraceEnabled() && resolved.resolved()) { - log.trace("Resolved {} to {}", ua, resolved); + if (logger != null && logger.isTraceEnabled() && resolved.resolved()) { + logger.trace("Resolved {} to {}", ua, resolved); } } else { if (named.size() > 0) { @@ -724,6 +736,12 @@ private LogicalPlan resolveDrop(Drop drop, List childOutput) { } private LogicalPlan resolveRename(Rename rename, List childrenOutput) { + List projections = projectionsForRename(rename, childrenOutput, log); + + return new EsqlProject(rename.source(), rename.child(), projections); + } + + public static List projectionsForRename(Rename rename, List childrenOutput, Logger logger) { List projections = new ArrayList<>(childrenOutput); int renamingsCount = rename.renamings().size(); @@ -736,7 +754,7 @@ private LogicalPlan resolveRename(Rename rename, List childrenOutput) // remove attributes overwritten by a renaming: `| keep a, b, c | rename a as b` projections.removeIf(x -> x.name().equals(alias.name())); - var resolved = maybeResolveAttribute(ua, childrenOutput); + var resolved = maybeResolveAttribute(ua, childrenOutput, logger); if (resolved instanceof UnsupportedAttribute || resolved.resolved()) { var realiased = (NamedExpression) alias.replaceChildren(List.of(resolved)); projections.replaceAll(x -> x.equals(resolved) ? realiased : x); @@ -779,7 +797,7 @@ private LogicalPlan resolveRename(Rename rename, List childrenOutput) // add unresolved renamings to later trip the Verifier. projections.addAll(unresolved); - return new EsqlProject(rename.source(), rename.child(), projections); + return projections; } private LogicalPlan resolveEnrich(Enrich enrich, List childrenOutput) { @@ -845,7 +863,7 @@ private static List potentialCandidatesIfNoMatchesFound( Set names = new HashSet<>(attrList.size()); for (var a : attrList) { String nameCandidate = a.name(); - if (EsqlDataTypes.isPrimitive(a.dataType())) { + if (DataType.isPrimitive(a.dataType())) { names.add(nameCandidate); } } @@ -928,15 +946,15 @@ public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { } private static Expression cast(ScalarFunction f, EsqlFunctionRegistry registry) { + if (f instanceof In in) { + return processIn(in); + } if (f instanceof EsqlScalarFunction esf) { return processScalarFunction(esf, registry); } if (f instanceof EsqlArithmeticOperation || f instanceof BinaryComparison) { return processBinaryOperator((BinaryOperator) f); } - if (f instanceof In in) { - return processIn(in); - } return f; } @@ -1068,13 +1086,29 @@ public static Expression castStringLiteral(Expression from, DataType target) { * Any fields which could not be resolved by conversion functions will be converted to UnresolvedAttribute instances in a later rule * (See UnresolveUnionTypes below). */ - private static class ResolveUnionTypes extends BaseAnalyzerRule { + private static class ResolveUnionTypes extends Rule { record TypeResolutionKey(String fieldName, DataType fieldType) {} + private List unionFieldAttributes; + @Override - protected LogicalPlan doRule(LogicalPlan plan) { - List unionFieldAttributes = new ArrayList<>(); + public LogicalPlan apply(LogicalPlan plan) { + unionFieldAttributes = new ArrayList<>(); + // Collect field attributes from previous runs + plan.forEachUp(EsRelation.class, rel -> { + for (Attribute attr : rel.output()) { + if (attr instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField) { + unionFieldAttributes.add(fa); + } + } + }); + + return plan.transformUp(LogicalPlan.class, p -> p.resolved() || p.childrenResolved() == false ? p : doRule(p)); + } + + private LogicalPlan doRule(LogicalPlan plan) { + int alreadyAddedUnionFieldAttributes = unionFieldAttributes.size(); // See if the eval function has an unresolved MultiTypeEsField field // Replace the entire convert function with a new FieldAttribute (containing type conversion knowledge) plan = plan.transformExpressionsOnly( @@ -1082,15 +1116,16 @@ protected LogicalPlan doRule(LogicalPlan plan) { convert -> resolveConvertFunction(convert, unionFieldAttributes) ); // If no union fields were generated, return the plan as is - if (unionFieldAttributes.isEmpty()) { + if (unionFieldAttributes.size() == alreadyAddedUnionFieldAttributes) { return plan; } // In ResolveRefs the aggregates are resolved from the groupings, which might have an unresolved MultiTypeEsField. // Now that we have resolved those, we need to re-resolve the aggregates. - if (plan instanceof EsqlAggregate agg) { + if (plan instanceof Aggregate agg) { + // TODO once inlinestats supports expressions in groups we'll likely need the same sort of extraction here // If the union-types resolution occurred in a child of the aggregate, we need to check the groupings - plan = agg.transformExpressionsOnly(FieldAttribute.class, UnresolveUnionTypes::checkUnresolved); + plan = agg.transformExpressionsOnly(FieldAttribute.class, UnionTypesCleanup::checkUnresolved); // Aggregates where the grouping key comes from a union-type field need to be resolved against the grouping key Map resolved = new HashMap<>(); @@ -1103,23 +1138,21 @@ protected LogicalPlan doRule(LogicalPlan plan) { plan = plan.transformExpressionsOnly(UnresolvedAttribute.class, ua -> resolveAttribute(ua, resolved)); } - // Otherwise drop the converted attributes after the alias function, as they are only needed for this function, and - // the original version of the attribute should still be seen as unconverted. - plan = dropConvertedAttributes(plan, unionFieldAttributes); - // And add generated fields to EsRelation, so these new attributes will appear in the OutputExec of the Fragment // and thereby get used in FieldExtractExec plan = plan.transformDown(EsRelation.class, esr -> { - List output = esr.output(); List missing = new ArrayList<>(); for (FieldAttribute fa : unionFieldAttributes) { - if (output.stream().noneMatch(a -> a.id().equals(fa.id()))) { + // Using outputSet().contains looks by NameId, resp. uses semanticEquals. + if (esr.outputSet().contains(fa) == false) { missing.add(fa); } } + if (missing.isEmpty() == false) { - output.addAll(missing); - return new EsRelation(esr.source(), esr.index(), output, esr.indexMode(), esr.frozen()); + List newOutput = new ArrayList<>(esr.output()); + newOutput.addAll(missing); + return new EsRelation(esr.source(), esr.index(), newOutput, esr.indexMode(), esr.frozen()); } return esr; }); @@ -1135,17 +1168,6 @@ private Expression resolveAttribute(UnresolvedAttribute ua, Map unionFieldAttributes) { - List projections = new ArrayList<>(plan.output()); - for (var e : unionFieldAttributes) { - projections.removeIf(p -> p.id().equals(e.id())); - } - if (projections.size() != plan.output().size()) { - return new EsqlProject(plan.source(), plan, projections); - } - return plan; - } - private Expression resolveConvertFunction(AbstractConvertFunction convert, List unionFieldAttributes) { if (convert.field() instanceof FieldAttribute fa && fa.field() instanceof InvalidMappedField imf) { HashMap typeResolutions = new HashMap<>(); @@ -1174,7 +1196,13 @@ private Expression createIfDoesNotAlreadyExist( MultiTypeEsField resolvedField, List unionFieldAttributes ) { - var unionFieldAttribute = new FieldAttribute(fa.source(), fa.name(), resolvedField); // Generates new ID for the field + // Generate new ID for the field and suffix it with the data type to maintain unique attribute names. + String unionTypedFieldName = LogicalPlanOptimizer.rawTemporaryName( + fa.name(), + "converted_to", + resolvedField.getDataType().typeName() + ); + FieldAttribute unionFieldAttribute = new FieldAttribute(fa.source(), fa.parent(), unionTypedFieldName, resolvedField); int existingIndex = unionFieldAttributes.indexOf(unionFieldAttribute); if (existingIndex >= 0) { // Do not generate multiple name/type combinations with different IDs @@ -1207,23 +1235,30 @@ private Expression typeSpecificConvert(AbstractConvertFunction convert, Source s } /** - * If there was no AbstractConvertFunction that resolved multi-type fields in the ResolveUnionTypes rules, - * then there could still be some FieldAttributes that contain unresolved MultiTypeEsFields. - * These need to be converted back to actual UnresolvedAttribute in order for validation to generate appropriate failures. + * {@link ResolveUnionTypes} creates new, synthetic attributes for union types: + * If there was no {@code AbstractConvertFunction} that resolved multi-type fields in the {@link ResolveUnionTypes} rule, + * then there could still be some {@code FieldAttribute}s that contain unresolved {@link MultiTypeEsField}s. + * These need to be converted back to actual {@code UnresolvedAttribute} in order for validation to generate appropriate failures. + *

    + * Finally, if {@code client_ip} is present in 2 indices, once with type {@code ip} and once with type {@code keyword}, + * using {@code EVAL x = to_ip(client_ip)} will create a single attribute @{code $$client_ip$converted_to$ip}. + * This should not spill into the query output, so we drop such attributes at the end. */ - private static class UnresolveUnionTypes extends AnalyzerRules.AnalyzerRule { - @Override - protected boolean skipResolved() { - return false; - } + private static class UnionTypesCleanup extends Rule { + public LogicalPlan apply(LogicalPlan plan) { + LogicalPlan planWithCheckedUnionTypes = plan.transformUp(LogicalPlan.class, p -> { + if (p instanceof EsRelation esRelation) { + // Leave esRelation as InvalidMappedField so that UNSUPPORTED fields can still pass through + return esRelation; + } + return p.transformExpressionsOnly(FieldAttribute.class, UnionTypesCleanup::checkUnresolved); + }); - @Override - protected LogicalPlan rule(LogicalPlan plan) { - if (plan instanceof EsRelation esRelation) { - // Leave esRelation as InvalidMappedField so that UNSUPPORTED fields can still pass through - return esRelation; - } - return plan.transformExpressionsOnly(FieldAttribute.class, UnresolveUnionTypes::checkUnresolved); + // To drop synthetic attributes at the end, we need to compute the plan's output. + // This is only legal to do if the plan is resolved. + return planWithCheckedUnionTypes.resolved() + ? planWithoutSyntheticAttributes(planWithCheckedUnionTypes) + : planWithCheckedUnionTypes; } static Attribute checkUnresolved(FieldAttribute fa) { @@ -1233,5 +1268,20 @@ static Attribute checkUnresolved(FieldAttribute fa) { } return fa; } + + private static LogicalPlan planWithoutSyntheticAttributes(LogicalPlan plan) { + List output = plan.output(); + List newOutput = new ArrayList<>(output.size()); + + for (Attribute attr : output) { + // TODO: this should really use .synthetic() + // https://github.com/elastic/elasticsearch/issues/105821 + if (attr.name().startsWith(FieldAttribute.SYNTHETIC_ATTRIBUTE_NAME_PREFIX) == false) { + newOutput.add(attr); + } + } + + return newOutput.size() == output.size() ? plan : new Project(Source.EMPTY, plan, newOutput); + } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerRules.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerRules.java index 3314129fae405..242c947e56de9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerRules.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerRules.java @@ -20,8 +20,6 @@ import java.util.function.Predicate; import java.util.function.Supplier; -import static java.util.Collections.singletonList; - public final class AnalyzerRules { public abstract static class AnalyzerRule extends Rule { @@ -138,14 +136,6 @@ public static List maybeResolveAgainstList( ) .toList(); - return singletonList( - ua.withUnresolvedMessage( - "Reference [" - + ua.qualifiedName() - + "] is ambiguous (to disambiguate use quotes or qualifiers); " - + "matches any of " - + refs - ) - ); + throw new IllegalStateException("Reference [" + ua.qualifiedName() + "] is ambiguous; " + "matches any of " + refs); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java index 790142bef6a86..bda222c2b3e3f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java @@ -9,14 +9,17 @@ import org.elasticsearch.xpack.esql.core.analyzer.TableInfo; import org.elasticsearch.xpack.esql.plan.logical.Enrich; -import org.elasticsearch.xpack.esql.plan.logical.EsqlUnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; import java.util.ArrayList; import java.util.List; import static java.util.Collections.emptyList; +/** + * This class is part of the planner. Acts somewhat like a linker, to find the indices and enrich policies referenced by the query. + */ public class PreAnalyzer { public static class PreAnalysis { @@ -43,7 +46,7 @@ protected PreAnalysis doPreAnalyze(LogicalPlan plan) { List indices = new ArrayList<>(); List unresolvedEnriches = new ArrayList<>(); - plan.forEachUp(EsqlUnresolvedRelation.class, p -> { indices.add(new TableInfo(p.table(), p.frozen())); }); + plan.forEachUp(UnresolvedRelation.class, p -> indices.add(new TableInfo(p.table(), p.frozen()))); plan.forEachUp(Enrich.class, unresolvedEnriches::add); // mark plan as preAnalyzed (if it were marked, there would be no analysis) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index a4e0d99b0d3fc..8d1b96570c7a5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -42,7 +42,6 @@ import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; import org.elasticsearch.xpack.esql.stats.FeatureMetric; import org.elasticsearch.xpack.esql.stats.Metrics; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.util.ArrayList; import java.util.BitSet; @@ -57,6 +56,10 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; +/** + * This class is part of the planner. Responsible for failing impossible queries with a human-readable error message. In particular, this + * step does type resolution and fails queries based on invalid type expressions. + */ public class Verifier { private final Metrics metrics; @@ -343,7 +346,7 @@ private static void checkRegexExtractOnlyOnStrings(LogicalPlan p, Set f if (p instanceof RegexExtract re) { Expression expr = re.input(); DataType type = expr.dataType(); - if (EsqlDataTypes.isString(type) == false) { + if (DataType.isString(type) == false) { failures.add( fail( expr, @@ -360,7 +363,7 @@ private static void checkRegexExtractOnlyOnStrings(LogicalPlan p, Set f private static void checkRow(LogicalPlan p, Set failures) { if (p instanceof Row row) { row.fields().forEach(a -> { - if (EsqlDataTypes.isRepresentable(a.dataType()) == false) { + if (DataType.isRepresentable(a.dataType()) == false) { failures.add(fail(a, "cannot use [{}] directly in a row assignment", a.child().sourceText())); } }); @@ -372,7 +375,7 @@ private static void checkEvalFields(LogicalPlan p, Set failures) { eval.fields().forEach(field -> { // check supported types DataType dataType = field.dataType(); - if (EsqlDataTypes.isRepresentable(dataType) == false) { + if (DataType.isRepresentable(dataType) == false) { failures.add( fail(field, "EVAL does not support type [{}] in expression [{}]", dataType.typeName(), field.child().sourceText()) ); @@ -524,7 +527,7 @@ private static void checkForSortOnSpatialTypes(LogicalPlan p, Set local if (p instanceof OrderBy ob) { ob.forEachExpression(Attribute.class, attr -> { DataType dataType = attr.dataType(); - if (EsqlDataTypes.isSpatial(dataType)) { + if (DataType.isSpatial(dataType)) { localFailures.add(fail(attr, "cannot sort on " + dataType.typeName())); } }); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java index 2425fa24b17c2..373bf5a99056d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java @@ -83,7 +83,6 @@ import org.elasticsearch.xpack.esql.planner.EsPhysicalOperationProviders; import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.util.ArrayList; @@ -464,10 +463,8 @@ private static class LookupRequest extends TransportRequest implements IndicesRe super(in); this.sessionId = in.readString(); this.shardId = new ShardId(in); - String inputDataType = (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_EXTENDED_ENRICH_INPUT_TYPE)) - ? in.readString() - : "unknown"; - this.inputDataType = EsqlDataTypes.fromTypeName(inputDataType); + String inputDataType = (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) ? in.readString() : "unknown"; + this.inputDataType = DataType.fromTypeName(inputDataType); this.matchType = in.readString(); this.matchField = in.readString(); try (BlockStreamInput bsi = new BlockStreamInput(in, blockFactory)) { @@ -483,7 +480,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(sessionId); out.writeWriteable(shardId); - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_EXTENDED_ENRICH_INPUT_TYPE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeString(inputDataType.typeName()); } out.writeString(matchType); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java index 82eda9679074d..2d42241d77ada 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java @@ -37,11 +37,11 @@ import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.esql.analysis.EnrichResolution; import org.elasticsearch.xpack.esql.core.index.EsIndex; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.session.IndexResolver; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.util.ArrayList; @@ -192,7 +192,7 @@ private Tuple mergeLookupResults( EsField field = m.getValue(); field = new EsField( field.getName(), - EsqlDataTypes.fromTypeName(field.getDataType().typeName()), + DataType.fromTypeName(field.getDataType().typeName()), field.getProperties(), field.isAggregatable(), field.isAlias() diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java index c8074d29e0576..d36ab3e18f336 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java @@ -30,7 +30,6 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.evaluator.mapper.ExpressionMapper; -import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.InMapper; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.InsensitiveEqualsMapper; import org.elasticsearch.xpack.esql.planner.Layout; @@ -39,7 +38,6 @@ public final class EvalMapper { private static final List> MAPPERS = List.of( - InMapper.IN_MAPPER, new InsensitiveEqualsMapper(), new BooleanLogic(), new Nots(), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/InMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/InMapper.java deleted file mode 100644 index 430590e1cb240..0000000000000 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/InMapper.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison; - -import org.elasticsearch.compute.data.Block; -import org.elasticsearch.compute.data.BlockFactory; -import org.elasticsearch.compute.data.BooleanBlock; -import org.elasticsearch.compute.data.BooleanVector; -import org.elasticsearch.compute.data.Page; -import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.compute.operator.EvalOperator; -import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; -import org.elasticsearch.core.Releasables; -import org.elasticsearch.xpack.esql.evaluator.EvalMapper; -import org.elasticsearch.xpack.esql.evaluator.mapper.ExpressionMapper; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; -import org.elasticsearch.xpack.esql.planner.Layout; - -import java.util.ArrayList; -import java.util.BitSet; -import java.util.List; - -public class InMapper extends ExpressionMapper { - - public static final InMapper IN_MAPPER = new InMapper(); - - private InMapper() {} - - @SuppressWarnings({ "rawtypes", "unchecked" }) - @Override - public ExpressionEvaluator.Factory map(In in, Layout layout) { - List listEvaluators = new ArrayList<>(in.list().size()); - in.list().forEach(e -> { - Equals eq = new Equals(in.source(), in.value(), e); - ExpressionEvaluator.Factory eqEvaluator = EvalMapper.toEvaluator(eq, layout); - listEvaluators.add(eqEvaluator); - }); - return dvrCtx -> new InExpressionEvaluator(dvrCtx, listEvaluators.stream().map(fac -> fac.get(dvrCtx)).toList()); - } - - record InExpressionEvaluator(DriverContext driverContext, List listEvaluators) - implements - EvalOperator.ExpressionEvaluator { - @Override - public Block eval(Page page) { - int positionCount = page.getPositionCount(); - boolean[] values = new boolean[positionCount]; - BitSet nulls = new BitSet(positionCount); // at least one evaluation resulted in NULL on a row - boolean nullInValues = false; // set when NULL's added in the values list: `field IN (valueA, null, valueB)` - - for (int i = 0; i < listEvaluators().size(); i++) { - var evaluator = listEvaluators.get(i); - try (BooleanBlock block = (BooleanBlock) evaluator.eval(page)) { - BooleanVector vector = block.asVector(); - if (vector != null) { - updateValues(vector, values); - } else { - if (block.areAllValuesNull()) { - nullInValues = true; - } else { - updateValues(block, values, nulls); - } - } - } - } - - return evalWithNulls(driverContext.blockFactory(), values, nulls, nullInValues); - } - - private static void updateValues(BooleanVector vector, boolean[] values) { - for (int p = 0; p < values.length; p++) { - values[p] |= vector.getBoolean(p); - } - } - - private static void updateValues(BooleanBlock block, boolean[] values, BitSet nulls) { - for (int p = 0; p < values.length; p++) { - if (block.isNull(p)) { - nulls.set(p); - } else { - int start = block.getFirstValueIndex(p); - int end = start + block.getValueCount(p); - for (int i = start; i < end; i++) { // if MV_ANY is true, evaluation is true - if (block.getBoolean(i)) { - values[p] = true; - break; - } - } - } - } - } - - private static Block evalWithNulls(BlockFactory blockFactory, boolean[] values, BitSet nulls, boolean nullInValues) { - if (nulls.isEmpty() && nullInValues == false) { - return blockFactory.newBooleanArrayVector(values, values.length).asBlock(); - } else { - // 3VL: true trumps null; null trumps false. - for (int i = 0; i < values.length; i++) { - if (values[i]) { - nulls.clear(i); - } else if (nullInValues) { - nulls.set(i); - } // else: leave nulls as is - } - if (nulls.isEmpty()) { - // no nulls and no multi-values means we must use a Vector - return blockFactory.newBooleanArrayVector(values, values.length).asBlock(); - } else { - return blockFactory.newBooleanArrayBlock(values, values.length, null, nulls, Block.MvOrdering.UNORDERED); - } - } - } - - @Override - public void close() { - Releasables.closeExpectNoException(() -> Releasables.close(listEvaluators)); - } - } -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java index 8f7fcef0ff07e..b97374d179a44 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java @@ -12,7 +12,6 @@ import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.util.Locale; @@ -66,21 +65,21 @@ public static Expression.TypeResolution isExact(Expression e, String operationNa private static final String[] POINT_TYPE_NAMES = new String[] { GEO_POINT.typeName(), CARTESIAN_POINT.typeName() }; private static final String[] NON_SPATIAL_TYPE_NAMES = DataType.types() .stream() - .filter(EsqlDataTypes::isRepresentable) - .filter(t -> EsqlDataTypes.isSpatial(t) == false) + .filter(DataType::isRepresentable) + .filter(t -> DataType.isSpatial(t) == false) .map(DataType::esType) .toArray(String[]::new); public static Expression.TypeResolution isSpatialPoint(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) { - return isType(e, EsqlDataTypes::isSpatialPoint, operationName, paramOrd, POINT_TYPE_NAMES); + return isType(e, DataType::isSpatialPoint, operationName, paramOrd, POINT_TYPE_NAMES); } public static Expression.TypeResolution isSpatial(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) { - return isType(e, EsqlDataTypes::isSpatial, operationName, paramOrd, SPATIAL_TYPE_NAMES); + return isType(e, DataType::isSpatial, operationName, paramOrd, SPATIAL_TYPE_NAMES); } public static Expression.TypeResolution isNotSpatial(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) { - return isType(e, t -> EsqlDataTypes.isSpatial(t) == false, operationName, paramOrd, NON_SPATIAL_TYPE_NAMES); + return isType(e, t -> DataType.isSpatial(t) == false, operationName, paramOrd, NON_SPATIAL_TYPE_NAMES); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/NamedExpressions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/NamedExpressions.java index d0c8adfd3c858..624ea9a030208 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/NamedExpressions.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/NamedExpressions.java @@ -33,7 +33,8 @@ public static List mergeOutputAttributes( /** * Merges output expressions of a command given the new attributes plus the existing inputs that are emitted as outputs. * As a general rule, child output will come first in the list, followed by the new fields. - * In case of name collisions, only last entry is preserved (previous expressions with the same name are discarded) + * In case of name collisions, only the last entry is preserved (previous expressions with the same name are discarded) + * and the new attributes have precedence over the child output. * @param fields the fields added by the command * @param childOutput the command input that has to be propagated as output * @return diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 9a4236cbd96fd..de7e63e16d53e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -71,6 +71,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cos; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cosh; import org.elasticsearch.xpack.esql.expression.function.scalar.math.E; +import org.elasticsearch.xpack.esql.expression.function.scalar.math.Exp; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Floor; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Log; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Log10; @@ -270,6 +271,7 @@ private FunctionDefinition[][] functions() { def(Cos.class, Cos::new, "cos"), def(Cosh.class, Cosh::new, "cosh"), def(E.class, E::new, "e"), + def(Exp.class, Exp::new, "exp"), def(Floor.class, Floor::new, "floor"), def(Greatest.class, Greatest::new, "greatest"), def(Log.class, Log::new, "log"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionInfo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionInfo.java index 98c191eddfb06..94e3aa4e1dd68 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionInfo.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionInfo.java @@ -25,7 +25,7 @@ /** * The description of the function rendered in {@code META FUNCTIONS} - * and the docs. + * and the docs. These should be complete sentences. */ String description() default ""; @@ -39,6 +39,11 @@ */ String note() default ""; + /** + * Extra information rendered at the bottom of the function docs. + */ + String appendix() default ""; + /** * Is this an aggregation (true) or a scalar function (false). */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java index 22c4aa9c6bf07..a553361f60a18 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java @@ -104,6 +104,13 @@ public UnsupportedEsField field() { return (UnsupportedEsField) super.field(); } + @Override + public String fieldName() { + // The super fieldName uses parents to compute the path; this class ignores parents, so we need to rely on the name instead. + // Using field().getName() would be wrong: for subfields like parent.subfield that would return only the last part, subfield. + return name(); + } + @Override protected NodeInfo info() { return NodeInfo.create(this, UnsupportedAttribute::new, name(), field(), hasCustomMessage ? message : null, id()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java index 5e61f69758a47..7686d10a03d9e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java @@ -116,10 +116,10 @@ protected TypeResolution resolveType() { boolean resolved = resolution.resolved(); resolution = isType( field(), - dt -> resolved && dt != DataType.UNSIGNED_LONG, + dt -> resolved && dt != DataType.UNSIGNED_LONG && dt != DataType.SOURCE, sourceText(), DEFAULT, - "any exact type except unsigned_long or counter types" + "any exact type except unsigned_long, _source, or counter types" ); if (resolution.unresolved() || precision == null) { return resolution; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java index 98748fad681c2..4438ccec04c4c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java @@ -13,6 +13,7 @@ import org.elasticsearch.compute.aggregation.MaxBooleanAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MaxDoubleAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MaxIntAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.MaxIpAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MaxLongAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -36,7 +37,7 @@ public class Max extends AggregateFunction implements ToAggregator, SurrogateExp public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Max", Max::new); @FunctionInfo( - returnType = { "boolean", "double", "integer", "long", "date" }, + returnType = { "boolean", "double", "integer", "long", "date", "ip" }, description = "The maximum value of a field.", isAggregation = true, examples = { @@ -49,7 +50,7 @@ public class Max extends AggregateFunction implements ToAggregator, SurrogateExp tag = "docsStatsMaxNestedExpression" ) } ) - public Max(Source source, @Param(name = "field", type = { "boolean", "double", "integer", "long", "date" }) Expression field) { + public Max(Source source, @Param(name = "field", type = { "boolean", "double", "integer", "long", "date", "ip" }) Expression field) { super(source, field); } @@ -76,11 +77,12 @@ public Max replaceChildren(List newChildren) { protected TypeResolution resolveType() { return TypeResolutions.isType( this, - e -> e == DataType.BOOLEAN || e == DataType.DATETIME || (e.isNumeric() && e != DataType.UNSIGNED_LONG), + e -> e == DataType.BOOLEAN || e == DataType.DATETIME || e == DataType.IP || (e.isNumeric() && e != DataType.UNSIGNED_LONG), sourceText(), DEFAULT, "boolean", "datetime", + "ip", "numeric except unsigned_long or counter types" ); } @@ -105,6 +107,9 @@ public final AggregatorFunctionSupplier supplier(List inputChannels) { if (type == DataType.DOUBLE) { return new MaxDoubleAggregatorFunctionSupplier(inputChannels); } + if (type == DataType.IP) { + return new MaxIpAggregatorFunctionSupplier(inputChannels); + } throw EsqlIllegalArgumentException.illegalDataType(type); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java index f712786bcff4b..490d227206e06 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java @@ -13,6 +13,7 @@ import org.elasticsearch.compute.aggregation.MinBooleanAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MinDoubleAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MinIntAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.MinIpAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MinLongAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -36,7 +37,7 @@ public class Min extends AggregateFunction implements ToAggregator, SurrogateExp public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Min", Min::new); @FunctionInfo( - returnType = { "boolean", "double", "integer", "long", "date" }, + returnType = { "boolean", "double", "integer", "long", "date", "ip" }, description = "The minimum value of a field.", isAggregation = true, examples = { @@ -49,7 +50,7 @@ public class Min extends AggregateFunction implements ToAggregator, SurrogateExp tag = "docsStatsMinNestedExpression" ) } ) - public Min(Source source, @Param(name = "field", type = { "boolean", "double", "integer", "long", "date" }) Expression field) { + public Min(Source source, @Param(name = "field", type = { "boolean", "double", "integer", "long", "date", "ip" }) Expression field) { super(source, field); } @@ -76,11 +77,12 @@ public Min replaceChildren(List newChildren) { protected TypeResolution resolveType() { return TypeResolutions.isType( this, - e -> e == DataType.BOOLEAN || e == DataType.DATETIME || (e.isNumeric() && e != DataType.UNSIGNED_LONG), + e -> e == DataType.BOOLEAN || e == DataType.DATETIME || e == DataType.IP || (e.isNumeric() && e != DataType.UNSIGNED_LONG), sourceText(), DEFAULT, "boolean", "datetime", + "ip", "numeric except unsigned_long or counter types" ); } @@ -105,6 +107,9 @@ public final AggregatorFunctionSupplier supplier(List inputChannels) { if (type == DataType.DOUBLE) { return new MinDoubleAggregatorFunctionSupplier(inputChannels); } + if (type == DataType.IP) { + return new MinIpAggregatorFunctionSupplier(inputChannels); + } throw EsqlIllegalArgumentException.illegalDataType(type); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java index b65e78b431159..54cebc7daad5d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; @@ -28,7 +29,6 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isFoldable; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; public class Percentile extends NumericAggregate { @@ -41,9 +41,33 @@ public class Percentile extends NumericAggregate { private final Expression percentile; @FunctionInfo( - returnType = { "double", "integer", "long" }, - description = "The value at which a certain percentage of observed values occur.", - isAggregation = true + returnType = "double", + description = "Returns the value at which a certain percentage of observed values occur. " + + "For example, the 95th percentile is the value which is greater than 95% of the " + + "observed values and the 50th percentile is the `MEDIAN`.", + appendix = """ + [discrete] + [[esql-percentile-approximate]] + ==== `PERCENTILE` is (usually) approximate + + include::../../../aggregations/metrics/percentile-aggregation.asciidoc[tag=approximate] + + [WARNING] + ==== + `PERCENTILE` is also {wikipedia}/Nondeterministic_algorithm[non-deterministic]. + This means you can get slightly different results using the same data. + ==== + """, + isAggregation = true, + examples = { + @Example(file = "stats_percentile", tag = "percentile"), + @Example( + description = "The expression can use inline functions. For example, to calculate a percentile " + + "of the maximum values of a multivalued column, first use `MV_MAX` to get the " + + "maximum value per row, and use the result with the `PERCENTILE` function", + file = "stats_percentile", + tag = "docsStatsPercentileNestedExpression" + ), } ) public Percentile( Source source, @@ -101,7 +125,13 @@ protected TypeResolution resolveType() { return resolution; } - return isNumeric(percentile, sourceText(), SECOND).and(isFoldable(percentile, sourceText(), SECOND)); + return isType( + percentile, + dt -> dt.isNumeric() && dt != DataType.UNSIGNED_LONG, + sourceText(), + SECOND, + "numeric except unsigned_long" + ).and(isFoldable(percentile, sourceText(), SECOND)); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java index 682590bb7e857..f5597b7d64e81 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java @@ -26,7 +26,6 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.planner.ToAggregator; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.time.Duration; @@ -125,7 +124,7 @@ protected TypeResolution resolveType() { ); if (unit != null) { resolution = resolution.and( - isType(unit, dt -> dt.isWholeNumber() || EsqlDataTypes.isTemporalAmount(dt), sourceText(), SECOND, "time_duration") + isType(unit, dt -> dt.isWholeNumber() || DataType.isTemporalAmount(dt), sourceText(), SECOND, "time_duration") ); } return resolution; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java index e15cf774c3c3f..4f85a15732a6f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.expression.SurrogateExpression; +import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSum; @@ -37,7 +38,20 @@ public class Sum extends NumericAggregate implements SurrogateExpression { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Sum", Sum::new); - @FunctionInfo(returnType = "long", description = "The sum of a numeric field.", isAggregation = true) + @FunctionInfo( + returnType = { "long", "double" }, + description = "The sum of a numeric expression.", + isAggregation = true, + examples = { + @Example(file = "stats", tag = "sum"), + @Example( + description = "The expression can use inline functions. For example, to calculate " + + "the sum of each employee's maximum salary changes, apply the " + + "`MV_MAX` function to each row and then sum the results", + file = "stats", + tag = "docsStatsSumNestedExpression" + ) } + ) public Sum(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) { super(source, field); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java index c966ef7afb7c9..4927acc3e1cd9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java @@ -12,8 +12,10 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.TopBooleanAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.TopDoubleAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.TopIntAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.TopIpAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.TopLongAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -46,7 +48,7 @@ public class Top extends AggregateFunction implements ToAggregator, SurrogateExp private static final String ORDER_DESC = "DESC"; @FunctionInfo( - returnType = { "double", "integer", "long", "date" }, + returnType = { "boolean", "double", "integer", "long", "date", "ip" }, description = "Collects the top values for a field. Includes repeated values.", isAggregation = true, examples = @Example(file = "stats_top", tag = "top") @@ -55,7 +57,7 @@ public Top( Source source, @Param( name = "field", - type = { "double", "integer", "long", "date" }, + type = { "boolean", "double", "integer", "long", "date", "ip" }, description = "The field to collect the top values for." ) Expression field, @Param(name = "limit", type = { "integer" }, description = "The maximum number of values to collect.") Expression limit, @@ -120,9 +122,15 @@ protected TypeResolution resolveType() { var typeResolution = isType( field(), - dt -> dt == DataType.DATETIME || dt.isNumeric() && dt != DataType.UNSIGNED_LONG, + dt -> dt == DataType.BOOLEAN + || dt == DataType.DATETIME + || dt == DataType.IP + || (dt.isNumeric() && dt != DataType.UNSIGNED_LONG), sourceText(), FIRST, + "boolean", + "date", + "ip", "numeric except unsigned_long or counter types" ).and(isNotNullAndFoldable(limitField(), sourceText(), SECOND)) .and(isType(limitField(), dt -> dt == DataType.INTEGER, sourceText(), SECOND, "integer")) @@ -176,6 +184,12 @@ public AggregatorFunctionSupplier supplier(List inputChannels) { if (type == DataType.DOUBLE) { return new TopDoubleAggregatorFunctionSupplier(inputChannels, limitValue(), orderValue()); } + if (type == DataType.BOOLEAN) { + return new TopBooleanAggregatorFunctionSupplier(inputChannels, limitValue(), orderValue()); + } + if (type == DataType.IP) { + return new TopIpAggregatorFunctionSupplier(inputChannels, limitValue(), orderValue()); + } throw EsqlIllegalArgumentException.illegalDataType(type); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/package-info.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/package-info.java index f5b40df6fa619..055e34ad5a633 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/package-info.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/package-info.java @@ -118,6 +118,33 @@ *

    * *
  3. + * The methods in the aggregator will define how it will work: + *
      + *
    • + * Adding the `type init()` method will autogenerate the code to manage the state, using your returned value + * as the initial value for each group. + *
      • + *
    • + * Adding the `type initSingle()` or `type initGrouping()` methods will use the state object you return there instead. + *

      + * You will also have to provide `evaluateIntermediate()` and `evaluateFinal()` methods this way. + *

      + *
      • + *
    + * Depending on the way you use, adapt your `combine*()` methods to receive one or other type as their first parameters. + *
    4. + *
  4. + * If it's also a {@link org.elasticsearch.compute.ann.GroupingAggregator}, you should provide the same methods as commented before: + *
      + *
    • + * Add an `initGrouping()`, unless you're using the `init()` method + *
      • + *
    • + * Add all the other methods, with the state parameter of the type of your `initGrouping()`. + *
      • + *
    + *
    5. + *
  5. * Make a test for your aggregator. * You can copy an existing one from {@code x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/}. *

    diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java index 3ce51b8086dd0..8547e5c6f5730 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java @@ -33,7 +33,6 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Div; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mul; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.time.ZoneId; @@ -239,7 +238,7 @@ public ExpressionEvaluator.Factory toEvaluator(Function dt.isWholeNumber() || EsqlDataTypes.isTemporalAmount(dt), + dt -> dt.isWholeNumber() || DataType.isTemporalAmount(dt), sourceText(), SECOND, "integral", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java index 0e9dbf3057c1b..980d3ab0e7552 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java @@ -42,6 +42,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.math.Ceil; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cos; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cosh; +import org.elasticsearch.xpack.esql.expression.function.scalar.math.Exp; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Floor; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Log10; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Signum; @@ -80,6 +81,7 @@ public static List getNamedWriteables() { entries.add(Ceil.ENTRY); entries.add(Cos.ENTRY); entries.add(Cosh.ENTRY); + entries.add(Exp.ENTRY); entries.add(Floor.ENTRY); entries.add(FromBase64.ENTRY); entries.add(IsNotNull.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java index 0fed02f89fd92..b731a400deba3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java @@ -26,7 +26,6 @@ import org.elasticsearch.xpack.esql.expression.function.Warnings; import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.util.ArrayList; @@ -50,7 +49,7 @@ public abstract class AbstractConvertFunction extends UnaryScalarFunction { // the numeric types convert functions need to handle; the other numeric types are converted upstream to one of these private static final List NUMERIC_TYPES = List.of(DataType.INTEGER, DataType.LONG, DataType.UNSIGNED_LONG, DataType.DOUBLE); - public static final List STRING_TYPES = DataType.types().stream().filter(EsqlDataTypes::isString).toList(); + public static final List STRING_TYPES = DataType.types().stream().filter(DataType::isString).toList(); protected AbstractConvertFunction(Source source, Expression field) { super(source, field); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java index 5a57e98be38b9..f9dcdeb342cb5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java @@ -26,7 +26,6 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.time.ZoneId; @@ -129,7 +128,7 @@ private ChronoField chronoField() { if (chronoField == null) { Expression field = children().get(0); try { - if (field.foldable() && EsqlDataTypes.isString(field.dataType())) { + if (field.foldable() && DataType.isString(field.dataType())) { chronoField = (ChronoField) STRING_TO_CHRONO_FIELD.convert(field.fold()); } } catch (Exception e) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java index 84a1a6e77ea73..bfc1bbaa5101d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java @@ -27,7 +27,6 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.session.EsqlConfiguration; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.util.List; @@ -146,7 +145,7 @@ public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator + ) { + var field = toEvaluator.apply(field()); + var fieldType = field().dataType(); + + if (fieldType == DataType.DOUBLE) { + return new ExpDoubleEvaluator.Factory(source(), field); + } + if (fieldType == DataType.INTEGER) { + return new ExpIntEvaluator.Factory(source(), field); + } + if (fieldType == DataType.LONG) { + return new ExpLongEvaluator.Factory(source(), field); + } + if (fieldType == DataType.UNSIGNED_LONG) { + return new ExpUnsignedLongEvaluator.Factory(source(), field); + } + + throw EsqlIllegalArgumentException.illegalDataType(fieldType); + } + + @Override + public Expression replaceChildren(List newChildren) { + return new Exp(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Exp::new, field()); + } + + @Override + public DataType dataType() { + return DataType.DOUBLE; + } + + @Evaluator(extraName = "Double") + static double process(double val) { + return Math.exp(val); + } + + @Evaluator(extraName = "Int") + static double process(int val) { + return Math.exp(val); + } + + @Evaluator(extraName = "Long") + static double process(long val) { + return Math.exp(val); + } + + @Evaluator(extraName = "UnsignedLong") + static double processUnsignedLong(long val) { + return Math.exp(NumericUtils.unsignedLongToDouble(val)); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java index dc4b78d980c28..deb170d9e569c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java @@ -30,7 +30,6 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.planner.PlannerUtils; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.util.Arrays; @@ -132,14 +131,14 @@ protected TypeResolution resolveType() { return new TypeResolution("Unresolved children"); } - TypeResolution resolution = isType(field1, EsqlDataTypes::isRepresentable, sourceText(), FIRST, "representable"); + TypeResolution resolution = isType(field1, DataType::isRepresentable, sourceText(), FIRST, "representable"); if (resolution.unresolved()) { return resolution; } dataType = field1.dataType(); if (dataType == DataType.NULL) { dataType = field2.dataType(); - return isType(field2, EsqlDataTypes::isRepresentable, sourceText(), SECOND, "representable"); + return isType(field2, DataType::isRepresentable, sourceText(), SECOND, "representable"); } return isType(field2, t -> t == dataType, sourceText(), SECOND, dataType.typeName()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvg.java index 01f24365be225..9f678641060ff 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvg.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvg.java @@ -27,8 +27,8 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; +import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.unsignedLongToDouble; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isRepresentable; /** * Reduce a multivalued field to a single valued field containing the average value. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java index faf7d36e4a24c..f0e56c3df6b1a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java @@ -20,7 +20,6 @@ import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.util.List; @@ -74,7 +73,7 @@ public String getWriteableName() { @Override protected TypeResolution resolveFieldType() { - return isType(field(), EsqlDataTypes::isRepresentable, sourceText(), null, "representable"); + return isType(field(), DataType::isRepresentable, sourceText(), null, "representable"); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java index d17bc26ab808b..b17ddddb422ce 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java @@ -14,11 +14,11 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.planner.PlannerUtils; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.util.List; @@ -86,7 +86,7 @@ public String getWriteableName() { @Override protected TypeResolution resolveFieldType() { - return isType(field(), EsqlDataTypes::isRepresentable, sourceText(), null, "representable"); + return isType(field(), DataType::isRepresentable, sourceText(), null, "representable"); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java index 25e6a85a485c1..37095356343f9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java @@ -22,11 +22,11 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.planner.PlannerUtils; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.util.List; @@ -103,7 +103,7 @@ public String getWriteableName() { @Override protected TypeResolution resolveFieldType() { - return isType(field(), EsqlDataTypes::isRepresentable, sourceText(), null, "representable"); + return isType(field(), DataType::isRepresentable, sourceText(), null, "representable"); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java index 2a9a498ecf9d3..11789f1bb3513 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java @@ -22,11 +22,11 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.planner.PlannerUtils; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.util.List; @@ -103,7 +103,7 @@ public String getWriteableName() { @Override protected TypeResolution resolveFieldType() { - return isType(field(), EsqlDataTypes::isRepresentable, sourceText(), null, "representable"); + return isType(field(), DataType::isRepresentable, sourceText(), null, "representable"); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java index 24873cc1da2e9..08f510211f67b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java @@ -26,8 +26,8 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isRepresentable; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isSpatial; +import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable; +import static org.elasticsearch.xpack.esql.core.type.DataType.isSpatial; /** * Reduce a multivalued field to a single valued field containing the maximum value. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedian.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedian.java index 4e7d6dd4e29b2..e9e6899117805 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedian.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedian.java @@ -31,9 +31,9 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; +import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.bigIntegerToUnsignedLong; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.unsignedLongToBigInteger; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isRepresentable; /** * Reduce a multivalued field to a single valued field containing the average value. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java index 205a09953fde3..6b57e15acf304 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java @@ -26,8 +26,8 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isRepresentable; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isSpatial; +import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable; +import static org.elasticsearch.xpack.esql.core.type.DataType.isSpatial; /** * Reduce a multivalued field to a single valued field containing the minimum value. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java index 3728f4305d5c7..c332e94b20049 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java @@ -33,7 +33,6 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.planner.PlannerUtils; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.util.Arrays; @@ -153,7 +152,7 @@ protected TypeResolution resolveType() { return new TypeResolution("Unresolved children"); } - TypeResolution resolution = isType(field, EsqlDataTypes::isRepresentable, sourceText(), FIRST, "representable"); + TypeResolution resolution = isType(field, DataType::isRepresentable, sourceText(), FIRST, "representable"); if (resolution.unresolved()) { return resolution; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java index ee83236ac6a63..ae7baffdf562d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java @@ -44,7 +44,6 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.planner.PlannerUtils; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.util.Arrays; @@ -128,7 +127,7 @@ protected TypeResolution resolveType() { return new TypeResolution("Unresolved children"); } - TypeResolution resolution = isType(field, EsqlDataTypes::isRepresentable, sourceText(), FIRST, "representable"); + TypeResolution resolution = isType(field, DataType::isRepresentable, sourceText(), FIRST, "representable"); if (resolution.unresolved()) { return resolution; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSum.java index eabf5e20ad1b0..cec4ce007d994 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSum.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSum.java @@ -27,8 +27,8 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; +import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable; import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAddExact; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isRepresentable; /** * Reduce a multivalued field to a single valued field containing the sum of all values. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java index 1beef40ce0c42..d34ff30d9b87b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java @@ -20,7 +20,6 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes; import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.util.List; @@ -163,7 +162,7 @@ protected TypeResolution isSameSpatialType( ? isType(expression, dt -> dt == spatialDataType, operationName, paramOrd, compatibleTypeNames(spatialDataType)) : isType( expression, - dt -> EsqlDataTypes.isSpatial(dt) && spatialCRSCompatible(spatialDataType, dt), + dt -> DataType.isSpatial(dt) && spatialCRSCompatible(spatialDataType, dt), operationName, paramOrd, compatibleTypeNames(spatialDataType) @@ -180,12 +179,12 @@ public void setCrsType(DataType dataType) { private static final String[] CARTESIAN_TYPE_NAMES = new String[] { GEO_POINT.typeName(), GEO_SHAPE.typeName() }; protected static boolean spatialCRSCompatible(DataType spatialDataType, DataType otherDataType) { - return EsqlDataTypes.isSpatialGeo(spatialDataType) && EsqlDataTypes.isSpatialGeo(otherDataType) - || EsqlDataTypes.isSpatialGeo(spatialDataType) == false && EsqlDataTypes.isSpatialGeo(otherDataType) == false; + return DataType.isSpatialGeo(spatialDataType) && DataType.isSpatialGeo(otherDataType) + || DataType.isSpatialGeo(spatialDataType) == false && DataType.isSpatialGeo(otherDataType) == false; } static String[] compatibleTypeNames(DataType spatialDataType) { - return EsqlDataTypes.isSpatialGeo(spatialDataType) ? GEO_TYPE_NAMES : CARTESIAN_TYPE_NAMES; + return DataType.isSpatialGeo(spatialDataType) ? GEO_TYPE_NAMES : CARTESIAN_TYPE_NAMES; } @Override @@ -214,8 +213,8 @@ public enum SpatialCrsType { UNSPECIFIED; public static SpatialCrsType fromDataType(DataType dataType) { - return EsqlDataTypes.isSpatialGeo(dataType) ? SpatialCrsType.GEO - : EsqlDataTypes.isSpatial(dataType) ? SpatialCrsType.CARTESIAN + return DataType.isSpatialGeo(dataType) ? SpatialCrsType.GEO + : DataType.isSpatial(dataType) ? SpatialCrsType.CARTESIAN : SpatialCrsType.UNSPECIFIED; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java index 6c2d11ab0ad16..afa2ba833dcd1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java @@ -31,7 +31,6 @@ import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.util.HashMap; @@ -221,7 +220,7 @@ public SpatialRelatesFunction surrogate() { SpatialContainsGeoSourceAndConstantEvaluator.Factory::new ) ); - if (EsqlDataTypes.isSpatialPoint(spatialType)) { + if (DataType.isSpatialPoint(spatialType)) { evaluatorMap.put( SpatialEvaluatorFactory.SpatialEvaluatorKey.fromSources(spatialType, otherType).withLeftDocValues(), new SpatialEvaluatorFactory.SpatialEvaluatorFactoryWithFields( @@ -253,7 +252,7 @@ public SpatialRelatesFunction surrogate() { SpatialContainsCartesianSourceAndConstantEvaluator.Factory::new ) ); - if (EsqlDataTypes.isSpatialPoint(spatialType)) { + if (DataType.isSpatialPoint(spatialType)) { evaluatorMap.put( SpatialEvaluatorFactory.SpatialEvaluatorKey.fromSources(spatialType, otherType).withLeftDocValues(), new SpatialEvaluatorFactory.SpatialEvaluatorFactoryWithFields( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java index e5520079e1b10..9e37bf4c8fa51 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java @@ -29,7 +29,6 @@ import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.util.HashMap; @@ -163,7 +162,7 @@ public Object fold() { SpatialDisjointGeoSourceAndConstantEvaluator.Factory::new ) ); - if (EsqlDataTypes.isSpatialPoint(spatialType)) { + if (DataType.isSpatialPoint(spatialType)) { evaluatorMap.put( SpatialEvaluatorFactory.SpatialEvaluatorKey.fromSources(spatialType, otherType).withLeftDocValues(), new SpatialEvaluatorFactory.SpatialEvaluatorFactoryWithFields( @@ -195,7 +194,7 @@ public Object fold() { SpatialDisjointCartesianSourceAndConstantEvaluator.Factory::new ) ); - if (EsqlDataTypes.isSpatialPoint(spatialType)) { + if (DataType.isSpatialPoint(spatialType)) { evaluatorMap.put( SpatialEvaluatorFactory.SpatialEvaluatorKey.fromSources(spatialType, otherType).withLeftDocValues(), new SpatialEvaluatorFactory.SpatialEvaluatorFactoryWithFields( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java index 045690340f6ac..b7aaededf76f5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java @@ -29,7 +29,6 @@ import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.util.HashMap; @@ -161,7 +160,7 @@ public Object fold() { SpatialIntersectsGeoSourceAndConstantEvaluator.Factory::new ) ); - if (EsqlDataTypes.isSpatialPoint(spatialType)) { + if (DataType.isSpatialPoint(spatialType)) { evaluatorMap.put( SpatialEvaluatorFactory.SpatialEvaluatorKey.fromSources(spatialType, otherType).withLeftDocValues(), new SpatialEvaluatorFactory.SpatialEvaluatorFactoryWithFields( @@ -193,7 +192,7 @@ public Object fold() { SpatialIntersectsCartesianSourceAndConstantEvaluator.Factory::new ) ); - if (EsqlDataTypes.isSpatialPoint(spatialType)) { + if (DataType.isSpatialPoint(spatialType)) { evaluatorMap.put( SpatialEvaluatorFactory.SpatialEvaluatorKey.fromSources(spatialType, otherType).withLeftDocValues(), new SpatialEvaluatorFactory.SpatialEvaluatorFactoryWithFields( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java index 68005ecbfed47..927c7aed936da 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java @@ -24,7 +24,6 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.util.Map; @@ -78,7 +77,7 @@ private static boolean isPushableFieldAttribute(Expression exp, Predicate>. `RLIKE` usually acts on a field placed on + the left-hand side of the operator, but it can also act on a constant (literal) + expression. The right-hand side of the operator represents the pattern.""", examples = @Example(file = "docs", tag = "rlike")) + public RLike( + Source source, + @Param(name = "str", type = { "keyword", "text" }, description = "A literal value.") Expression value, + @Param(name = "pattern", type = { "keyword", "text" }, description = "A regular expression.") RLikePattern pattern + ) { super(source, value, pattern); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java index 7e03b3e821f20..7c2ecd0c60e49 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java @@ -48,7 +48,7 @@ public class Substring extends EsqlScalarFunction implements OptionalArgument { @FunctionInfo( returnType = "keyword", - description = "Returns a substring of a string, specified by a start position and an optional length", + description = "Returns a substring of a string, specified by a start position and an optional length.", examples = { @Example(file = "docs", tag = "substring", description = "This example returns the first three characters of every last name:"), @Example(file = "docs", tag = "substringEnd", description = """ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java index 325cc0aea4461..15470bb56b29f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java @@ -47,8 +47,8 @@ also act on a constant (literal) expression. The right-hand side of the operator * `?` matches one character.""", examples = @Example(file = "docs", tag = "like")) public WildcardLike( Source source, - @Param(name = "str", type = { "keyword", "text" }) Expression left, - @Param(name = "pattern", type = { "keyword", "text" }) WildcardPattern pattern + @Param(name = "str", type = { "keyword", "text" }, description = "A literal expression.") Expression left, + @Param(name = "pattern", type = { "keyword", "text" }, description = "Pattern.") WildcardPattern pattern ) { super(source, left, pattern, false); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Add.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Add.java index fcf71900c4198..be1c2ddd67539 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Add.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Add.java @@ -15,6 +15,9 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryComparisonInversible; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.util.NumericUtils; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import java.io.IOException; import java.time.DateTimeException; @@ -30,7 +33,23 @@ public class Add extends DateTimeArithmeticOperation implements BinaryComparisonInversible { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Add", Add::new); - public Add(Source source, Expression left, Expression right) { + @FunctionInfo( + returnType = { "double", "integer", "long", "date_period", "datetime", "time_duration", "unsigned_long" }, + description = "Add two numbers together. " + "If either field is <> then the result is `null`." + ) + public Add( + Source source, + @Param( + name = "lhs", + description = "A numeric value or a date time value.", + type = { "double", "integer", "long", "date_period", "datetime", "time_duration", "unsigned_long" } + ) Expression left, + @Param( + name = "rhs", + description = "A numeric value or a date time value.", + type = { "double", "integer", "long", "date_period", "datetime", "time_duration", "unsigned_long" } + ) Expression right + ) { super( source, left, @@ -101,9 +120,9 @@ public static long processUnsignedLongs(long lhs, long rhs) { return unsignedLongAddExact(lhs, rhs); } - @Evaluator(extraName = "Doubles") + @Evaluator(extraName = "Doubles", warnExceptions = { ArithmeticException.class }) static double processDoubles(double lhs, double rhs) { - return lhs + rhs; + return NumericUtils.asFiniteNumber(lhs + rhs); } @Evaluator(extraName = "Datetimes", warnExceptions = { ArithmeticException.class, DateTimeException.class }) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java index 45cc5b9bdc5c0..5b7cc74faed86 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java @@ -14,7 +14,6 @@ import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.time.Duration; @@ -27,9 +26,9 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD; import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION; import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTime; +import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTimeOrTemporal; import static org.elasticsearch.xpack.esql.core.type.DataType.isNull; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isDateTimeOrTemporal; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isTemporalAmount; +import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount; public abstract class DateTimeArithmeticOperation extends EsqlArithmeticOperation { /** Arithmetic (quad) function. */ @@ -71,7 +70,7 @@ interface DatetimeArithmeticEvaluator { protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { return TypeResolutions.isType( e, - t -> t.isNumeric() || EsqlDataTypes.isDateTimeOrTemporal(t) || DataType.isNull(t), + t -> t.isNumeric() || DataType.isDateTimeOrTemporal(t) || DataType.isNull(t), sourceText(), paramOrdinal, "datetime", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Div.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Div.java index 6d84ce3558571..0e4c506a90d85 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Div.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Div.java @@ -16,6 +16,8 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.NumericUtils; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import java.io.IOException; @@ -27,7 +29,18 @@ public class Div extends EsqlArithmeticOperation implements BinaryComparisonInve private DataType type; - public Div(Source source, Expression left, Expression right) { + @FunctionInfo( + returnType = { "double", "integer", "long", "unsigned_long" }, + description = "Divide one number by another. " + + "If either field is <> then the result is `null`.", + note = "Division of two integer types will yield an integer result, rounding towards 0. " + + "If you need floating point division, <> one of the arguments to a `DOUBLE`." + ) + public Div( + Source source, + @Param(name = "lhs", description = "A numeric value.", type = { "double", "integer", "long", "unsigned_long" }) Expression left, + @Param(name = "rhs", description = "A numeric value.", type = { "double", "integer", "long", "unsigned_long" }) Expression right + ) { this(source, left, right, null); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mod.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mod.java index 151c886bfdf8d..2381e3df9b617 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mod.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mod.java @@ -14,6 +14,8 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.util.NumericUtils; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import java.io.IOException; @@ -23,7 +25,16 @@ public class Mod extends EsqlArithmeticOperation { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Mod", Mod::new); - public Mod(Source source, Expression left, Expression right) { + @FunctionInfo( + returnType = { "double", "integer", "long", "unsigned_long" }, + description = "Divide one number by another and return the remainder. " + + "If either field is <> then the result is `null`." + ) + public Mod( + Source source, + @Param(name = "lhs", description = "A numeric value.", type = { "double", "integer", "long", "unsigned_long" }) Expression left, + @Param(name = "rhs", description = "A numeric value.", type = { "double", "integer", "long", "unsigned_long" }) Expression right + ) { super( source, left, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mul.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mul.java index 08a01fbffcca2..a73562ff153b2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mul.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mul.java @@ -14,6 +14,9 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryComparisonInversible; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.util.NumericUtils; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import java.io.IOException; @@ -23,7 +26,16 @@ public class Mul extends EsqlArithmeticOperation implements BinaryComparisonInversible { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Mul", Mul::new); - public Mul(Source source, Expression left, Expression right) { + @FunctionInfo( + returnType = { "double", "integer", "long", "unsigned_long" }, + description = "Multiply two numbers together. " + + "If either field is <> then the result is `null`." + ) + public Mul( + Source source, + @Param(name = "lhs", description = "A numeric value.", type = { "double", "integer", "long", "unsigned_long" }) Expression left, + @Param(name = "rhs", description = "A numeric value.", type = { "double", "integer", "long", "unsigned_long" }) Expression right + ) { super( source, left, @@ -92,9 +104,9 @@ static long processUnsignedLongs(long lhs, long rhs) { return unsignedLongMultiplyExact(lhs, rhs); } - @Evaluator(extraName = "Doubles") + @Evaluator(extraName = "Doubles", warnExceptions = { ArithmeticException.class }) static double processDoubles(double lhs, double rhs) { - return lhs * rhs; + return NumericUtils.asFiniteNumber(lhs * rhs); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java index d1ed5579c4485..67b770d14339e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java @@ -17,6 +17,8 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; import java.io.IOException; @@ -29,12 +31,23 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD; import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isTemporalAmount; +import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount; public class Neg extends UnaryScalarFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Neg", Neg::new); - public Neg(Source source, Expression field) { + @FunctionInfo( + returnType = { "double", "integer", "long", "date_period", "time_duration" }, + description = "Returns the negation of the argument." + ) + public Neg( + Source source, + @Param( + name = "field", + description = "A numeric value or a date time interval.", + type = { "double", "integer", "long", "date_period", "time_duration" } + ) Expression field + ) { super(source, field); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Sub.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Sub.java index 43398b7750b0d..ccaf2a30632eb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Sub.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Sub.java @@ -16,7 +16,9 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import org.elasticsearch.xpack.esql.core.util.NumericUtils; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import java.io.IOException; import java.time.DateTimeException; @@ -33,7 +35,24 @@ public class Sub extends DateTimeArithmeticOperation implements BinaryComparisonInversible { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Sub", Sub::new); - public Sub(Source source, Expression left, Expression right) { + @FunctionInfo( + returnType = { "double", "integer", "long", "date_period", "datetime", "time_duration", "unsigned_long" }, + description = "Subtract one number from another. " + + "If either field is <> then the result is `null`." + ) + public Sub( + Source source, + @Param( + name = "lhs", + description = "A numeric value or a date time value.", + type = { "double", "integer", "long", "date_period", "datetime", "time_duration", "unsigned_long" } + ) Expression left, + @Param( + name = "rhs", + description = "A numeric value or a date time value.", + type = { "double", "integer", "long", "date_period", "datetime", "time_duration", "unsigned_long" } + ) Expression right + ) { super( source, left, @@ -68,7 +87,7 @@ public String getWriteableName() { protected TypeResolution resolveType() { TypeResolution resolution = super.resolveType(); // As opposed to general date time arithmetics, we cannot subtract a datetime from something else. - if (resolution.resolved() && EsqlDataTypes.isDateTimeOrTemporal(dataType()) && DataType.isDateTime(right().dataType())) { + if (resolution.resolved() && DataType.isDateTimeOrTemporal(dataType()) && DataType.isDateTime(right().dataType())) { return new TypeResolution( format( null, @@ -114,9 +133,9 @@ static long processUnsignedLongs(long lhs, long rhs) { return unsignedLongSubtractExact(lhs, rhs); } - @Evaluator(extraName = "Doubles") + @Evaluator(extraName = "Doubles", warnExceptions = { ArithmeticException.class }) static double processDoubles(double lhs, double rhs) { - return lhs - rhs; + return NumericUtils.asFiniteNumber(lhs - rhs); } @Evaluator(extraName = "Datetimes", warnExceptions = { ArithmeticException.class, DateTimeException.class }) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java index 26a74e7bdb03c..32e15deb07b4e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java @@ -14,6 +14,8 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import java.time.ZoneId; @@ -43,7 +45,54 @@ public class Equals extends EsqlBinaryComparison implements Negatable> then the result is `null`.", + note = "This is pushed to the underlying search index if one side of the comparison is constant " + + "and the other side is a field in the index that has both an <> and <>." + ) + public Equals( + Source source, + @Param( + name = "lhs", + type = { + "boolean", + "cartesian_point", + "cartesian_shape", + "date", + "double", + "geo_point", + "geo_shape", + "integer", + "ip", + "keyword", + "long", + "text", + "unsigned_long", + "version" }, + description = "An expression." + ) Expression left, + @Param( + name = "rhs", + type = { + "boolean", + "cartesian_point", + "cartesian_shape", + "date", + "double", + "geo_point", + "geo_shape", + "integer", + "ip", + "keyword", + "long", + "text", + "unsigned_long", + "version" }, + description = "An expression." + ) Expression right + ) { super(source, left, right, BinaryComparisonOperation.EQ, evaluatorMap); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java index 8ce8bf30ef617..33981a3337737 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java @@ -14,6 +14,8 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import java.time.ZoneId; @@ -38,7 +40,26 @@ public class GreaterThan extends EsqlBinaryComparison implements Negatable> then the result is `null`.", + note = "This is pushed to the underlying search index if one side of the comparison is constant " + + "and the other side is a field in the index that has both an <> and <>." + ) + public GreaterThan( + Source source, + @Param( + name = "lhs", + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" }, + description = "An expression." + ) Expression left, + @Param( + name = "rhs", + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" }, + description = "An expression." + ) Expression right + ) { super(source, left, right, BinaryComparisonOperation.GT, evaluatorMap); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java index d7bfec75dabfc..ffcd99558c6ad 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java @@ -14,6 +14,8 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import java.time.ZoneId; @@ -38,7 +40,26 @@ public class GreaterThanOrEqual extends EsqlBinaryComparison implements Negatabl Map.entry(DataType.IP, GreaterThanOrEqualKeywordsEvaluator.Factory::new) ); - public GreaterThanOrEqual(Source source, Expression left, Expression right) { + @FunctionInfo( + returnType = { "boolean" }, + description = "Check if one field is greater than or equal to another. " + + "If either field is <> then the result is `null`.", + note = "This is pushed to the underlying search index if one side of the comparison is constant " + + "and the other side is a field in the index that has both an <> and <>." + ) + public GreaterThanOrEqual( + Source source, + @Param( + name = "lhs", + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" }, + description = "An expression." + ) Expression left, + @Param( + name = "rhs", + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" }, + description = "An expression." + ) Expression right + ) { super(source, left, right, BinaryComparisonOperation.GTE, evaluatorMap); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java index b7ebf114501cd..636b31fcc691b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java @@ -7,39 +7,115 @@ package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.Comparisons; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.CollectionUtils; import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cast; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry; import java.io.IOException; +import java.util.BitSet; +import java.util.Collections; import java.util.List; +import java.util.function.Function; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; +import static org.elasticsearch.xpack.esql.core.type.DataType.IP; +import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; +import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; +import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; import static org.elasticsearch.xpack.esql.core.util.StringUtils.ordinal; -public class In extends org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.In { +public class In extends EsqlScalarFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "In", In::new); + private final Expression value; + private final List list; + @FunctionInfo( returnType = "boolean", description = "The `IN` operator allows testing whether a field or expression equals an element in a list of literals, " - + "fields or expressions:", + + "fields or expressions.", examples = @Example(file = "row", tag = "in-with-expressions") ) - public In(Source source, Expression value, List list) { - super(source, value, list); + public In( + Source source, + @Param( + name = "field", + type = { + "boolean", + "cartesian_point", + "cartesian_shape", + "double", + "geo_point", + "geo_shape", + "integer", + "ip", + "keyword", + "long", + "text", + "version" }, + description = "An expression." + ) Expression value, + @Param( + name = "inlist", + type = { + "boolean", + "cartesian_point", + "cartesian_shape", + "double", + "geo_point", + "geo_shape", + "integer", + "ip", + "keyword", + "long", + "text", + "version" }, + description = "A list of items." + ) List list + ) { + super(source, CollectionUtils.combine(list, value)); + this.value = value; + this.list = list; + } + + public Expression value() { + return value; + } + + public List list() { + return list; + } + + @Override + public DataType dataType() { + return BOOLEAN; } private In(StreamInput in) throws IOException { @@ -53,8 +129,8 @@ private In(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { source().writeTo(out); - out.writeNamedWriteable(value()); - out.writeNamedWriteableCollection(list()); + out.writeNamedWriteable(value); + out.writeNamedWriteableCollection(list); } @Override @@ -63,8 +139,8 @@ public String getWriteableName() { } @Override - protected NodeInfo info() { - return NodeInfo.create(this, In::new, value(), list()); + protected NodeInfo info() { + return NodeInfo.create(this, In::new, value, list); } @Override @@ -76,52 +152,40 @@ public Expression replaceChildren(List newChildren) { public boolean foldable() { // QL's In fold()s to null, if value() is null, but isn't foldable() unless all children are // TODO: update this null check in QL too? - return Expressions.isNull(value()) || super.foldable(); + return Expressions.isNull(value) + || Expressions.foldable(children()) + || (Expressions.foldable(list) && list.stream().allMatch(Expressions::isNull)); } @Override - public Boolean fold() { - if (Expressions.isNull(value()) || list().stream().allMatch(Expressions::isNull)) { + public Object fold() { + if (Expressions.isNull(value) || list.stream().allMatch(Expressions::isNull)) { return null; } - // QL's `In` fold() doesn't handle BytesRef and can't know if this is Keyword/Text, Version or IP anyway. - // `In` allows comparisons of same type only (safe for numerics), so it's safe to apply InProcessor directly with no implicit - // (non-numerical) conversions. - return apply(value().fold(), list().stream().map(Expression::fold).toList()); + return super.fold(); } - private static Boolean apply(Object input, List values) { - Boolean result = Boolean.FALSE; - for (Object v : values) { - Boolean compResult = Comparisons.eq(input, v); - if (compResult == null) { - result = null; - } else if (compResult == Boolean.TRUE) { - return Boolean.TRUE; - } - } - return result; - } - - @Override protected boolean areCompatible(DataType left, DataType right) { - if (left == DataType.UNSIGNED_LONG || right == DataType.UNSIGNED_LONG) { + if (left == UNSIGNED_LONG || right == UNSIGNED_LONG) { // automatic numerical conversions not applicable for UNSIGNED_LONG, see Verifier#validateUnsignedLongOperator(). return left == right; } - return EsqlDataTypes.areCompatible(left, right); + if (DataType.isSpatial(left) && DataType.isSpatial(right)) { + return left == right; + } + return DataType.areCompatible(left, right); } @Override protected TypeResolution resolveType() { // TODO: move the foldability check from QL's In to SQL's and remove this method - TypeResolution resolution = EsqlTypeResolutions.isExact(value(), functionName(), DEFAULT); + TypeResolution resolution = EsqlTypeResolutions.isExact(value, functionName(), DEFAULT); if (resolution.unresolved()) { return resolution; } - DataType dt = value().dataType(); - for (int i = 0; i < list().size(); i++) { - Expression listValue = list().get(i); + DataType dt = value.dataType(); + for (int i = 0; i < list.size(); i++) { + Expression listValue = list.get(i); if (areCompatible(dt, listValue.dataType()) == false) { return new TypeResolution( format( @@ -139,4 +203,126 @@ protected TypeResolution resolveType() { // TODO: move the foldability check fro return TypeResolution.TYPE_RESOLVED; } + + @Override + protected Expression canonicalize() { + // order values for commutative operators + List canonicalValues = Expressions.canonicalize(list); + Collections.sort(canonicalValues, (l, r) -> Integer.compare(l.hashCode(), r.hashCode())); + return new In(source(), value, canonicalValues); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator( + Function toEvaluator + ) { + var commonType = commonType(); + EvalOperator.ExpressionEvaluator.Factory lhs; + EvalOperator.ExpressionEvaluator.Factory[] factories; + if (commonType.isNumeric()) { + lhs = Cast.cast(source(), value.dataType(), commonType, toEvaluator.apply(value)); + factories = list.stream() + .map(e -> Cast.cast(source(), e.dataType(), commonType, toEvaluator.apply(e))) + .toArray(EvalOperator.ExpressionEvaluator.Factory[]::new); + } else { + lhs = toEvaluator.apply(value); + factories = list.stream().map(e -> toEvaluator.apply(e)).toArray(EvalOperator.ExpressionEvaluator.Factory[]::new); + } + + if (commonType == BOOLEAN) { + return new InBooleanEvaluator.Factory(source(), lhs, factories); + } + if (commonType == DOUBLE) { + return new InDoubleEvaluator.Factory(source(), lhs, factories); + } + if (commonType == INTEGER) { + return new InIntEvaluator.Factory(source(), lhs, factories); + } + if (commonType == LONG || commonType == DATETIME || commonType == UNSIGNED_LONG) { + return new InLongEvaluator.Factory(source(), lhs, factories); + } + if (commonType == KEYWORD + || commonType == TEXT + || commonType == IP + || commonType == VERSION + || commonType == UNSUPPORTED + || DataType.isSpatial(commonType)) { + return new InBytesRefEvaluator.Factory(source(), toEvaluator.apply(value), factories); + } + if (commonType == NULL) { + return EvalOperator.CONSTANT_NULL_FACTORY; + } + throw EsqlIllegalArgumentException.illegalDataType(commonType); + } + + private DataType commonType() { + DataType commonType = value.dataType(); + for (Expression e : list) { + if (e.dataType() == NULL && value.dataType() != NULL) { + continue; + } + if (DataType.isSpatial(commonType)) { + if (e.dataType() == commonType) { + continue; + } else { + commonType = NULL; + break; + } + } + commonType = EsqlDataTypeRegistry.INSTANCE.commonType(commonType, e.dataType()); + } + return commonType; + } + + static boolean process(BitSet nulls, BitSet mvs, int lhs, int[] rhs) { + for (int i = 0; i < rhs.length; i++) { + if ((nulls != null && nulls.get(i)) || (mvs != null && mvs.get(i))) { + continue; + } + Boolean compResult = Comparisons.eq(lhs, rhs[i]); + if (compResult == Boolean.TRUE) { + return true; + } + } + return false; + } + + static boolean process(BitSet nulls, BitSet mvs, long lhs, long[] rhs) { + for (int i = 0; i < rhs.length; i++) { + if ((nulls != null && nulls.get(i)) || (mvs != null && mvs.get(i))) { + continue; + } + Boolean compResult = Comparisons.eq(lhs, rhs[i]); + if (compResult == Boolean.TRUE) { + return true; + } + } + return false; + } + + static boolean process(BitSet nulls, BitSet mvs, double lhs, double[] rhs) { + for (int i = 0; i < rhs.length; i++) { + if ((nulls != null && nulls.get(i)) || (mvs != null && mvs.get(i))) { + continue; + } + Boolean compResult = Comparisons.eq(lhs, rhs[i]); + if (compResult == Boolean.TRUE) { + return true; + } + } + return false; + } + + static boolean process(BitSet nulls, BitSet mvs, BytesRef lhs, BytesRef[] rhs) { + for (int i = 0; i < rhs.length; i++) { + if ((nulls != null && nulls.get(i)) || (mvs != null && mvs.get(i))) { + continue; + } + Boolean compResult = Comparisons.eq(lhs, rhs[i]); + if (compResult == Boolean.TRUE) { + return true; + } + } + return false; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InEvaluator.java.st b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InEvaluator.java.st new file mode 100644 index 0000000000000..bcbc1ae9ccced --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InEvaluator.java.st @@ -0,0 +1,273 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +$if(BytesRef)$ +import org.apache.lucene.util.BytesRef; +$endif$ +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +$if(int)$ +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +$elseif(long)$ +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +$elseif(double)$ +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.DoubleVector; +$elseif(BytesRef)$ +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +$elseif(boolean)$ +import org.elasticsearch.compute.data.BooleanVector; +$endif$ +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +import java.util.Arrays; +import java.util.BitSet; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link In}. + * This class is generated. Edit {@code InEvaluator.java.st} instead. + */ +public class In$Type$Evaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator[] rhs; + + private final DriverContext driverContext; + + public In$Type$Evaluator( + Source source, + EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator[] rhs, + DriverContext driverContext + ) { + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try ($Type$Block lhsBlock = ($Type$Block) lhs.eval(page)) { + $Type$Block[] rhsBlocks = new $Type$Block[rhs.length]; + try (Releasable rhsRelease = Releasables.wrap(rhsBlocks)) { + for (int i = 0; i < rhsBlocks.length; i++) { + rhsBlocks[i] = ($Type$Block) rhs[i].eval(page); + } + $Type$Vector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + $Type$Vector[] rhsVectors = new $Type$Vector[rhs.length]; + for (int i = 0; i < rhsBlocks.length; i++) { + rhsVectors[i] = rhsBlocks[i].asVector(); + if (rhsVectors[i] == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + } + return eval(page.getPositionCount(), lhsVector, rhsVectors); + } + } + } + + private BooleanBlock eval(int positionCount, $Type$Block lhsBlock, $Type$Block[] rhsBlocks) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { +$if(int)$ + int[] rhsValues = new int[rhs.length]; +$elseif(long)$ + long[] rhsValues = new long[rhs.length]; +$elseif(double)$ + double[] rhsValues = new double[rhs.length]; +$elseif(boolean)$ + boolean hasTrue = false; + boolean hasFalse = false; +$elseif(BytesRef)$ + $Type$[] rhsValues = new $Type$[rhs.length]; + BytesRef lhsScratch = new BytesRef(); + BytesRef[] rhsScratch = new BytesRef[rhs.length]; + for (int i = 0; i < rhs.length; i++) { + rhsScratch[i] = new BytesRef(); + } +$endif$ + BitSet nulls = new BitSet(rhs.length); + BitSet mvs = new BitSet(rhs.length); + boolean foundMatch; + for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue; + } + // unpack rhsBlocks into rhsValues + nulls.clear(); + mvs.clear(); +$if(boolean)$ + hasTrue = false; + hasFalse = false; +$endif$ + for (int i = 0; i < rhsBlocks.length; i++) { + if (rhsBlocks[i].isNull(p)) { + nulls.set(i); + continue; + } + if (rhsBlocks[i].getValueCount(p) > 1) { + mvs.set(i); + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + continue; + } +$if(boolean)$ + if (hasTrue && hasFalse) { + continue; + } +$endif$ + int o = rhsBlocks[i].getFirstValueIndex(p); +$if(BytesRef)$ + rhsValues[i] = rhsBlocks[i].getBytesRef(o, rhsScratch[i]); +$elseif(boolean)$ + if (rhsBlocks[i].getBoolean(o)) { + hasTrue = true; + } else { + hasFalse = true; + } +$else$ + rhsValues[i] = rhsBlocks[i].get$Type$(o); +$endif$ + } + if (nulls.cardinality() == rhsBlocks.length || mvs.cardinality() == rhsBlocks.length) { + result.appendNull(); + continue; + } +$if(boolean)$ + foundMatch = lhsBlock.getBoolean(lhsBlock.getFirstValueIndex(p)) ? hasTrue : hasFalse; +$elseif(BytesRef)$ + foundMatch = In.process(nulls, mvs, lhsBlock.getBytesRef(lhsBlock.getFirstValueIndex(p), lhsScratch), rhsValues); +$else$ + foundMatch = In.process(nulls, mvs, lhsBlock.get$Type$(lhsBlock.getFirstValueIndex(p)), rhsValues); +$endif$ + if (foundMatch) { + result.appendBoolean(true); + } else { + if (nulls.cardinality() > 0) { + result.appendNull(); + } else { + result.appendBoolean(false); + } + } + } + return result.build(); + } + } + + private BooleanBlock eval(int positionCount, $Type$Vector lhsVector, $Type$Vector[] rhsVectors) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { +$if(int)$ + int[] rhsValues = new int[rhs.length]; +$elseif(long)$ + long[] rhsValues = new long[rhs.length]; +$elseif(double)$ + double[] rhsValues = new double[rhs.length]; +$elseif(boolean)$ + boolean hasTrue = false; + boolean hasFalse = false; +$elseif(BytesRef)$ + $Type$[] rhsValues = new $Type$[rhs.length]; + BytesRef lhsScratch = new BytesRef(); + BytesRef[] rhsScratch = new BytesRef[rhs.length]; + for (int i = 0; i < rhs.length; i++) { + rhsScratch[i] = new BytesRef(); + } +$endif$ + for (int p = 0; p < positionCount; p++) { + // unpack rhsVectors into rhsValues +$if(boolean)$ + hasTrue = false; + hasFalse = false; +$endif$ + for (int i = 0; i < rhsVectors.length; i++) { +$if(boolean)$ + if (hasTrue && hasFalse) { + continue; + } +$endif$ +$if(BytesRef)$ + rhsValues[i] = rhsVectors[i].getBytesRef(p, rhsScratch[i]); +$elseif(boolean)$ + if (rhsVectors[i].getBoolean(p)) { + hasTrue = true; + } else { + hasFalse = true; + } +$else$ + rhsValues[i] = rhsVectors[i].get$Type$(p); +$endif$ + } +$if(BytesRef)$ + result.appendBoolean(In.process(null, null, lhsVector.getBytesRef(p, lhsScratch), rhsValues)); +$elseif(boolean)$ + result.appendBoolean(lhsVector.getBoolean(p) ? hasTrue : hasFalse); +$else$ + result.appendBoolean(In.process(null, null, lhsVector.get$Type$(p), rhsValues)); +$endif$ + } + return result.build(); + } + } + + @Override + public String toString() { + return "In$Type$Evaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, () -> Releasables.close(rhs)); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + private final EvalOperator.ExpressionEvaluator.Factory lhs; + private final EvalOperator.ExpressionEvaluator.Factory[] rhs; + + Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, EvalOperator.ExpressionEvaluator.Factory[] rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public In$Type$Evaluator get(DriverContext context) { + EvalOperator.ExpressionEvaluator[] rhs = Arrays.stream(this.rhs) + .map(a -> a.get(context)) + .toArray(EvalOperator.ExpressionEvaluator[]::new); + return new In$Type$Evaluator(source, lhs.get(context), rhs, context); + } + + @Override + public String toString() { + return "In$Type$Evaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java index 0afc9e0280f4c..d11f5c9b68532 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java @@ -19,7 +19,6 @@ import org.elasticsearch.xpack.esql.evaluator.mapper.ExpressionMapper; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cast; import org.elasticsearch.xpack.esql.planner.Layout; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import static org.elasticsearch.xpack.esql.evaluator.EvalMapper.toEvaluator; @@ -36,7 +35,7 @@ public final ExpressionEvaluator.Factory map(InsensitiveEquals bc, Layout layout var leftEval = toEvaluator(bc.left(), layout); var rightEval = toEvaluator(bc.right(), layout); if (leftType == DataType.KEYWORD || leftType == DataType.TEXT) { - if (bc.right().foldable() && EsqlDataTypes.isString(rightType)) { + if (bc.right().foldable() && DataType.isString(rightType)) { BytesRef rightVal = BytesRefs.toBytesRef(bc.right().fold()); Automaton automaton = InsensitiveEquals.automaton(rightVal); return dvrCtx -> new InsensitiveEqualsConstantEvaluator( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java index b1562b6dc2be4..a9a1f774f4ebb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java @@ -14,6 +14,8 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import java.time.ZoneId; @@ -38,7 +40,26 @@ public class LessThan extends EsqlBinaryComparison implements Negatable> then the result is `null`.", + note = "This is pushed to the underlying search index if one side of the comparison is constant " + + "and the other side is a field in the index that has both an <> and <>." + ) + public LessThan( + Source source, + @Param( + name = "lhs", + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" }, + description = "An expression." + ) Expression left, + @Param( + name = "rhs", + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" }, + description = "An expression." + ) Expression right + ) { this(source, left, right, null); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java index c31e055c8dd1a..5623750ea0f91 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java @@ -14,6 +14,8 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import java.time.ZoneId; @@ -38,7 +40,26 @@ public class LessThanOrEqual extends EsqlBinaryComparison implements Negatable> then the result is `null`.", + note = "This is pushed to the underlying search index if one side of the comparison is constant " + + "and the other side is a field in the index that has both an <> and <>." + ) + public LessThanOrEqual( + Source source, + @Param( + name = "lhs", + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" }, + description = "An expression." + ) Expression left, + @Param( + name = "rhs", + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" }, + description = "An expression." + ) Expression right + ) { this(source, left, right, null); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java index 179ff61d9c017..d61953587f79b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java @@ -14,6 +14,8 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import java.time.ZoneId; @@ -43,7 +45,54 @@ public class NotEquals extends EsqlBinaryComparison implements Negatable> then the result is `null`.", + note = "This is pushed to the underlying search index if one side of the comparison is constant " + + "and the other side is a field in the index that has both an <> and <>." + ) + public NotEquals( + Source source, + @Param( + name = "lhs", + type = { + "boolean", + "cartesian_point", + "cartesian_shape", + "date", + "double", + "geo_point", + "geo_shape", + "integer", + "ip", + "keyword", + "long", + "text", + "unsigned_long", + "version" }, + description = "An expression." + ) Expression left, + @Param( + name = "rhs", + type = { + "boolean", + "cartesian_point", + "cartesian_shape", + "date", + "double", + "geo_point", + "geo_shape", + "integer", + "ip", + "keyword", + "long", + "text", + "unsigned_long", + "version" }, + description = "An expression." + ) Expression right + ) { super(source, left, right, BinaryComparisonOperation.NEQ, evaluatorMap); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java index e4051523c7a5e..218a478b8425c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java @@ -36,6 +36,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Filter; import org.elasticsearch.xpack.esql.plan.logical.Grok; +import org.elasticsearch.xpack.esql.plan.logical.InlineStats; import org.elasticsearch.xpack.esql.plan.logical.Limit; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.Lookup; @@ -147,6 +148,7 @@ public static List namedTypeEntries() { of(LogicalPlan.class, EsqlProject.class, PlanNamedTypes::writeEsqlProject, PlanNamedTypes::readEsqlProject), of(LogicalPlan.class, Filter.class, PlanNamedTypes::writeFilter, PlanNamedTypes::readFilter), of(LogicalPlan.class, Grok.class, PlanNamedTypes::writeGrok, PlanNamedTypes::readGrok), + of(LogicalPlan.class, InlineStats.class, (PlanStreamOutput out, InlineStats v) -> v.writeTo(out), InlineStats::new), of(LogicalPlan.class, Join.class, (out, p) -> p.writeTo(out), Join::new), of(LogicalPlan.class, Limit.class, PlanNamedTypes::writeLimit, PlanNamedTypes::readLimit), of(LogicalPlan.class, LocalRelation.class, (out, p) -> p.writeTo(out), LocalRelation::new), @@ -272,9 +274,7 @@ static EnrichExec readEnrichExec(PlanStreamInput in) throws IOException { final PhysicalPlan child = in.readPhysicalPlanNode(); final NamedExpression matchField = in.readNamedWriteable(NamedExpression.class); final String policyName = in.readString(); - final String matchType = (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_EXTENDED_ENRICH_TYPES)) - ? in.readString() - : "match"; + final String matchType = (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) ? in.readString() : "match"; final String policyMatchField = in.readString(); final Map concreteIndices; final Enrich.Mode mode; @@ -307,7 +307,7 @@ static void writeEnrichExec(PlanStreamOutput out, EnrichExec enrich) throws IOEx out.writePhysicalPlanNode(enrich.child()); out.writeNamedWriteable(enrich.matchField()); out.writeString(enrich.policyName()); - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_EXTENDED_ENRICH_TYPES)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeString(enrich.matchType()); } out.writeString(enrich.policyMatchField()); @@ -392,7 +392,7 @@ static FragmentExec readFragmentExec(PlanStreamInput in) throws IOException { in.readLogicalPlanNode(), in.readOptionalNamedWriteable(QueryBuilder.class), in.readOptionalVInt(), - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_REDUCER_NODE_FRAGMENT) ? in.readOptionalPhysicalPlanNode() : null + in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) ? in.readOptionalPhysicalPlanNode() : null ); } @@ -401,7 +401,7 @@ static void writeFragmentExec(PlanStreamOutput out, FragmentExec fragmentExec) t out.writeLogicalPlanNode(fragmentExec.fragment()); out.writeOptionalNamedWriteable(fragmentExec.esFilter()); out.writeOptionalVInt(fragmentExec.estimatedRowSize()); - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_REDUCER_NODE_FRAGMENT)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { out.writeOptionalPhysicalPlanNode(fragmentExec.reducer()); } } @@ -565,8 +565,7 @@ static void writeEsRelation(PlanStreamOutput out, EsRelation relation) throws IO } private static boolean supportingEsSourceOptions(TransportVersion version) { - return version.onOrAfter(TransportVersions.ESQL_ES_SOURCE_OPTIONS) - && version.before(TransportVersions.ESQL_REMOVE_ES_SOURCE_OPTIONS); + return version.onOrAfter(TransportVersions.V_8_14_0) && version.before(TransportVersions.ESQL_REMOVE_ES_SOURCE_OPTIONS); } private static void readEsSourceOptions(PlanStreamInput in) throws IOException { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java index 9a2ae742c2feb..edb8c0cde6052 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java @@ -58,6 +58,13 @@ import static org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer.operators; import static org.elasticsearch.xpack.esql.optimizer.rules.OptimizerRules.TransformDirection.UP; +/** + *

    This class is part of the planner. Data node level logical optimizations. At this point we have access to + * {@link org.elasticsearch.xpack.esql.stats.SearchStats} which provides access to metadata about the index.

    + * + *

    NB: This class also reapplies all the rules from {@link LogicalPlanOptimizer#operators()} and {@link LogicalPlanOptimizer#cleanup()} + *

    + */ public class LocalLogicalPlanOptimizer extends ParameterizedRuleExecutor { public LocalLogicalPlanOptimizer(LocalLogicalOptimizerContext localLogicalOptimizerContext) { @@ -140,7 +147,8 @@ else if (plan instanceof Project project) { Map nullLiteral = Maps.newLinkedHashMapWithExpectedSize(DataType.types().size()); for (NamedExpression projection : projections) { - if (projection instanceof FieldAttribute f && stats.exists(f.qualifiedName()) == false) { + // Do not use the attribute name, this can deviate from the field name for union types. + if (projection instanceof FieldAttribute f && stats.exists(f.fieldName()) == false) { DataType dt = f.dataType(); Alias nullAlias = nullLiteral.get(f.dataType()); // save the first field as null (per datatype) @@ -170,7 +178,8 @@ else if (plan instanceof Project project) { || plan instanceof TopN) { plan = plan.transformExpressionsOnlyUp( FieldAttribute.class, - f -> stats.exists(f.qualifiedName()) ? f : Literal.of(f, null) + // Do not use the attribute name, this can deviate from the field name for union types. + f -> stats.exists(f.fieldName()) ? f : Literal.of(f, null) ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java index c03dc46216621..b9be9b7ad029e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java @@ -95,6 +95,10 @@ import static org.elasticsearch.xpack.esql.optimizer.rules.OptimizerRules.TransformDirection.UP; import static org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.StatsType.COUNT; +/** + * Manages field extraction and pushing parts of the query into Lucene. (Query elements that are not pushed into Lucene are executed via + * the compute engine) + */ public class LocalPhysicalPlanOptimizer extends ParameterizedRuleExecutor { public static final EsqlTranslatorHandler TRANSLATOR_HANDLER = new EsqlTranslatorHandler(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index 50819b8ee7480..96be1249a76ee 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -15,10 +15,15 @@ import org.elasticsearch.xpack.esql.core.expression.AttributeMap; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.NameId; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.Order; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.rule.ParameterizedRule; import org.elasticsearch.xpack.esql.core.rule.ParameterizedRuleExecutor; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.optimizer.rules.AddDefaultTopN; import org.elasticsearch.xpack.esql.optimizer.rules.BooleanFunctionEqualsElimination; import org.elasticsearch.xpack.esql.optimizer.rules.BooleanSimplification; @@ -64,6 +69,7 @@ import org.elasticsearch.xpack.esql.optimizer.rules.SubstituteSpatialSurrogates; import org.elasticsearch.xpack.esql.optimizer.rules.SubstituteSurrogates; import org.elasticsearch.xpack.esql.optimizer.rules.TranslateMetricsAggregate; +import org.elasticsearch.xpack.esql.plan.GeneratingPlan; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.OrderBy; @@ -71,16 +77,40 @@ import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import static java.util.Arrays.asList; import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputExpressions; +/** + *

    This class is part of the planner

    + *

    Global optimizations based strictly on the structure of the query (i.e. not factoring in information about the backing indices). + * The bulk of query transformations happen in this step.

    + * + *

    Global optimizations based strictly on the structure of the query (i.e. not factoring in information about the backing indices). The + * bulk of query transformations happen in this step. This has three important sub-phases:

    + *
      + *
    • The {@link LogicalPlanOptimizer#substitutions()} phase rewrites things to expand out shorthand in the syntax. For example, + * a nested expression embedded in a stats gets replaced with an eval followed by a stats, followed by another eval. This phase + * also applies surrogates, such as replacing an average with a sum divided by a count.
      • + *
    • {@link LogicalPlanOptimizer#operators()} (NB: The word "operator" is extremely overloaded and referrers to many different + * things.) transform the tree in various different ways. This includes folding (i.e. computing constant expressions at parse + * time), combining expressions, dropping redundant clauses, and some normalization such as putting literals on the right whenever + * possible. These rules are run in a loop until none of the rules make any changes to the plan (there is also a safety shut off + * after many iterations, although hitting that is considered a bug)
      • + *
    • {@link LogicalPlanOptimizer#cleanup()} Which can replace sorts+limit with a TopN
      • + *
    + * + *

    Note that the {@link LogicalPlanOptimizer#operators()} and {@link LogicalPlanOptimizer#cleanup()} steps are reapplied at the + * {@link LocalLogicalPlanOptimizer} layer.

    + */ public class LogicalPlanOptimizer extends ParameterizedRuleExecutor { private final LogicalVerifier verifier = LogicalVerifier.INSTANCE; @@ -89,6 +119,34 @@ public LogicalPlanOptimizer(LogicalOptimizerContext optimizerContext) { super(optimizerContext); } + public static String temporaryName(Expression inner, Expression outer, int suffix) { + String in = toString(inner); + String out = toString(outer); + return rawTemporaryName(in, out, String.valueOf(suffix)); + } + + public static String locallyUniqueTemporaryName(String inner, String outer) { + return FieldAttribute.SYNTHETIC_ATTRIBUTE_NAME_PREFIX + inner + "$" + outer + "$" + new NameId(); + } + + public static String rawTemporaryName(String inner, String outer, String suffix) { + return FieldAttribute.SYNTHETIC_ATTRIBUTE_NAME_PREFIX + inner + "$" + outer + "$" + suffix; + } + + static String toString(Expression ex) { + return ex instanceof AggregateFunction af ? af.functionName() : extractString(ex); + } + + static String extractString(Expression ex) { + return ex instanceof NamedExpression ne ? ne.name() : limitToString(ex.sourceText()).replace(' ', '_'); + } + + static int TO_STRING_LIMIT = 16; + + static String limitToString(String string) { + return string.length() > TO_STRING_LIMIT ? string.substring(0, TO_STRING_LIMIT - 1) + ">" : string; + } + public LogicalPlan optimize(LogicalPlan verified) { var optimized = execute(verified); @@ -150,7 +208,7 @@ protected static Batch operators() { new PropagateNullable(), new BooleanFunctionEqualsElimination(), new CombineDisjunctionsToIn(), - new SimplifyComparisonsArithmetics(EsqlDataTypes::areCompatible), + new SimplifyComparisonsArithmetics(DataType::areCompatible), // prune/elimination new PruneFilters(), new PruneColumns(), @@ -189,35 +247,26 @@ public static LogicalPlan skipPlan(UnaryPlan plan, LocalSupplier supplier) { /** * Pushes LogicalPlans which generate new attributes (Eval, Grok/Dissect, Enrich), past OrderBys and Projections. - * Although it seems arbitrary whether the OrderBy or the Eval is executed first, this transformation ensures that OrderBys only - * separated by an eval can be combined by PushDownAndCombineOrderBy. - * - * E.g.: - * - * ... | sort a | eval x = b + 1 | sort x - * - * becomes - * - * ... | eval x = b + 1 | sort a | sort x - * - * Ordering the Evals before the OrderBys has the advantage that it's always possible to order the plans like this. + * Although it seems arbitrary whether the OrderBy or the generating plan is executed first, this transformation ensures that OrderBys + * only separated by e.g. an Eval can be combined by {@link PushDownAndCombineOrderBy}. + *

    + * E.g. {@code ... | sort a | eval x = b + 1 | sort x} becomes {@code ... | eval x = b + 1 | sort a | sort x} + *

    + * Ordering the generating plans before the OrderBys has the advantage that it's always possible to order the plans like this. * E.g., in the example above it would not be possible to put the eval after the two orderBys. - * - * In case one of the Eval's fields would shadow the orderBy's attributes, we rename the attribute first. - * - * E.g. - * - * ... | sort a | eval a = b + 1 | ... - * - * becomes - * - * ... | eval $$a = a | eval a = b + 1 | sort $$a | drop $$a + *

    + * In case one of the generating plan's attributes would shadow the OrderBy's attributes, we alias the generated attribute first. + *

    + * E.g. {@code ... | sort a | eval a = b + 1 | ...} becomes {@code ... | eval $$a = a | eval a = b + 1 | sort $$a | drop $$a ...} + *

    + * In case the generating plan's attributes would shadow the Project's attributes, we rename the generated attributes in place. + *

    + * E.g. {@code ... | rename a as z | eval a = b + 1 | ...} becomes {@code ... eval $$a = b + 1 | rename a as z, $$a as a ...} */ - public static LogicalPlan pushGeneratingPlanPastProjectAndOrderBy(UnaryPlan generatingPlan, List generatedAttributes) { + public static > LogicalPlan pushGeneratingPlanPastProjectAndOrderBy(Plan generatingPlan) { LogicalPlan child = generatingPlan.child(); - if (child instanceof OrderBy orderBy) { - Set evalFieldNames = new LinkedHashSet<>(Expressions.names(generatedAttributes)); + Set evalFieldNames = new LinkedHashSet<>(Expressions.names(generatingPlan.generatedAttributes())); // Look for attributes in the OrderBy's expressions and create aliases with temporary names for them. AttributeReplacement nonShadowedOrders = renameAttributesInExpressions(evalFieldNames, orderBy.order()); @@ -238,9 +287,66 @@ public static LogicalPlan pushGeneratingPlanPastProjectAndOrderBy(UnaryPlan gene } return orderBy.replaceChild(generatingPlan.replaceChild(orderBy.child())); - } else if (child instanceof Project) { - var projectWithEvalChild = pushDownPastProject(generatingPlan); - return projectWithEvalChild.withProjections(mergeOutputExpressions(generatedAttributes, projectWithEvalChild.projections())); + } else if (child instanceof Project project) { + // We need to account for attribute shadowing: a rename might rely on a name generated in an Eval/Grok/Dissect/Enrich. + // E.g. in: + // + // Eval[[2 * x{f}#1 AS y]] + // \_Project[[x{f}#1, y{f}#2, y{f}#2 AS z]] + // + // Just moving the Eval down breaks z because we shadow y{f}#2. + // Instead, we use a different alias in the Eval, eventually renaming back to y: + // + // Project[[x{f}#1, y{f}#2 as z, $$y{r}#3 as y]] + // \_Eval[[2 * x{f}#1 as $$y]] + + List generatedAttributes = generatingPlan.generatedAttributes(); + + @SuppressWarnings("unchecked") + Plan generatingPlanWithResolvedExpressions = (Plan) resolveRenamesFromProject(generatingPlan, project); + + Set namesReferencedInRenames = new HashSet<>(); + for (NamedExpression ne : project.projections()) { + if (ne instanceof Alias as) { + namesReferencedInRenames.addAll(as.child().references().names()); + } + } + Map renameGeneratedAttributeTo = newNamesForConflictingAttributes( + generatingPlan.generatedAttributes(), + namesReferencedInRenames + ); + List newNames = generatedAttributes.stream() + .map(attr -> renameGeneratedAttributeTo.getOrDefault(attr.name(), attr.name())) + .toList(); + Plan generatingPlanWithRenamedAttributes = generatingPlanWithResolvedExpressions.withGeneratedNames(newNames); + + // Put the project at the top, but include the generated attributes. + // Any generated attributes that had to be renamed need to be re-renamed to their original names. + List generatedAttributesRenamedToOriginal = new ArrayList<>(generatedAttributes.size()); + List renamedGeneratedAttributes = generatingPlanWithRenamedAttributes.generatedAttributes(); + for (int i = 0; i < generatedAttributes.size(); i++) { + Attribute originalAttribute = generatedAttributes.get(i); + Attribute renamedAttribute = renamedGeneratedAttributes.get(i); + if (originalAttribute.name().equals(renamedAttribute.name())) { + generatedAttributesRenamedToOriginal.add(renamedAttribute); + } else { + generatedAttributesRenamedToOriginal.add( + new Alias( + originalAttribute.source(), + originalAttribute.name(), + originalAttribute.qualifier(), + renamedAttribute, + originalAttribute.id(), + originalAttribute.synthetic() + ) + ); + } + } + + Project projectWithGeneratingChild = project.replaceChild(generatingPlanWithRenamedAttributes.replaceChild(project.child())); + return projectWithGeneratingChild.withProjections( + mergeOutputExpressions(generatedAttributesRenamedToOriginal, projectWithGeneratingChild.projections()) + ); } return generatingPlan; @@ -264,8 +370,9 @@ private static AttributeReplacement renameAttributesInExpressions( rewrittenExpressions.add(expr.transformUp(Attribute.class, attr -> { if (attributeNamesToRename.contains(attr.name())) { Alias renamedAttribute = aliasesForReplacedAttributes.computeIfAbsent(attr, a -> { - String tempName = SubstituteSurrogates.rawTemporaryName(a.name(), "temp_name", a.id().toString()); + String tempName = locallyUniqueTemporaryName(a.name(), "temp_name"); // TODO: this should be synthetic + // blocked on https://github.com/elastic/elasticsearch/issues/98703 return new Alias(a.source(), tempName, null, a, null, false); }); return renamedAttribute.toAttribute(); @@ -278,16 +385,28 @@ private static AttributeReplacement renameAttributesInExpressions( return new AttributeReplacement(rewrittenExpressions, aliasesForReplacedAttributes); } + private static Map newNamesForConflictingAttributes( + List potentiallyConflictingAttributes, + Set reservedNames + ) { + if (reservedNames.isEmpty()) { + return Map.of(); + } + + Map renameAttributeTo = new HashMap<>(); + for (Attribute attr : potentiallyConflictingAttributes) { + String name = attr.name(); + if (reservedNames.contains(name)) { + renameAttributeTo.putIfAbsent(name, locallyUniqueTemporaryName(name, "temp_name")); + } + } + + return renameAttributeTo; + } + public static Project pushDownPastProject(UnaryPlan parent) { if (parent.child() instanceof Project project) { - AttributeMap.Builder aliasBuilder = AttributeMap.builder(); - project.forEachExpression(Alias.class, a -> aliasBuilder.put(a.toAttribute(), a.child())); - var aliases = aliasBuilder.build(); - - var expressionsWithResolvedAliases = (UnaryPlan) parent.transformExpressionsOnly( - ReferenceAttribute.class, - r -> aliases.resolve(r, r) - ); + UnaryPlan expressionsWithResolvedAliases = resolveRenamesFromProject(parent, project); return project.replaceChild(expressionsWithResolvedAliases.replaceChild(project.child())); } else { @@ -295,6 +414,14 @@ public static Project pushDownPastProject(UnaryPlan parent) { } } + private static UnaryPlan resolveRenamesFromProject(UnaryPlan plan, Project project) { + AttributeMap.Builder aliasBuilder = AttributeMap.builder(); + project.forEachExpression(Alias.class, a -> aliasBuilder.put(a.toAttribute(), a.child())); + var aliases = aliasBuilder.build(); + + return (UnaryPlan) plan.transformExpressionsOnly(ReferenceAttribute.class, r -> aliases.resolve(r, r)); + } + public abstract static class ParameterizedOptimizerRule extends ParameterizedRule< SubPlan, LogicalPlan, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java index bff76fb1a706e..90581e8bfeaaa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java @@ -8,16 +8,17 @@ package org.elasticsearch.xpack.esql.optimizer; import org.elasticsearch.xpack.esql.common.Failures; +import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.plan.QueryPlan; +import org.elasticsearch.xpack.esql.plan.GeneratingPlan; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; -import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.MvExpand; -import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; import org.elasticsearch.xpack.esql.plan.logical.Row; import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; @@ -36,6 +37,9 @@ import org.elasticsearch.xpack.esql.plan.physical.RowExec; import org.elasticsearch.xpack.esql.plan.physical.ShowExec; +import java.util.HashSet; +import java.util.Set; + import static org.elasticsearch.xpack.esql.common.Failure.fail; class OptimizerRules { @@ -49,9 +53,24 @@ void checkPlan(P p, Failures failures) { AttributeSet input = p.inputSet(); AttributeSet generated = generates(p); AttributeSet missing = refs.subtract(input).subtract(generated); - if (missing.size() > 0) { + if (missing.isEmpty() == false) { failures.add(fail(p, "Plan [{}] optimized incorrectly due to missing references {}", p.nodeString(), missing)); } + + Set outputAttributeNames = new HashSet<>(); + Set outputAttributeIds = new HashSet<>(); + for (Attribute outputAttr : p.output()) { + if (outputAttributeNames.add(outputAttr.name()) == false || outputAttributeIds.add(outputAttr.id()) == false) { + failures.add( + fail( + p, + "Plan [{}] optimized incorrectly due to duplicate output attribute {}", + p.nodeString(), + outputAttr.toString() + ) + ); + } + } } protected AttributeSet references(P p) { @@ -83,18 +102,12 @@ protected AttributeSet generates(LogicalPlan logicalPlan) { || logicalPlan instanceof Aggregate) { return logicalPlan.outputSet(); } - if (logicalPlan instanceof Eval eval) { - return new AttributeSet(Expressions.asAttributes(eval.fields())); - } - if (logicalPlan instanceof RegexExtract extract) { - return new AttributeSet(extract.extractedFields()); + if (logicalPlan instanceof GeneratingPlan generating) { + return new AttributeSet(generating.generatedAttributes()); } if (logicalPlan instanceof MvExpand mvExpand) { return new AttributeSet(mvExpand.expanded()); } - if (logicalPlan instanceof Enrich enrich) { - return new AttributeSet(Expressions.asAttributes(enrich.enrichFields())); - } return AttributeSet.EMPTY; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java index e9fd6a713945c..4237852551e8a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java @@ -46,8 +46,8 @@ import static java.util.Collections.singletonList; /** - * Performs global (coordinator) optimization of the physical plan. - * Local (data-node) optimizations occur later by operating just on a plan fragment (subplan). + * This class is part of the planner. Performs global (coordinator) optimization of the physical plan. Local (data-node) optimizations + * occur later by operating just on a plan {@link FragmentExec} (subplan). */ public class PhysicalPlanOptimizer extends ParameterizedRuleExecutor { private static final Iterable> rules = initializeRules(true); @@ -122,7 +122,9 @@ public PhysicalPlan apply(PhysicalPlan plan) { if (p instanceof HashJoinExec join) { attributes.removeAll(join.addedFields()); for (Attribute rhs : join.rightFields()) { - attributes.remove(rhs); + if (join.leftFields().stream().anyMatch(x -> x.semanticEquals(rhs)) == false) { + attributes.remove(rhs); + } } } if (p instanceof EnrichExec ee) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/BooleanSimplification.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/BooleanSimplification.java index 2a3f7fb9d1244..6cb5bb29b5dd4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/BooleanSimplification.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/BooleanSimplification.java @@ -8,14 +8,155 @@ package org.elasticsearch.xpack.esql.optimizer.rules; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; +import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryPredicate; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; +import org.elasticsearch.xpack.esql.core.type.DataType; -public final class BooleanSimplification extends OptimizerRules.BooleanSimplification { +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE; +import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE; +import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.combineAnd; +import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.combineOr; +import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.inCommon; +import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.splitAnd; +import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.splitOr; +import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.subtract; +import static org.elasticsearch.xpack.esql.core.util.CollectionUtils.combine; + +public final class BooleanSimplification extends OptimizerRules.OptimizerExpressionRule { public BooleanSimplification() { - super(); + super(OptimizerRules.TransformDirection.UP); } @Override + public Expression rule(ScalarFunction e) { + if (e instanceof And || e instanceof Or) { + return simplifyAndOr((BinaryPredicate) e); + } + if (e instanceof Not) { + return simplifyNot((Not) e); + } + + return e; + } + + private static Expression simplifyAndOr(BinaryPredicate bc) { + Expression l = bc.left(); + Expression r = bc.right(); + + if (bc instanceof And) { + if (TRUE.equals(l)) { + return r; + } + if (TRUE.equals(r)) { + return l; + } + + if (FALSE.equals(l) || FALSE.equals(r)) { + return new Literal(bc.source(), Boolean.FALSE, DataType.BOOLEAN); + } + if (l.semanticEquals(r)) { + return l; + } + + // + // common factor extraction -> (a || b) && (a || c) => a || (b && c) + // + List leftSplit = splitOr(l); + List rightSplit = splitOr(r); + + List common = inCommon(leftSplit, rightSplit); + if (common.isEmpty()) { + return bc; + } + List lDiff = subtract(leftSplit, common); + List rDiff = subtract(rightSplit, common); + // (a || b || c || ... ) && (a || b) => (a || b) + if (lDiff.isEmpty() || rDiff.isEmpty()) { + return combineOr(common); + } + // (a || b || c || ... ) && (a || b || d || ... ) => ((c || ...) && (d || ...)) || a || b + Expression combineLeft = combineOr(lDiff); + Expression combineRight = combineOr(rDiff); + return combineOr(combine(common, new And(combineLeft.source(), combineLeft, combineRight))); + } + + if (bc instanceof Or) { + if (TRUE.equals(l) || TRUE.equals(r)) { + return new Literal(bc.source(), Boolean.TRUE, DataType.BOOLEAN); + } + + if (FALSE.equals(l)) { + return r; + } + if (FALSE.equals(r)) { + return l; + } + + if (l.semanticEquals(r)) { + return l; + } + + // + // common factor extraction -> (a && b) || (a && c) => a && (b || c) + // + List leftSplit = splitAnd(l); + List rightSplit = splitAnd(r); + + List common = inCommon(leftSplit, rightSplit); + if (common.isEmpty()) { + return bc; + } + List lDiff = subtract(leftSplit, common); + List rDiff = subtract(rightSplit, common); + // (a || b || c || ... ) && (a || b) => (a || b) + if (lDiff.isEmpty() || rDiff.isEmpty()) { + return combineAnd(common); + } + // (a || b || c || ... ) && (a || b || d || ... ) => ((c || ...) && (d || ...)) || a || b + Expression combineLeft = combineAnd(lDiff); + Expression combineRight = combineAnd(rDiff); + return combineAnd(combine(common, new Or(combineLeft.source(), combineLeft, combineRight))); + } + + // TODO: eliminate conjunction/disjunction + return bc; + + } + + @SuppressWarnings("rawtypes") + private Expression simplifyNot(Not n) { + Expression c = n.field(); + + if (TRUE.semanticEquals(c)) { + return new Literal(n.source(), Boolean.FALSE, DataType.BOOLEAN); + } + if (FALSE.semanticEquals(c)) { + return new Literal(n.source(), Boolean.TRUE, DataType.BOOLEAN); + } + + Expression negated = maybeSimplifyNegatable(c); + if (negated != null) { + return negated; + } + + if (c instanceof Not) { + return ((Not) c).field(); + } + + return n; + } + + /** + * @param e + * @return the negated expression or {@code null} if the parameter is not an instance of {@code Negatable} + */ protected Expression maybeSimplifyNegatable(Expression e) { return null; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToIn.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToIn.java index 2dc2f0e504303..42d4bf730a644 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToIn.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToIn.java @@ -71,9 +71,6 @@ public Expression rule(Or or) { } } else if (exp instanceof In in) { found.computeIfAbsent(in.value(), k -> new LinkedHashSet<>()).addAll(in.list()); - if (zoneId == null) { - zoneId = in.zoneId(); - } } else { ors.add(exp); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/FoldNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/FoldNull.java index 6e01811b8527c..0e864c13ca6aa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/FoldNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/FoldNull.java @@ -7,10 +7,36 @@ package org.elasticsearch.xpack.esql.optimizer.rules; +import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.Nullability; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; + +public class FoldNull extends OptimizerRules.OptimizerExpressionRule { + + public FoldNull() { + super(OptimizerRules.TransformDirection.UP); + } -public class FoldNull extends OptimizerRules.FoldNull { @Override + public Expression rule(Expression e) { + Expression result = tryReplaceIsNullIsNotNull(e); + if (result != e) { + return result; + } else if (e instanceof In in) { + if (Expressions.isNull(in.value())) { + return Literal.of(in, null); + } + } else if (e instanceof Alias == false + && e.nullable() == Nullability.TRUE + && Expressions.anyMatch(e.children(), Expressions::isNull)) { + return Literal.of(e, null); + } + return e; + } + protected Expression tryReplaceIsNullIsNotNull(Expression e) { return e; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/OptimizerRules.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/OptimizerRules.java index 6f6260fd0de27..6bc0d9016eb9f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/OptimizerRules.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/OptimizerRules.java @@ -6,305 +6,12 @@ */ package org.elasticsearch.xpack.esql.optimizer.rules; -import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.Expressions; -import org.elasticsearch.xpack.esql.core.expression.Literal; -import org.elasticsearch.xpack.esql.core.expression.Nullability; -import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; -import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryPredicate; -import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; -import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates; -import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; -import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; -import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; -import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; -import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.In; import org.elasticsearch.xpack.esql.core.rule.Rule; -import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.ReflectionUtils; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; -import java.util.function.BiFunction; - -import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE; -import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE; -import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.combineAnd; -import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.combineOr; -import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.inCommon; -import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.splitAnd; -import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.splitOr; -import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.subtract; -import static org.elasticsearch.xpack.esql.core.util.CollectionUtils.combine; - public final class OptimizerRules { - public static class BooleanSimplification extends OptimizerExpressionRule { - - public BooleanSimplification() { - super(TransformDirection.UP); - } - - @Override - public Expression rule(ScalarFunction e) { - if (e instanceof And || e instanceof Or) { - return simplifyAndOr((BinaryPredicate) e); - } - if (e instanceof Not) { - return simplifyNot((Not) e); - } - - return e; - } - - private static Expression simplifyAndOr(BinaryPredicate bc) { - Expression l = bc.left(); - Expression r = bc.right(); - - if (bc instanceof And) { - if (TRUE.equals(l)) { - return r; - } - if (TRUE.equals(r)) { - return l; - } - - if (FALSE.equals(l) || FALSE.equals(r)) { - return new Literal(bc.source(), Boolean.FALSE, DataType.BOOLEAN); - } - if (l.semanticEquals(r)) { - return l; - } - - // - // common factor extraction -> (a || b) && (a || c) => a || (b && c) - // - List leftSplit = splitOr(l); - List rightSplit = splitOr(r); - - List common = inCommon(leftSplit, rightSplit); - if (common.isEmpty()) { - return bc; - } - List lDiff = subtract(leftSplit, common); - List rDiff = subtract(rightSplit, common); - // (a || b || c || ... ) && (a || b) => (a || b) - if (lDiff.isEmpty() || rDiff.isEmpty()) { - return combineOr(common); - } - // (a || b || c || ... ) && (a || b || d || ... ) => ((c || ...) && (d || ...)) || a || b - Expression combineLeft = combineOr(lDiff); - Expression combineRight = combineOr(rDiff); - return combineOr(combine(common, new And(combineLeft.source(), combineLeft, combineRight))); - } - - if (bc instanceof Or) { - if (TRUE.equals(l) || TRUE.equals(r)) { - return new Literal(bc.source(), Boolean.TRUE, DataType.BOOLEAN); - } - - if (FALSE.equals(l)) { - return r; - } - if (FALSE.equals(r)) { - return l; - } - - if (l.semanticEquals(r)) { - return l; - } - - // - // common factor extraction -> (a && b) || (a && c) => a && (b || c) - // - List leftSplit = splitAnd(l); - List rightSplit = splitAnd(r); - - List common = inCommon(leftSplit, rightSplit); - if (common.isEmpty()) { - return bc; - } - List lDiff = subtract(leftSplit, common); - List rDiff = subtract(rightSplit, common); - // (a || b || c || ... ) && (a || b) => (a || b) - if (lDiff.isEmpty() || rDiff.isEmpty()) { - return combineAnd(common); - } - // (a || b || c || ... ) && (a || b || d || ... ) => ((c || ...) && (d || ...)) || a || b - Expression combineLeft = combineAnd(lDiff); - Expression combineRight = combineAnd(rDiff); - return combineAnd(combine(common, new Or(combineLeft.source(), combineLeft, combineRight))); - } - - // TODO: eliminate conjunction/disjunction - return bc; - } - - @SuppressWarnings("rawtypes") - private Expression simplifyNot(Not n) { - Expression c = n.field(); - - if (TRUE.semanticEquals(c)) { - return new Literal(n.source(), Boolean.FALSE, DataType.BOOLEAN); - } - if (FALSE.semanticEquals(c)) { - return new Literal(n.source(), Boolean.TRUE, DataType.BOOLEAN); - } - - Expression negated = maybeSimplifyNegatable(c); - if (negated != null) { - return negated; - } - - if (c instanceof Not) { - return ((Not) c).field(); - } - - return n; - } - - /** - * @param e - * @return the negated expression or {@code null} if the parameter is not an instance of {@code Negatable} - */ - protected Expression maybeSimplifyNegatable(Expression e) { - if (e instanceof Negatable) { - return ((Negatable) e).negate(); - } - return null; - } - } - - public static class FoldNull extends OptimizerExpressionRule { - - public FoldNull() { - super(TransformDirection.UP); - } - - @Override - public Expression rule(Expression e) { - Expression result = tryReplaceIsNullIsNotNull(e); - if (result != e) { - return result; - } else if (e instanceof In in) { - if (Expressions.isNull(in.value())) { - return Literal.of(in, null); - } - } else if (e instanceof Alias == false - && e.nullable() == Nullability.TRUE - && Expressions.anyMatch(e.children(), Expressions::isNull)) { - return Literal.of(e, null); - } - return e; - } - - protected Expression tryReplaceIsNullIsNotNull(Expression e) { - if (e instanceof IsNotNull isnn) { - if (isnn.field().nullable() == Nullability.FALSE) { - return new Literal(e.source(), Boolean.TRUE, DataType.BOOLEAN); - } - } else if (e instanceof IsNull isn) { - if (isn.field().nullable() == Nullability.FALSE) { - return new Literal(e.source(), Boolean.FALSE, DataType.BOOLEAN); - } - } - return e; - } - } - - // a IS NULL AND a IS NOT NULL -> FALSE - // a IS NULL AND a > 10 -> a IS NULL and FALSE - // can be extended to handle null conditions where available - public static class PropagateNullable extends OptimizerExpressionRule { - - public PropagateNullable() { - super(TransformDirection.DOWN); - } - - @Override - public Expression rule(And and) { - List splits = Predicates.splitAnd(and); - - Set nullExpressions = new LinkedHashSet<>(); - Set notNullExpressions = new LinkedHashSet<>(); - List others = new LinkedList<>(); - - // first find isNull/isNotNull - for (Expression ex : splits) { - if (ex instanceof IsNull isn) { - nullExpressions.add(isn.field()); - } else if (ex instanceof IsNotNull isnn) { - notNullExpressions.add(isnn.field()); - } - // the rest - else { - others.add(ex); - } - } - - // check for is isNull and isNotNull --> FALSE - if (Sets.haveNonEmptyIntersection(nullExpressions, notNullExpressions)) { - return Literal.of(and, Boolean.FALSE); - } - - // apply nullability across relevant/matching expressions - - // first against all nullable expressions - // followed by all not-nullable expressions - boolean modified = replace(nullExpressions, others, splits, this::nullify); - modified |= replace(notNullExpressions, others, splits, this::nonNullify); - if (modified) { - // reconstruct the expression - return Predicates.combineAnd(splits); - } - return and; - } - - /** - * Replace the given 'pattern' expressions against the target expression. - * If a match is found, the matching expression will be replaced by the replacer result - * or removed if null is returned. - */ - private static boolean replace( - Iterable pattern, - List target, - List originalExpressions, - BiFunction replacer - ) { - boolean modified = false; - for (Expression s : pattern) { - for (int i = 0; i < target.size(); i++) { - Expression t = target.get(i); - // identify matching expressions - if (t.anyMatch(s::semanticEquals)) { - Expression replacement = replacer.apply(t, s); - // if the expression has changed, replace it - if (replacement != t) { - modified = true; - target.set(i, replacement); - originalExpressions.replaceAll(e -> t.semanticEquals(e) ? replacement : e); - } - } - } - } - return modified; - } - - // default implementation nullifies all nullable expressions - protected Expression nullify(Expression exp, Expression nullExp) { - return exp.nullable() == Nullability.TRUE ? Literal.of(exp, null) : exp; - } - - // placeholder for non-null - protected Expression nonNullify(Expression exp, Expression nonNullExp) { - return exp; - } - } public abstract static class OptimizerRule extends Rule { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PropagateNullable.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PropagateNullable.java index 08c560c326e81..a56f0633fe286 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PropagateNullable.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PropagateNullable.java @@ -7,14 +7,105 @@ package org.elasticsearch.xpack.esql.optimizer.rules; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; +import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; +import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull; import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.LinkedList; import java.util.List; +import java.util.Set; +import java.util.function.BiFunction; + +// a IS NULL AND a IS NOT NULL -> FALSE +// a IS NULL AND a > 10 -> a IS NULL and FALSE +// can be extended to handle null conditions where available +public class PropagateNullable extends OptimizerRules.OptimizerExpressionRule { + + public PropagateNullable() { + super(OptimizerRules.TransformDirection.DOWN); + } + + @Override + public Expression rule(And and) { + List splits = Predicates.splitAnd(and); + + Set nullExpressions = new LinkedHashSet<>(); + Set notNullExpressions = new LinkedHashSet<>(); + List others = new LinkedList<>(); + + // first find isNull/isNotNull + for (Expression ex : splits) { + if (ex instanceof IsNull isn) { + nullExpressions.add(isn.field()); + } else if (ex instanceof IsNotNull isnn) { + notNullExpressions.add(isnn.field()); + } + // the rest + else { + others.add(ex); + } + } + + // check for is isNull and isNotNull --> FALSE + if (Sets.haveNonEmptyIntersection(nullExpressions, notNullExpressions)) { + return Literal.of(and, Boolean.FALSE); + } + + // apply nullability across relevant/matching expressions + + // first against all nullable expressions + // followed by all not-nullable expressions + boolean modified = replace(nullExpressions, others, splits, this::nullify); + modified |= replace(notNullExpressions, others, splits, this::nonNullify); + if (modified) { + // reconstruct the expression + return Predicates.combineAnd(splits); + } + return and; + } + + /** + * Replace the given 'pattern' expressions against the target expression. + * If a match is found, the matching expression will be replaced by the replacer result + * or removed if null is returned. + */ + private static boolean replace( + Iterable pattern, + List target, + List originalExpressions, + BiFunction replacer + ) { + boolean modified = false; + for (Expression s : pattern) { + for (int i = 0; i < target.size(); i++) { + Expression t = target.get(i); + // identify matching expressions + if (t.anyMatch(s::semanticEquals)) { + Expression replacement = replacer.apply(t, s); + // if the expression has changed, replace it + if (replacement != t) { + modified = true; + target.set(i, replacement); + originalExpressions.replaceAll(e -> t.semanticEquals(e) ? replacement : e); + } + } + } + } + return modified; + } + + // placeholder for non-null + protected Expression nonNullify(Expression exp, Expression nonNullExp) { + return exp; + } -public class PropagateNullable extends OptimizerRules.PropagateNullable { protected Expression nullify(Expression exp, Expression nullExp) { if (exp instanceof Coalesce) { List newChildren = new ArrayList<>(exp.children()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEnrich.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEnrich.java index 7185f63964c34..5e6def37cbf04 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEnrich.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEnrich.java @@ -11,11 +11,9 @@ import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; -import static org.elasticsearch.xpack.esql.core.expression.Expressions.asAttributes; - public final class PushDownEnrich extends OptimizerRules.OptimizerRule { @Override protected LogicalPlan rule(Enrich en) { - return LogicalPlanOptimizer.pushGeneratingPlanPastProjectAndOrderBy(en, asAttributes(en.enrichFields())); + return LogicalPlanOptimizer.pushGeneratingPlanPastProjectAndOrderBy(en); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEval.java index 92c25a60bba77..1f5fd072f267c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEval.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEval.java @@ -11,11 +11,9 @@ import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; -import static org.elasticsearch.xpack.esql.core.expression.Expressions.asAttributes; - public final class PushDownEval extends OptimizerRules.OptimizerRule { @Override protected LogicalPlan rule(Eval eval) { - return LogicalPlanOptimizer.pushGeneratingPlanPastProjectAndOrderBy(eval, asAttributes(eval.fields())); + return LogicalPlanOptimizer.pushGeneratingPlanPastProjectAndOrderBy(eval); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownRegexExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownRegexExtract.java index d24a61f89dd7f..3f64f47e11879 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownRegexExtract.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownRegexExtract.java @@ -14,6 +14,6 @@ public final class PushDownRegexExtract extends OptimizerRules.OptimizerRule { @Override protected LogicalPlan rule(RegexExtract re) { - return LogicalPlanOptimizer.pushGeneratingPlanPastProjectAndOrderBy(re, re.extractedFields()); + return LogicalPlanOptimizer.pushGeneratingPlanPastProjectAndOrderBy(re); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceOrderByExpressionWithEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceOrderByExpressionWithEval.java index 02fc98428f14a..749a1bc59127a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceOrderByExpressionWithEval.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceOrderByExpressionWithEval.java @@ -18,7 +18,7 @@ import java.util.ArrayList; import java.util.List; -import static org.elasticsearch.xpack.esql.optimizer.rules.SubstituteSurrogates.rawTemporaryName; +import static org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer.rawTemporaryName; public final class ReplaceOrderByExpressionWithEval extends OptimizerRules.OptimizerRule { private static int counter = 0; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsAggExpressionWithEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsAggExpressionWithEval.java index 31b543cd115df..6cd61d03f1ffa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsAggExpressionWithEval.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsAggExpressionWithEval.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.util.CollectionUtils; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; +import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; @@ -149,6 +150,6 @@ protected LogicalPlan rule(Aggregate aggregate) { } static String syntheticName(Expression expression, Expression af, int counter) { - return SubstituteSurrogates.temporaryName(expression, af, counter); + return LogicalPlanOptimizer.temporaryName(expression, af, counter); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsNestedExpressionWithEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsNestedExpressionWithEval.java index 0979b745a6607..5bae99e1204f3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsNestedExpressionWithEval.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsNestedExpressionWithEval.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction; +import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; @@ -140,6 +141,6 @@ protected LogicalPlan rule(Aggregate aggregate) { } static String syntheticName(Expression expression, AggregateFunction af, int counter) { - return SubstituteSurrogates.temporaryName(expression, af, counter); + return LogicalPlanOptimizer.temporaryName(expression, af, counter); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SubstituteSurrogates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SubstituteSurrogates.java index 2307f6324e942..fffda5622dd7e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SubstituteSurrogates.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SubstituteSurrogates.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.esql.expression.SurrogateExpression; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.expression.function.aggregate.Rate; +import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; @@ -79,7 +80,7 @@ protected LogicalPlan rule(Aggregate aggregate) { var attr = aggFuncToAttr.get(af); // the agg doesn't exist in the Aggregate, create an alias for it and save its attribute if (attr == null) { - var temporaryName = temporaryName(af, agg, counter[0]++); + var temporaryName = LogicalPlanOptimizer.temporaryName(af, agg, counter[0]++); // create a synthetic alias (so it doesn't clash with a user defined name) var newAlias = new Alias(agg.source(), temporaryName, null, af, null, true); attr = newAlias.toAttribute(); @@ -132,28 +133,4 @@ protected LogicalPlan rule(Aggregate aggregate) { return plan; } - - public static String temporaryName(Expression inner, Expression outer, int suffix) { - String in = toString(inner); - String out = toString(outer); - return rawTemporaryName(in, out, String.valueOf(suffix)); - } - - public static String rawTemporaryName(String inner, String outer, String suffix) { - return "$$" + inner + "$" + outer + "$" + suffix; - } - - static int TO_STRING_LIMIT = 16; - - static String toString(Expression ex) { - return ex instanceof AggregateFunction af ? af.functionName() : extractString(ex); - } - - static String extractString(Expression ex) { - return ex instanceof NamedExpression ne ? ne.name() : limitToString(ex.sourceText()).replace(' ', '_'); - } - - static String limitToString(String string) { - return string.length() > 16 ? string.substring(0, TO_STRING_LIMIT - 1) + ">" : string; - } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/package-info.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/package-info.java index 0d45ce10b1966..17d317bedbb6f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/package-info.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/package-info.java @@ -6,35 +6,183 @@ */ /** - * ES|QL Overview and Documentation Links + * The ES|QL query language. * - *

    Major Components

    + *

    Overview

    + * ES|QL is a typed query language which consists of many small languages separated by the {@code |} + * character. Like this: + * + * 
    {@code
+ *   FROM foo
+ * | WHERE a > 1
+ * | STATS m=MAX(j)
+ * | SORT m ASC
+ * | LIMIT 10
+ * }
    + * + *

    + * Here the {@code FROM}, {@code WHERE}, {@code STATS}, {@code SORT}, and {@code LIMIT} keywords + * enable the mini-language for selecting indices, filtering documents, calculate aggregates, + * sorting results, and limiting the number of results respectively. + *

    + * + *

    Language Design Goals

    + * In designing ES|QL we have some principals and rules of thumb: + *
      + *
    • Don't waste people's time
      • + *
    • Progress over perfection
      • + *
    • Design for Elasticsearch
      • + *
    • Be inspired by the best
      • + *
    + * + *

    Don't waste people's time

    + *
      + *
    • Queries should not fail at runtime. Instead we should return a + * {@link org.elasticsearch.xpack.esql.expression.function.Warnings warning} and {@code null}.
      • + *
    • It is ok to fail a query up front at analysis time. Just not after it's + * started.
      • + *
    • It is better if things can be made to work.
      • + *
    • But genuinely confusing requests require the query writing to make a choice.
      • + *
    + *

    + * As you can see this is a real tight rope, but we try to follow the rules above in order. Examples: + *

    + *
      + *
    • If {@link org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatetime TO_DATETIME} + * receives an invalid date at runtime, it emits a WARNING.
      • + *
    • If {@link org.elasticsearch.xpack.esql.expression.function.scalar.date.DateExtract DATE_EXTRACT} + * receives an invalid extract configuration at query parsing time it fails to start the query.
      • + *
    • {@link org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add 1 + 3.2} + * promotes both sides to a {@code double}.
      • + *
    • {@link org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add 1 + "32"} + * fails at query compile time and the query writer must decide to either write + * {@code CONCAT(TO_STRING(1), "32")} or {@code 1 + TO_INT("32")}.
      • + *
    + * + *

    Progress over perfection

    + *
      + *
    • Stability is super important for released features.
      • + *
    • But we need to experiment and get feedback. So mark features {@code experimental} when + * there's any question about how they should work.
      • + *
    • Experimental features shouldn't live forever because folks will get tired of waiting + * and use them in production anyway. We don't officially support them in production but + * we will feel bad if they break.
      • + *
    + * + *

    Design for Elasticsearch

    + * We must design the language for Elasticsearch, celebrating its advantages + * smoothing out its and quirks. + *
      + *
    • {@link org.elasticsearch.index.fielddata doc_values} sometimes sorts field values and + * sometimes sorts and removes duplicates. We couldn't hide this even if we want to and + * most folks are ok with it. ES|QL has to be useful in those cases.
      • + *
    • Multivalued fields are very easy to index in Elasticsearch so they should be easy to + * read in ES|QL. They should be easy to work with in ES|QL too, but we + * haven't gotten that far yet.
      • + *
    + * + *

    Be inspired by the best

    + * We'll frequently have lots of different choices on how to implement a feature. We should talk + * and figure out the best way for us, especially considering Elasticsearch's advantages and quirks. + * But we should also look to our data-access-forebears: *
      - *
    • {@link org.elasticsearch.compute} - The compute engine drives query execution + *
    • PostgreSQL is the + * GOAT SQL implementation. It's a joy + * to use for everything but dates. Use DB Fiddle + * to link to syntax examples.
      • + *
    • Oracle + * is pretty good about dates. It's fine about a lot of things but PostgreSQL is better.
      • + *
    • MS SQL Server + * has a silly name but it's documentation is wonderful.
      • + *
    • SPL + * is super familiar to our users and it's a piped query language.
      • + *
    + * + *

    Major Components

    + * + *

    Compute Engine

    + *{@link org.elasticsearch.compute} - The compute engine drives query execution *
      *
    • {@link org.elasticsearch.compute.data.Block} - fundamental unit of data. Operations vectorize over blocks.
      • *
    • {@link org.elasticsearch.compute.data.Page} - Data is broken up into pages (which are collections of blocks) to * manage size in memory
      • *
    - * - *
  6. {@link org.elasticsearch.xpack.esql.core} - Core Utility Classes + * + *

    Core Classes

    + * {@link org.elasticsearch.xpack.esql.core} - Core Classes *
      + *
    • {@link org.elasticsearch.xpack.esql.session.EsqlSession} - Connects all major components and contains the high-level code for + * query execution
      • *
    • {@link org.elasticsearch.xpack.esql.core.type.DataType} - ES|QL is a typed language, and all the supported data types * are listed in this collection.
      • *
    • {@link org.elasticsearch.xpack.esql.core.expression.Expression} - Expression is the basis for all functions in ES|QL, * but see also {@link org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper}
      • + *
    • {@link org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry} - Resolves function names to + * function implementations.
      • + *
    • {@link org.elasticsearch.xpack.esql.action.RestEsqlQueryAction Sync} and + * {@link org.elasticsearch.xpack.esql.action.RestEsqlAsyncQueryAction async} HTTP API entry points
      • + *
    • {@link org.elasticsearch.xpack.esql.plan.logical.Phased} - Marks a {@link org.elasticsearch.xpack.esql.plan.logical.LogicalPlan} + * node as requiring multiple ESQL executions to run.
      • + *
    + * + *

    Query Planner

    + *

    The query planner encompasses the logic of how to serve a query. Essentially, this covers everything from the output of the Antlr + * parser through to the actual computations and lucene operations.

    + *

    Two key concepts in the planner layer:

    + *
      + *
    • Logical vs Physical optimization - Logical optimizations refer to things that can be done strictly based on the structure + * of the query, while Physical optimizations take into account information about the index or indices the query will execute + * against
      • + *
    • Local vs non-local operations - "local" refers to operations happening on the data nodes, while non-local operations generally + * happen on the coordinating node and can apply to all participating nodes in the query
      • + *
    + *

    Query Planner Steps

    + *
      + *
    • {@link org.elasticsearch.xpack.esql.parser.LogicalPlanBuilder LogicalPlanBuilder} translates from Antlr data structures to our + * data structures
      • + *
    • {@link org.elasticsearch.xpack.esql.analysis.PreAnalyzer PreAnalyzer} finds involved indices
      • + *
    • {@link org.elasticsearch.xpack.esql.analysis.Analyzer Analyzer} resolves references
      • + *
    • {@link org.elasticsearch.xpack.esql.analysis.Verifier Verifier} does type checking
      • + *
    • {@link org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer LogicalPlanOptimizer} applies many optimizations
      • + *
    • {@link org.elasticsearch.xpack.esql.planner.Mapper Mapper} translates logical plans to phyisical plans
      • + *
    • {@link org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizer PhysicalPlanOptimizer} - decides what plan fragments to + * send to which data nodes
      • + *
    • {@link org.elasticsearch.xpack.esql.optimizer.LocalLogicalPlanOptimizer LocalLogicalPlanOptimizer} applies index-specific + * optimizations, and reapplies top level logical optimizations
      • + *
    • {@link org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizer LocalPhysicalPlanOptimizer} Lucene push down and + * similar
      • + *
    • {@link org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner LocalExecutionPlanner} Creates the compute engine objects + * to carry out the query
      • *
    - *
    7. - *
  7. org.elasticsearch.compute.gen - ES|QL generates code for evaluators, which are type-specific implementations of - * functions, designed to run over a {@link org.elasticsearch.compute.data.Block}
    8. - *
  8. {@link org.elasticsearch.xpack.esql.session.EsqlSession} - manages state across a query
    9. - *
  9. {@link org.elasticsearch.xpack.esql.expression.function.scalar} - Guide to writing scalar functions
    10. - *
  10. {@link org.elasticsearch.xpack.esql.expression.function.aggregate} - Guide to writing aggregation functions
    11. - *
  11. {@link org.elasticsearch.xpack.esql.analysis.Analyzer} - The first step in query processing
    12. - *
  12. {@link org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer} - Coordinator level logical optimizations
    13. - *
  13. {@link org.elasticsearch.xpack.esql.optimizer.LocalLogicalPlanOptimizer} - Data node level logical optimizations
    14. - *
  14. {@link org.elasticsearch.xpack.esql.action.RestEsqlQueryAction} - REST API entry point
    15. + * + * + *

    Guides

    + *
      + *
    • {@link org.elasticsearch.xpack.esql.expression.function.scalar Writing scalar functions}
      • + *
    • {@link org.elasticsearch.xpack.esql.expression.function.aggregate Writing aggregation functions}
      • *
    + * + *

    Code generation

    + * ES|QL uses two kinds of code generation which is uses mostly to + * monomorphize tight loops. That process would + * require a lot of copy-and-paste with small tweaks and some of us have copy-and-paste blindness so instead + * we use code generation. + *
      + *
    1. When possible we use StringTemplate to build + * Java files. These files typically look like {@code X-Blah.java.st} and are typically used for things + * like the different {@link org.elasticsearch.compute.data.Block} types and their subclasses and + * aggregation state. The templates themselves are easy to read and edit. This process is appropriate + * for cases where you just have to copy and paste something and change a few lines here and there. See + * {@code build.gradle} for the code generators.
      2. + *
    2. When that doesn't work, we use + * + * Annotation processing and JavaPoet to build the Java files. + * These files are typically the inner loops for {@link org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator} + * or {@link org.elasticsearch.compute.aggregation.AggregatorFunction}. The code generation is much more difficult + * to write and debug but much, much, much, much more flexible. The degree of control we have during this + * code generation is amazing but it is much harder to debug failures. See files in + * {@code org.elasticsearch.compute.gen} for the code generators.
      3. + *
    */ package org.elasticsearch.xpack.esql; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index 9769d286b484d..88279b65d2007 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -89,9 +89,22 @@ public abstract class ExpressionBuilder extends IdentifierBuilder { private int expressionDepth = 0; /** - * Maximum depth for nested expressions + * Maximum depth for nested expressions. + * Avoids StackOverflowErrors at parse time with very convoluted expressions, + * eg. EVAL x = sin(sin(sin(sin(sin(sin(sin(sin(sin(....sin(x)....) + * ANTLR parser is recursive, so the only way to prevent a StackOverflow is to detect how + * deep we are in the expression parsing and abort the query execution after a threshold + * + * This value is defined empirically, but the actual stack limit is highly + * dependent on the JVM and on the JIT. + * + * A value of 500 proved to be right below the stack limit, but it still triggered + * some CI failures (once every ~2000 iterations). see https://github.com/elastic/elasticsearch/issues/109846 + * Even though we didn't manage to reproduce the problem in real conditions, we decided + * to reduce the max allowed depth to 400 (that is still a pretty reasonable limit for real use cases) and be more safe. + * */ - public static final int MAX_EXPRESSION_DEPTH = 500; + public static final int MAX_EXPRESSION_DEPTH = 400; protected final QueryParams params; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index d1e0bdac0bf2f..6caab4abe8b38 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -26,7 +26,6 @@ import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.Order; -import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedStar; import org.elasticsearch.xpack.esql.core.parser.ParserUtils; @@ -41,8 +40,6 @@ import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Drop; import org.elasticsearch.xpack.esql.plan.logical.Enrich; -import org.elasticsearch.xpack.esql.plan.logical.EsqlAggregate; -import org.elasticsearch.xpack.esql.plan.logical.EsqlUnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Explain; import org.elasticsearch.xpack.esql.plan.logical.Filter; @@ -56,8 +53,10 @@ import org.elasticsearch.xpack.esql.plan.logical.OrderBy; import org.elasticsearch.xpack.esql.plan.logical.Rename; import org.elasticsearch.xpack.esql.plan.logical.Row; +import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.meta.MetaFunctions; import org.elasticsearch.xpack.esql.plan.logical.show.ShowInfo; +import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import java.util.ArrayList; import java.util.Arrays; @@ -74,9 +73,15 @@ import static org.elasticsearch.xpack.esql.core.parser.ParserUtils.source; import static org.elasticsearch.xpack.esql.core.parser.ParserUtils.typedParsing; import static org.elasticsearch.xpack.esql.core.parser.ParserUtils.visitList; +import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputExpressions; import static org.elasticsearch.xpack.esql.plan.logical.Enrich.Mode; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToInt; +/** + * Translates what we get back from Antlr into the data structures the rest of the planner steps will act on. Generally speaking, things + * which change the grammar will need to make changes here as well. + * + */ public class LogicalPlanBuilder extends ExpressionBuilder { private int queryDepth = 0; @@ -193,21 +198,20 @@ public PlanFactory visitDissectCommand(EsqlBaseParser.DissectCommandContext ctx) try { DissectParser parser = new DissectParser(pattern, appendSeparator); + Set referenceKeys = parser.referenceKeys(); - if (referenceKeys.size() > 0) { + if (referenceKeys.isEmpty() == false) { throw new ParsingException( src, "Reference keys not supported in dissect patterns: [%{*{}}]", referenceKeys.iterator().next() ); } - List keys = new ArrayList<>(); - for (var x : parser.outputKeys()) { - if (x.isEmpty() == false) { - keys.add(new ReferenceAttribute(src, x, DataType.KEYWORD)); - } - } - return new Dissect(src, p, expression(ctx.primaryExpression()), new Dissect.Parser(pattern, appendSeparator, parser), keys); + + Dissect.Parser esqlDissectParser = new Dissect.Parser(pattern, appendSeparator, parser); + List keys = esqlDissectParser.keyAttributes(src); + + return new Dissect(src, p, expression(ctx.primaryExpression()), esqlDissectParser, keys); } catch (DissectException e) { throw new ParsingException(src, "Invalid pattern for dissect: [{}]", pattern); } @@ -235,8 +239,9 @@ public Map visitCommandOptions(EsqlBaseParser.CommandOptionsCont } @Override + @SuppressWarnings("unchecked") public LogicalPlan visitRowCommand(EsqlBaseParser.RowCommandContext ctx) { - return new Row(source(ctx), visitFields(ctx.fields())); + return new Row(source(ctx), (List) (List) mergeOutputExpressions(visitFields(ctx.fields()), List.of())); } @Override @@ -271,13 +276,20 @@ public LogicalPlan visitFromCommand(EsqlBaseParser.FromCommandContext ctx) { } } } - return new EsqlUnresolvedRelation(source, table, Arrays.asList(metadataMap.values().toArray(Attribute[]::new)), IndexMode.STANDARD); + return new UnresolvedRelation( + source, + table, + false, + List.of(metadataMap.values().toArray(Attribute[]::new)), + IndexMode.STANDARD, + null + ); } @Override public PlanFactory visitStatsCommand(EsqlBaseParser.StatsCommandContext ctx) { final Stats stats = stats(source(ctx), ctx.grouping, ctx.stats); - return input -> new EsqlAggregate(source(ctx), input, Aggregate.AggregateType.STANDARD, stats.groupings, stats.aggregates); + return input -> new Aggregate(source(ctx), input, Aggregate.AggregateType.STANDARD, stats.groupings, stats.aggregates); } private record Stats(List groupings, List aggregates) { @@ -320,6 +332,9 @@ private void fail(Expression exp, String message, Object... args) { @Override public PlanFactory visitInlinestatsCommand(EsqlBaseParser.InlinestatsCommandContext ctx) { + if (false == EsqlPlugin.INLINESTATS_FEATURE_FLAG.isEnabled()) { + throw new ParsingException(source(ctx), "INLINESTATS command currently requires a snapshot build"); + } List aggregates = new ArrayList<>(visitFields(ctx.stats)); List groupings = visitGrouping(ctx.grouping); aggregates.addAll(groupings); @@ -459,16 +474,18 @@ public LogicalPlan visitMetricsCommand(EsqlBaseParser.MetricsCommandContext ctx) TableIdentifier table = new TableIdentifier(source, null, visitIndexPattern(ctx.indexPattern())); if (ctx.aggregates == null && ctx.grouping == null) { - return new EsqlUnresolvedRelation(source, table, List.of(), IndexMode.STANDARD); + return new UnresolvedRelation(source, table, false, List.of(), IndexMode.STANDARD, null); } final Stats stats = stats(source, ctx.grouping, ctx.aggregates); - var relation = new EsqlUnresolvedRelation( + var relation = new UnresolvedRelation( source, table, + false, List.of(new MetadataAttribute(source, MetadataAttribute.TSID_FIELD, DataType.KEYWORD, false)), - IndexMode.TIME_SERIES + IndexMode.TIME_SERIES, + null ); - return new EsqlAggregate(source, relation, Aggregate.AggregateType.METRICS, stats.groupings, stats.aggregates); + return new Aggregate(source, relation, Aggregate.AggregateType.METRICS, stats.groupings, stats.aggregates); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/GeneratingPlan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/GeneratingPlan.java new file mode 100644 index 0000000000000..49bab1f577b33 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/GeneratingPlan.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.plan; + +import org.elasticsearch.xpack.esql.core.expression.Attribute; + +import java.util.List; + +/** + * A plan that creates new {@link Attribute}s and appends them to the child {@link org.elasticsearch.xpack.esql.plan.logical.UnaryPlan}'s + * attributes. + * Attributes are appended on the right hand side of the child's input. In case of name conflicts, the rightmost attribute with + * a given name shadows any attributes left of it + * (c.f. {@link org.elasticsearch.xpack.esql.expression.NamedExpressions#mergeOutputAttributes(List, List)}). + */ +public interface GeneratingPlan> { + List generatedAttributes(); + + /** + * Create a new instance of this node with new output {@link Attribute}s using the given names. + * If an output attribute already has the desired name, we continue using it; otherwise, we + * create a new attribute with a new {@link org.elasticsearch.xpack.esql.core.expression.NameId}. + */ + // TODO: the generated attributes should probably become synthetic once renamed + // blocked on https://github.com/elastic/elasticsearch/issues/98703 + PlanType withGeneratedNames(List newNames); + + default void checkNumberOfNewNames(List newNames) { + if (newNames.size() != generatedAttributes().size()) { + throw new IllegalArgumentException( + "Number of new names is [" + newNames.size() + "] but there are [" + generatedAttributes().size() + "] existing names." + ); + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java index 5ab483e60d7b0..f3471e5ce8a0c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java @@ -23,8 +23,10 @@ import java.util.List; import java.util.Objects; -public class Aggregate extends UnaryPlan { +import static java.util.Collections.emptyList; +import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes; +public class Aggregate extends UnaryPlan implements Stats { public enum AggregateType { STANDARD, // include metrics aggregates such as rates @@ -50,6 +52,7 @@ static AggregateType readType(StreamInput in) throws IOException { private final AggregateType aggregateType; private final List groupings; private final List aggregates; + private List lazyOutput; public Aggregate( Source source, @@ -92,6 +95,11 @@ public Aggregate replaceChild(LogicalPlan newChild) { return new Aggregate(source(), newChild, aggregateType, groupings, aggregates); } + @Override + public Aggregate with(List newGroupings, List newAggregates) { + return new Aggregate(source(), child(), aggregateType(), newGroupings, newAggregates); + } + public AggregateType aggregateType() { return aggregateType; } @@ -111,7 +119,10 @@ public boolean expressionsResolved() { @Override public List output() { - return Expressions.asAttributes(aggregates); + if (lazyOutput == null) { + lazyOutput = mergeOutputAttributes(Expressions.asAttributes(aggregates()), emptyList()); + } + return lazyOutput; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Dissect.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Dissect.java index c0c564b1b36eb..1736ccc58e81c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Dissect.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Dissect.java @@ -10,9 +10,12 @@ import org.elasticsearch.dissect.DissectParser; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -21,6 +24,17 @@ public class Dissect extends RegexExtract { public record Parser(String pattern, String appendSeparator, DissectParser parser) { + public List keyAttributes(Source src) { + List keys = new ArrayList<>(); + for (var x : parser.outputKeys()) { + if (x.isEmpty() == false) { + keys.add(new ReferenceAttribute(src, x, DataType.KEYWORD)); + } + } + + return keys; + } + // Override hashCode and equals since the parser is considered equal if its pattern and // appendSeparator are equal ( and DissectParser uses reference equality ) @Override @@ -52,6 +66,11 @@ protected NodeInfo info() { return NodeInfo.create(this, Dissect::new, child(), input, parser, extractedFields); } + @Override + public Dissect withGeneratedNames(List newNames) { + return new Dissect(source(), child(), input, parser, renameExtractedFields(newNames)); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java index a4d553eae4749..a15acf111307d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java @@ -10,25 +10,32 @@ import org.elasticsearch.common.util.Maps; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.esql.core.capabilities.Resolvables; +import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.plan.GeneratingPlan; +import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; +import static org.elasticsearch.xpack.esql.core.expression.Expressions.asAttributes; import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes; -public class Enrich extends UnaryPlan { +public class Enrich extends UnaryPlan implements GeneratingPlan { private final Expression policyName; private final NamedExpression matchField; private final EnrichPolicy policy; private final Map concreteIndices; // cluster -> enrich indices + // This could be simplified by just always using an Alias. private final List enrichFields; private List output; @@ -126,6 +133,32 @@ public List output() { return output; } + @Override + public List generatedAttributes() { + return asAttributes(enrichFields); + } + + @Override + public Enrich withGeneratedNames(List newNames) { + checkNumberOfNewNames(newNames); + + List newEnrichFields = new ArrayList<>(enrichFields.size()); + for (int i = 0; i < enrichFields.size(); i++) { + NamedExpression enrichField = enrichFields.get(i); + String newName = newNames.get(i); + if (enrichField.name().equals(newName)) { + newEnrichFields.add(enrichField); + } else if (enrichField instanceof ReferenceAttribute ra) { + newEnrichFields.add(new Alias(ra.source(), newName, ra.qualifier(), ra, new NameId(), ra.synthetic())); + } else if (enrichField instanceof Alias a) { + newEnrichFields.add(new Alias(a.source(), newName, a.qualifier(), a.child(), new NameId(), a.synthetic())); + } else { + throw new IllegalArgumentException("Enrich field must be Alias or ReferenceAttribute"); + } + } + return new Enrich(source(), child(), mode(), policyName(), matchField(), policy(), concreteIndices(), newEnrichFields); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java index 382838a5968cc..866385e6c7c28 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java @@ -98,7 +98,7 @@ public boolean expressionsResolved() { @Override public int hashCode() { - return Objects.hash(index, indexMode, frozen); + return Objects.hash(index, indexMode, frozen, attrs); } @Override @@ -112,7 +112,10 @@ public boolean equals(Object obj) { } EsRelation other = (EsRelation) obj; - return Objects.equals(index, other.index) && indexMode == other.indexMode() && frozen == other.frozen; + return Objects.equals(index, other.index) + && indexMode == other.indexMode() + && frozen == other.frozen + && Objects.equals(attrs, other.attrs); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsqlAggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsqlAggregate.java deleted file mode 100644 index cc72823507f02..0000000000000 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsqlAggregate.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.plan.logical; - -import org.elasticsearch.xpack.esql.core.expression.Attribute; -import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.Expressions; -import org.elasticsearch.xpack.esql.core.expression.NamedExpression; -import org.elasticsearch.xpack.esql.core.tree.NodeInfo; -import org.elasticsearch.xpack.esql.core.tree.Source; - -import java.util.List; - -import static java.util.Collections.emptyList; -import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes; - -/** - * Extension of Aggregate for handling duplicates. - * In ESQL is it possible to declare multiple aggregations and groupings with the same name, with the last declaration in grouping - * winning. - * As some of these declarations can be invalid, for validation reasons we need to keep the data around yet allowing will lead to - * ambiguity in the output. - * Hence this class - to allow the declaration to be moved over and thus for the Verifier to pick up the declaration while providing - * a proper output. - * To simplify things, the Aggregate class will be replaced with a vanilla one. - */ -public class EsqlAggregate extends Aggregate { - - private List lazyOutput; - - public EsqlAggregate( - Source source, - LogicalPlan child, - AggregateType aggregateType, - List groupings, - List aggregates - ) { - super(source, child, aggregateType, groupings, aggregates); - } - - @Override - public List output() { - if (lazyOutput == null) { - lazyOutput = mergeOutputAttributes(Expressions.asAttributes(aggregates()), emptyList()); - } - - return lazyOutput; - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, EsqlAggregate::new, child(), aggregateType(), groupings(), aggregates()); - } - - @Override - public EsqlAggregate replaceChild(LogicalPlan newChild) { - return new EsqlAggregate(source(), newChild, aggregateType(), groupings(), aggregates()); - } -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsqlUnresolvedRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsqlUnresolvedRelation.java deleted file mode 100644 index 5b9cabd3807a2..0000000000000 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsqlUnresolvedRelation.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.plan.logical; - -import org.elasticsearch.index.IndexMode; -import org.elasticsearch.xpack.esql.core.expression.Attribute; -import org.elasticsearch.xpack.esql.core.expression.AttributeSet; -import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; -import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; -import org.elasticsearch.xpack.esql.core.plan.TableIdentifier; -import org.elasticsearch.xpack.esql.core.tree.NodeInfo; -import org.elasticsearch.xpack.esql.core.tree.Source; - -import java.util.List; - -public class EsqlUnresolvedRelation extends UnresolvedRelation { - - private final List metadataFields; - private final IndexMode indexMode; - - public EsqlUnresolvedRelation( - Source source, - TableIdentifier table, - List metadataFields, - IndexMode indexMode, - String unresolvedMessage - ) { - super(source, table, "", false, unresolvedMessage); - this.metadataFields = metadataFields; - this.indexMode = indexMode; - } - - public EsqlUnresolvedRelation(Source source, TableIdentifier table, List metadataFields, IndexMode indexMode) { - this(source, table, metadataFields, indexMode, null); - } - - public List metadataFields() { - return metadataFields; - } - - public IndexMode indexMode() { - return indexMode; - } - - @Override - public AttributeSet references() { - AttributeSet refs = super.references(); - if (indexMode == IndexMode.TIME_SERIES) { - refs = new AttributeSet(refs); - refs.add(new UnresolvedAttribute(source(), MetadataAttribute.TIMESTAMP_FIELD)); - } - return refs; - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, EsqlUnresolvedRelation::new, table(), metadataFields(), indexMode(), unresolvedMessage()); - } -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java index 20117a873c143..981ccae91e1d9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java @@ -10,15 +10,21 @@ import org.elasticsearch.xpack.esql.core.capabilities.Resolvables; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.AttributeMap; +import org.elasticsearch.xpack.esql.core.expression.NameId; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.plan.GeneratingPlan; +import java.util.ArrayList; import java.util.List; import java.util.Objects; +import static org.elasticsearch.xpack.esql.core.expression.Expressions.asAttributes; import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes; -public class Eval extends UnaryPlan { +public class Eval extends UnaryPlan implements GeneratingPlan { private final List fields; private List lazyOutput; @@ -41,6 +47,50 @@ public List output() { return lazyOutput; } + @Override + public List generatedAttributes() { + return asAttributes(fields); + } + + @Override + public Eval withGeneratedNames(List newNames) { + checkNumberOfNewNames(newNames); + + return new Eval(source(), child(), renameAliases(fields, newNames)); + } + + private List renameAliases(List originalAttributes, List newNames) { + AttributeMap.Builder aliasReplacedByBuilder = AttributeMap.builder(); + List newFields = new ArrayList<>(originalAttributes.size()); + for (int i = 0; i < originalAttributes.size(); i++) { + Alias field = originalAttributes.get(i); + String newName = newNames.get(i); + if (field.name().equals(newName)) { + newFields.add(field); + } else { + Alias newField = new Alias(field.source(), newName, field.qualifier(), field.child(), new NameId(), field.synthetic()); + newFields.add(newField); + aliasReplacedByBuilder.put(field.toAttribute(), newField.toAttribute()); + } + } + AttributeMap aliasReplacedBy = aliasReplacedByBuilder.build(); + + // We need to also update any references to the old attributes in the new attributes; e.g. + // EVAL x = 1, y = x + 1 + // renaming x, y to x1, y1 + // so far became + // EVAL x1 = 1, y1 = x + 1 + // - but x doesn't exist anymore, so replace it by x1 to obtain + // EVAL x1 = 1, y1 = x1 + 1 + + List newFieldsWithUpdatedRefs = new ArrayList<>(originalAttributes.size()); + for (Alias newField : newFields) { + newFieldsWithUpdatedRefs.add((Alias) newField.transformUp(ReferenceAttribute.class, r -> aliasReplacedBy.resolve(r, r))); + } + + return newFieldsWithUpdatedRefs; + } + @Override public boolean expressionsResolved() { return Resolvables.resolved(fields); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Grok.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Grok.java index 963fd318f814c..aba2f4a86be20 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Grok.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Grok.java @@ -103,6 +103,11 @@ public List output() { return NamedExpressions.mergeOutputAttributes(extractedFields, child().output()); } + @Override + public Grok withGeneratedNames(List newNames) { + return new Grok(source(), child(), input, parser, renameExtractedFields(newNames)); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java index 46ec56223384c..1c0d942537f8e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java @@ -7,21 +7,58 @@ package org.elasticsearch.xpack.esql.plan.logical; +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.core.Releasables; import org.elasticsearch.xpack.esql.core.capabilities.Resolvables; +import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import org.elasticsearch.xpack.esql.plan.logical.join.Join; +import org.elasticsearch.xpack.esql.plan.logical.join.JoinConfig; +import org.elasticsearch.xpack.esql.plan.logical.join.JoinType; +import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; +import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier; +import org.elasticsearch.xpack.esql.planner.PlannerUtils; +import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Objects; -public class InlineStats extends UnaryPlan { +import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes; + +/** + * Enriches the stream of data with the results of running a {@link Aggregate STATS}. + *

    + * This is a {@link Phased} operation that doesn't have a "native" implementation. + * Instead, it's implemented as first running a {@link Aggregate STATS} and then + * a {@link Join}. + *

    + */ +public class InlineStats extends UnaryPlan implements NamedWriteable, Phased, Stats { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + InlineStats.class, + "InlineStats", + InlineStats::new + ); private final List groupings; private final List aggregates; + private List lazyOutput; public InlineStats(Source source, LogicalPlan child, List groupings, List aggregates) { super(source, child); @@ -29,6 +66,28 @@ public InlineStats(Source source, LogicalPlan child, List groupings, this.aggregates = aggregates; } + public InlineStats(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readLogicalPlanNode(), + in.readNamedWriteableCollectionAsList(Expression.class), + in.readNamedWriteableCollectionAsList(NamedExpression.class) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeLogicalPlanNode(child()); + out.writeNamedWriteableCollection(groupings); + out.writeNamedWriteableCollection(aggregates); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected NodeInfo info() { return NodeInfo.create(this, InlineStats::new, child(), groupings, aggregates); @@ -39,10 +98,17 @@ public InlineStats replaceChild(LogicalPlan newChild) { return new InlineStats(source(), newChild, groupings, aggregates); } + @Override + public InlineStats with(List newGroupings, List newAggregates) { + return new InlineStats(source(), child(), newGroupings, newAggregates); + } + + @Override public List groupings() { return groupings; } + @Override public List aggregates() { return aggregates; } @@ -54,7 +120,19 @@ public boolean expressionsResolved() { @Override public List output() { - return Expressions.asAttributes(aggregates); + if (this.lazyOutput == null) { + List addedFields = new ArrayList<>(); + AttributeSet childOutput = child().outputSet(); + + for (NamedExpression agg : aggregates) { + if (childOutput.contains(agg) == false) { + addedFields.add(agg); + } + } + + this.lazyOutput = mergeOutputAttributes(addedFields, child().output()); + } + return lazyOutput; } @Override @@ -77,4 +155,102 @@ public boolean equals(Object obj) { && Objects.equals(aggregates, other.aggregates) && Objects.equals(child(), other.child()); } + + @Override + public LogicalPlan firstPhase() { + return new Aggregate(source(), child(), Aggregate.AggregateType.STANDARD, groupings, aggregates); + } + + @Override + public LogicalPlan nextPhase(List schema, List firstPhaseResult) { + if (equalsAndSemanticEquals(firstPhase().output(), schema) == false) { + throw new IllegalStateException("Unexpected first phase outputs: " + firstPhase().output() + " vs " + schema); + } + if (groupings.isEmpty()) { + return ungroupedNextPhase(schema, firstPhaseResult); + } + return groupedNextPhase(schema, firstPhaseResult); + } + + private LogicalPlan ungroupedNextPhase(List schema, List firstPhaseResult) { + if (firstPhaseResult.size() != 1) { + throw new IllegalArgumentException("expected single row"); + } + Page p = firstPhaseResult.get(0); + if (p.getPositionCount() != 1) { + throw new IllegalArgumentException("expected single row"); + } + List values = new ArrayList<>(schema.size()); + for (int i = 0; i < schema.size(); i++) { + Attribute s = schema.get(i); + Object value = BlockUtils.toJavaObject(p.getBlock(i), 0); + values.add(new Alias(source(), s.name(), null, new Literal(source(), value, s.dataType()), aggregates.get(i).id())); + } + return new Eval(source(), child(), values); + } + + private static boolean equalsAndSemanticEquals(List left, List right) { + if (left.equals(right) == false) { + return false; + } + for (int i = 0; i < left.size(); i++) { + if (left.get(i).semanticEquals(right.get(i)) == false) { + return false; + } + } + return true; + } + + private LogicalPlan groupedNextPhase(List schema, List firstPhaseResult) { + LocalRelation local = firstPhaseResultsToLocalRelation(schema, firstPhaseResult); + List groupingAttributes = new ArrayList<>(groupings.size()); + for (Expression g : groupings) { + if (g instanceof Attribute a) { + groupingAttributes.add(a); + } else { + throw new UnsupportedOperationException("INLINESTATS doesn't support expressions in grouping position yet"); + } + } + List leftFields = new ArrayList<>(groupingAttributes.size()); + List rightFields = new ArrayList<>(groupingAttributes.size()); + List rhsOutput = Join.makeReference(local.output()); + for (Attribute lhs : groupingAttributes) { + for (Attribute rhs : rhsOutput) { + if (lhs.name().equals(rhs.name())) { + leftFields.add(lhs); + rightFields.add(rhs); + break; + } + } + } + JoinConfig config = new JoinConfig(JoinType.LEFT, groupingAttributes, leftFields, rightFields); + return new Join(source(), child(), local, config); + } + + private LocalRelation firstPhaseResultsToLocalRelation(List schema, List firstPhaseResult) { + // Limit ourselves to 1mb of results similar to LOOKUP for now. + long bytesUsed = firstPhaseResult.stream().mapToLong(Page::ramBytesUsedByBlocks).sum(); + if (bytesUsed > ByteSizeValue.ofMb(1).getBytes()) { + throw new IllegalArgumentException("first phase result too large [" + ByteSizeValue.ofBytes(bytesUsed) + "] > 1mb"); + } + int positionCount = firstPhaseResult.stream().mapToInt(Page::getPositionCount).sum(); + Block.Builder[] builders = new Block.Builder[schema.size()]; + Block[] blocks; + try { + for (int b = 0; b < builders.length; b++) { + builders[b] = PlannerUtils.toElementType(schema.get(b).dataType()) + .newBlockBuilder(positionCount, PlannerUtils.NON_BREAKING_BLOCK_FACTORY); + } + for (Page p : firstPhaseResult) { + for (int b = 0; b < builders.length; b++) { + builders[b].copyFrom(p.getBlock(b), 0, p.getPositionCount()); + } + } + blocks = Block.Builder.buildAll(builders); + } finally { + Releasables.closeExpectNoException(builders); + } + return new LocalRelation(source(), schema, LocalSupplier.of(blocks)); + } + } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Phased.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Phased.java new file mode 100644 index 0000000000000..ba0f97cdfa30b --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Phased.java @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.plan.logical; + +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.xpack.esql.analysis.Analyzer; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.util.Holder; + +import java.util.List; + +/** + * Marks a {@link LogicalPlan} node as requiring multiple ESQL executions to run. + * All logical plans are now run by: + *
      + *
    1. {@link Analyzer analyzing} the entire query
      2. + *
    2. {@link Phased#extractFirstPhase extracting} the first phase from the + * logical plan
      3. + *
    3. if there isn't a first phase, run the entire logical plan and return the + * results. you are done.
      4. + *
    4. if there is first phase, run that
      5. + *
    5. {@link Phased#applyResultsFromFirstPhase applying} the results from the + * first phase into the logical plan
      6. + *
    6. start over from step 2 using the new logical plan
      7. + *
    + *

    For example, {@code INLINESTATS} is written like this:

    + * 
    {@code
+ * FROM foo
+ * | EVAL bar = a * b
+ * | INLINESTATS m = MAX(bar) BY b
+ * | WHERE m = bar
+ * | LIMIT 1
+ * }
    + *

    And it's split into:

    + * 
    {@code
+ * FROM foo
+ * | EVAL bar = a * b
+ * | STATS m = MAX(bar) BY b
+ * }
    + *

    and

    + * 
    {@code
+ * FROM foo
+ * | EVAL bar = a * b
+ * | LOOKUP (results of m = MAX(bar) BY b) ON b
+ * | WHERE m = bar
+ * | LIMIT 1
+ * }
    + *

    If there are multiple {@linkplain Phased} nodes in the plan we always + * operate on the lowest one first, counting from the data source "upwards". + * Generally that'll read left to right in the query. So:

    + * 
    {@code
+ * FROM foo | INLINESTATS | INLINESTATS
+ * }
    + * becomes + * 
    {@code
+ * FROM foo | STATS
+ * }
    + * and + * 
    {@code
+ * FROM foo | HASHJOIN | INLINESTATS
+ * }
    + * which is further broken into + * 
    {@code
+ * FROM foo | HASHJOIN | STATS
+ * }
    + * and finally: + * 
    {@code
+ * FROM foo | HASHJOIN | HASHJOIN
+ * }
    + */ +public interface Phased { + /** + * Return a {@link LogicalPlan} for the first "phase" of this operation. + * The result of this phase will be provided to {@link #nextPhase}. + */ + LogicalPlan firstPhase(); + + /** + * Use the results of plan provided from {@link #firstPhase} to produce the + * next phase of the query. + */ + LogicalPlan nextPhase(List schema, List firstPhaseResult); + + /** + * Find the first {@link Phased} operation and return it's {@link #firstPhase}. + * Or {@code null} if there aren't any {@linkplain Phased} operations. + */ + static LogicalPlan extractFirstPhase(LogicalPlan plan) { + if (false == plan.analyzed()) { + throw new IllegalArgumentException("plan must be analyzed"); + } + var firstPhase = new Holder(); + plan.forEachUp(t -> { + if (firstPhase.get() == null && t instanceof Phased phased) { + firstPhase.set(phased.firstPhase()); + } + }); + LogicalPlan firstPhasePlan = firstPhase.get(); + if (firstPhasePlan != null) { + firstPhasePlan.setAnalyzed(); + } + return firstPhasePlan; + } + + /** + * Merge the results of {@link #extractFirstPhase} into a {@link LogicalPlan} + * and produce a new {@linkplain LogicalPlan} that will execute the rest of the + * query. This plan may contain another + * {@link #firstPhase}. If it does then it will also need to be + * {@link #extractFirstPhase extracted} and the results will need to be applied + * again by calling this method. Eventually this will produce a plan which + * does not have a {@link #firstPhase} and that is the "final" + * phase of the plan. + */ + static LogicalPlan applyResultsFromFirstPhase(LogicalPlan plan, List schema, List result) { + if (false == plan.analyzed()) { + throw new IllegalArgumentException("plan must be analyzed"); + } + Holder seen = new Holder<>(false); + LogicalPlan applied = plan.transformUp(logicalPlan -> { + if (seen.get() == false && logicalPlan instanceof Phased phased) { + seen.set(true); + return phased.nextPhase(schema, result); + } + return logicalPlan; + }); + applied.setAnalyzed(); + return applied; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/RegexExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/RegexExtract.java index 649173f11dfaf..f95bee92d4e1a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/RegexExtract.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/RegexExtract.java @@ -9,14 +9,17 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.plan.GeneratingPlan; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes; -public abstract class RegexExtract extends UnaryPlan { +public abstract class RegexExtract extends UnaryPlan implements GeneratingPlan { protected final Expression input; protected final List extractedFields; @@ -40,10 +43,36 @@ public Expression input() { return input; } + /** + * Upon parsing, these are named according to the {@link Dissect} or {@link Grok} pattern, but can be renamed without changing the + * pattern. + */ public List extractedFields() { return extractedFields; } + @Override + public List generatedAttributes() { + return extractedFields; + } + + List renameExtractedFields(List newNames) { + checkNumberOfNewNames(newNames); + + List renamedExtractedFields = new ArrayList<>(extractedFields.size()); + for (int i = 0; i < newNames.size(); i++) { + Attribute extractedField = extractedFields.get(i); + String newName = newNames.get(i); + if (extractedField.name().equals(newName)) { + renamedExtractedFields.add(extractedField); + } else { + renamedExtractedFields.add(extractedFields.get(i).withName(newNames.get(i)).withId(new NameId())); + } + } + + return renamedExtractedFields; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Rename.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Rename.java index 5e4b45d7127fe..29ee7f0504c70 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Rename.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Rename.java @@ -7,7 +7,11 @@ package org.elasticsearch.xpack.esql.plan.logical; +import org.elasticsearch.xpack.esql.analysis.Analyzer.ResolveRefs; import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; @@ -28,6 +32,14 @@ public List renamings() { return renamings; } + @Override + public List output() { + // Normally shouldn't reach here, as Rename only exists before resolution. + List projectionsAfterResolution = ResolveRefs.projectionsForRename(this, this.child().output(), null); + + return Expressions.asAttributes(projectionsAfterResolution); + } + @Override public boolean expressionsResolved() { for (var alias : renamings) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Stats.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Stats.java new file mode 100644 index 0000000000000..1dde8e9e95990 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Stats.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.plan.logical; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; + +import java.util.List; + +/** + * STATS-like operations. Like {@link Aggregate} and {@link InlineStats}. + */ +public interface Stats { + /** + * Rebuild this plan with new groupings and new aggregates. + */ + Stats with(List newGroupings, List newAggregates); + + /** + * Have all the expressions in this plan been resolved? + */ + boolean expressionsResolved(); + + /** + * List containing both the aggregate expressions and grouping expressions. + */ + List aggregates(); + + /** + * List containing just the grouping expressions. + */ + List groupings(); + +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java index af19bc87f2c54..cd1817367167d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java @@ -6,8 +6,12 @@ */ package org.elasticsearch.xpack.esql.plan.logical; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.AttributeSet; +import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; +import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.plan.TableIdentifier; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -22,34 +26,35 @@ public class UnresolvedRelation extends LeafPlan implements Unresolvable { private final TableIdentifier table; private final boolean frozen; - private final String alias; + private final List metadataFields; + private final IndexMode indexMode; private final String unresolvedMsg; - public UnresolvedRelation(Source source, TableIdentifier table, String alias, boolean frozen) { - this(source, table, alias, frozen, null); - } - - public UnresolvedRelation(Source source, TableIdentifier table, String alias, boolean frozen, String unresolvedMessage) { + public UnresolvedRelation( + Source source, + TableIdentifier table, + boolean frozen, + List metadataFields, + IndexMode indexMode, + String unresolvedMessage + ) { super(source); this.table = table; - this.alias = alias; this.frozen = frozen; + this.metadataFields = metadataFields; + this.indexMode = indexMode; this.unresolvedMsg = unresolvedMessage == null ? "Unknown index [" + table.index() + "]" : unresolvedMessage; } @Override protected NodeInfo info() { - return NodeInfo.create(this, UnresolvedRelation::new, table, alias, frozen, unresolvedMsg); + return NodeInfo.create(this, UnresolvedRelation::new, table, frozen, metadataFields, indexMode, unresolvedMsg); } public TableIdentifier table() { return table; } - public String alias() { - return alias; - } - public boolean frozen() { return frozen; } @@ -69,14 +74,32 @@ public List output() { return Collections.emptyList(); } + public List metadataFields() { + return metadataFields; + } + + public IndexMode indexMode() { + return indexMode; + } + @Override public String unresolvedMessage() { return unresolvedMsg; } + @Override + public AttributeSet references() { + AttributeSet refs = super.references(); + if (indexMode == IndexMode.TIME_SERIES) { + refs = new AttributeSet(refs); + refs.add(new UnresolvedAttribute(source(), MetadataAttribute.TIMESTAMP_FIELD)); + } + return refs; + } + @Override public int hashCode() { - return Objects.hash(source(), table, alias, unresolvedMsg); + return Objects.hash(source(), table, metadataFields, indexMode, unresolvedMsg); } @Override @@ -91,8 +114,9 @@ public boolean equals(Object obj) { UnresolvedRelation other = (UnresolvedRelation) obj; return Objects.equals(table, other.table) - && Objects.equals(alias, other.alias) && Objects.equals(frozen, other.frozen) + && Objects.equals(metadataFields, other.metadataFields) + && indexMode == other.indexMode && Objects.equals(unresolvedMsg, other.unresolvedMsg); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EstimatesRowSize.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EstimatesRowSize.java index 40c9067efbeda..cfb6cce2579a2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EstimatesRowSize.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EstimatesRowSize.java @@ -103,24 +103,12 @@ public String toString() { static int estimateSize(DataType dataType) { ElementType elementType = PlannerUtils.toElementType(dataType); - return switch (elementType) { - case BOOLEAN -> 1; - case BYTES_REF -> switch (dataType.typeName()) { - case "ip" -> 16; // IP addresses, both IPv4 and IPv6, are encoded using 16 bytes. - case "version" -> 15; // 8.15.2-SNAPSHOT is 15 bytes, most are shorter, some can be longer - case "geo_point", "cartesian_point" -> 21; // WKB for points is typically 21 bytes. - case "geo_shape", "cartesian_shape" -> 200; // wild estimate, based on some test data (airport_city_boundaries) - default -> 50; // wild estimate for the size of a string. - }; - case DOC -> throw new EsqlIllegalArgumentException("can't load a [doc] with field extraction"); - case FLOAT -> Float.BYTES; - case DOUBLE -> Double.BYTES; - case INT -> Integer.BYTES; - case LONG -> Long.BYTES; - case NULL -> 0; - // TODO: provide a specific estimate for aggregated_metrics_double - case COMPOSITE -> 50; - case UNKNOWN -> throw new EsqlIllegalArgumentException("[unknown] can't be the result of field extraction"); - }; + if (elementType == ElementType.DOC) { + throw new EsqlIllegalArgumentException("can't load a [doc] with field extraction"); + } + if (elementType == ElementType.UNKNOWN) { + throw new EsqlIllegalArgumentException("[unknown] can't be the result of field extraction"); + } + return dataType.estimatedSize().orElse(50); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java index 87775d5048752..213d7266a0b1e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java @@ -51,6 +51,19 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT; import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; +/** + * Static class used to convert aggregate expressions to the named expressions that represent their intermediate state. + *

    + * At class load time, the mapper is populated with all supported aggregate functions and their intermediate state. + *

    + *

    + * Reflection is used to call the {@code intermediateStateDesc()}` static method of the aggregate functions, + * but the function classes are found based on the exising information within this class. + *

    + *

    + * This class must be updated when aggregations are created or updated, by adding the new aggs or types to the corresponding methods. + *

    + */ final class AggregateMapper { private static final List NUMERIC = List.of("Int", "Long", "Double"); @@ -147,7 +160,7 @@ private static Stream, Tuple>> typeAndNames(Class if (NumericAggregate.class.isAssignableFrom(clazz)) { types = NUMERIC; } else if (Max.class.isAssignableFrom(clazz) || Min.class.isAssignableFrom(clazz)) { - types = List.of("Boolean", "Int", "Long", "Double"); + types = List.of("Boolean", "Int", "Long", "Double", "Ip"); } else if (clazz == Count.class) { types = List.of(""); // no extra type distinction } else if (SpatialAggregateFunction.class.isAssignableFrom(clazz)) { @@ -157,7 +170,7 @@ private static Stream, Tuple>> typeAndNames(Class // TODO can't we figure this out from the function itself? types = List.of("Int", "Long", "Double", "Boolean", "BytesRef"); } else if (Top.class.isAssignableFrom(clazz)) { - types = List.of("Int", "Long", "Double"); + types = List.of("Boolean", "Int", "Long", "Double", "Ip"); } else if (Rate.class.isAssignableFrom(clazz)) { types = List.of("Int", "Long", "Double"); } else if (FromPartial.class.isAssignableFrom(clazz) || ToPartial.class.isAssignableFrom(clazz)) { @@ -266,7 +279,7 @@ private static DataType toDataType(ElementType elementType) { case INT -> DataType.INTEGER; case LONG -> DataType.LONG; case DOUBLE -> DataType.DOUBLE; - default -> throw new EsqlIllegalArgumentException("unsupported agg type: " + elementType); + case FLOAT, NULL, DOC, COMPOSITE, UNKNOWN -> throw new EsqlIllegalArgumentException("unsupported agg type: " + elementType); }; } @@ -278,6 +291,12 @@ private static String dataTypeToString(DataType type, Class aggClass) { if (aggClass == ToPartial.class || aggClass == FromPartial.class) { return ""; } + if ((aggClass == Max.class || aggClass == Min.class) && type.equals(DataType.IP)) { + return "Ip"; + } + if (aggClass == Top.class && type.equals(DataType.IP)) { + return "Ip"; + } if (type.equals(DataType.BOOLEAN)) { return "Boolean"; } else if (type.equals(DataType.INTEGER) || type.equals(DataType.COUNTER_INTEGER)) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 8611d2c6fa9fb..45989b4f563ce 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -33,6 +33,7 @@ import org.elasticsearch.compute.operator.OrdinalsGroupingOperator; import org.elasticsearch.compute.operator.SourceOperator; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -116,7 +117,8 @@ public final PhysicalOperation fieldExtractPhysicalOperation(FieldExtractExec fi DataType dataType = attr.dataType(); MappedFieldType.FieldExtractPreference fieldExtractPreference = PlannerUtils.extractPreference(docValuesAttrs.contains(attr)); ElementType elementType = PlannerUtils.toElementType(dataType, fieldExtractPreference); - String fieldName = attr.name(); + // Do not use the field attribute name, this can deviate from the field name for union types. + String fieldName = attr instanceof FieldAttribute fa ? fa.fieldName() : attr.name(); boolean isUnsupported = dataType == DataType.UNSUPPORTED; IntFunction loader = s -> getBlockLoaderFor(s, fieldName, isUnsupported, fieldExtractPreference, unionTypes); fields.add(new ValuesSourceReaderOperator.FieldInfo(fieldName, elementType, loader)); @@ -234,8 +236,10 @@ public final Operator.OperatorFactory ordinalGroupingOperatorFactory( // Costin: why are they ready and not already exposed in the layout? boolean isUnsupported = attrSource.dataType() == DataType.UNSUPPORTED; var unionTypes = findUnionTypes(attrSource); + // Do not use the field attribute name, this can deviate from the field name for union types. + String fieldName = attrSource instanceof FieldAttribute fa ? fa.fieldName() : attrSource.name(); return new OrdinalsGroupingOperator.OrdinalsGroupingOperatorFactory( - shardIdx -> getBlockLoaderFor(shardIdx, attrSource.name(), isUnsupported, NONE, unionTypes), + shardIdx -> getBlockLoaderFor(shardIdx, fieldName, isUnsupported, NONE, unionTypes), vsShardContexts, groupElementType, docChannel, @@ -323,6 +327,11 @@ public String indexName() { return ctx.getFullyQualifiedIndex().getName(); } + @Override + public IndexSettings indexSettings() { + return ctx.getIndexSettings(); + } + @Override public MappedFieldType.FieldExtractPreference fieldExtractPreference() { return fieldExtractPreference; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java index e87006ec7ee09..ef5eac444a2db 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java @@ -37,6 +37,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.InsensitiveEquals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; @@ -49,16 +50,19 @@ import java.time.OffsetTime; import java.time.ZoneId; import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; +import static org.elasticsearch.xpack.esql.core.planner.ExpressionTranslators.or; import static org.elasticsearch.xpack.esql.core.type.DataType.IP; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.HOUR_MINUTE_SECOND; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.versionToString; @@ -67,6 +71,7 @@ public final class EsqlExpressionTranslators { new EqualsIgnoreCaseTranslator(), new BinaryComparisons(), new SpatialRelatesTranslator(), + new InComparisons(), // Ranges is redundant until we start combining binary comparisons (see CombineBinaryComparisons in ql's OptimizerRules) // or introduce a BETWEEN keyword. new ExpressionTranslators.Ranges(), @@ -75,7 +80,6 @@ public final class EsqlExpressionTranslators { new ExpressionTranslators.IsNotNulls(), new ExpressionTranslators.Nots(), new ExpressionTranslators.Likes(), - new ExpressionTranslators.InComparisons(), new ExpressionTranslators.StringQueries(), new ExpressionTranslators.Matches(), new ExpressionTranslators.MultiMatches(), @@ -204,6 +208,8 @@ static Query translate(BinaryComparison bc, TranslatorHandler handler) { ZoneId zoneId = null; if (DataType.isDateTime(attribute.dataType())) { zoneId = bc.zoneId(); + value = dateTimeToString((Long) value); + format = DEFAULT_DATE_TIME_FORMATTER.pattern(); } if (bc instanceof GreaterThan) { return new RangeQuery(source, name, value, false, null, false, format, zoneId); @@ -401,4 +407,58 @@ static Query translate( } } } + + public static class InComparisons extends ExpressionTranslator { + + @Override + protected Query asQuery(In in, TranslatorHandler handler) { + return doTranslate(in, handler); + } + + public static Query doTranslate(In in, TranslatorHandler handler) { + return handler.wrapFunctionQuery(in, in.value(), () -> translate(in, handler)); + } + + private static boolean needsTypeSpecificValueHandling(DataType fieldType) { + return DataType.isDateTime(fieldType) || fieldType == IP || fieldType == VERSION || fieldType == UNSIGNED_LONG; + } + + private static Query translate(In in, TranslatorHandler handler) { + TypedAttribute attribute = checkIsPushableAttribute(in.value()); + + Set terms = new LinkedHashSet<>(); + List queries = new ArrayList<>(); + + for (Expression rhs : in.list()) { + if (DataType.isNull(rhs.dataType()) == false) { + if (needsTypeSpecificValueHandling(attribute.dataType())) { + // delegates to BinaryComparisons translator to ensure consistent handling of date and time values + Query query = BinaryComparisons.translate(new Equals(in.source(), in.value(), rhs), handler); + + if (query instanceof TermQuery) { + terms.add(((TermQuery) query).value()); + } else { + queries.add(query); + } + } else { + terms.add(valueOf(rhs)); + } + } + } + + if (terms.isEmpty() == false) { + String fieldName = pushableAttributeName(attribute); + queries.add(new TermsQuery(in.source(), fieldName, terms)); + } + + return queries.stream().reduce((q1, q2) -> or(in.source(), q1, q2)).get(); + } + + public static Object valueOf(Expression e) { + if (e.foldable()) { + return e.fold(); + } + throw new QlIllegalArgumentException("Cannot determine value for {}", e); + } + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index ddf5fa6eaf8a3..28855abfff73c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -58,6 +58,8 @@ import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.Order; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.enrich.EnrichLookupOperator; import org.elasticsearch.xpack.esql.enrich.EnrichLookupService; @@ -417,12 +419,14 @@ private PhysicalOperation planDissect(DissectExec dissect, LocalExecutionPlanner Layout.Builder layoutBuilder = source.layout.builder(); layoutBuilder.append(dissect.extractedFields()); final Expression expr = dissect.inputExpression(); - String[] attributeNames = Expressions.names(dissect.extractedFields()).toArray(new String[0]); + // Names in the pattern and layout can differ. + // Attributes need to be rename-able to avoid problems with shadowing - see GeneratingPlan resp. PushDownRegexExtract. + String[] patternNames = Expressions.names(dissect.parser().keyAttributes(Source.EMPTY)).toArray(new String[0]); Layout layout = layoutBuilder.build(); source = source.with( new StringExtractOperator.StringExtractOperatorFactory( - attributeNames, + patternNames, EvalMapper.toEvaluator(expr, layout), () -> (input) -> dissect.parser().parser().parse(input) ), @@ -439,11 +443,15 @@ private PhysicalOperation planGrok(GrokExec grok, LocalExecutionPlannerContext c Map fieldToPos = new HashMap<>(extractedFields.size()); Map fieldToType = new HashMap<>(extractedFields.size()); ElementType[] types = new ElementType[extractedFields.size()]; + List extractedFieldsFromPattern = grok.pattern().extractedFields(); for (int i = 0; i < extractedFields.size(); i++) { - Attribute extractedField = extractedFields.get(i); - ElementType type = PlannerUtils.toElementType(extractedField.dataType()); - fieldToPos.put(extractedField.name(), i); - fieldToType.put(extractedField.name(), type); + DataType extractedFieldType = extractedFields.get(i).dataType(); + // Names in pattern and layout can differ. + // Attributes need to be rename-able to avoid problems with shadowing - see GeneratingPlan resp. PushDownRegexExtract. + String patternName = extractedFieldsFromPattern.get(i).name(); + ElementType type = PlannerUtils.toElementType(extractedFieldType); + fieldToPos.put(patternName, i); + fieldToType.put(patternName, type); types[i] = type; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Mapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Mapper.java index 84ed4663496de..299149c6daabc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Mapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Mapper.java @@ -56,6 +56,14 @@ import static org.elasticsearch.xpack.esql.plan.physical.AggregateExec.Mode.FINAL; import static org.elasticsearch.xpack.esql.plan.physical.AggregateExec.Mode.PARTIAL; +/** + *

    This class is part of the planner

    + * + *

    Translates the logical plan into a physical plan. This is where we start to decide what will be executed on the data nodes and what + * will be executed on the coordinator nodes. This step creates {@link org.elasticsearch.xpack.esql.plan.physical.FragmentExec} instances, + * which represent logical plan fragments to be sent to the data nodes and {@link org.elasticsearch.xpack.esql.plan.physical.ExchangeExec} + * instances, which represent data being sent back from the data nodes to the coordinating node.

    + */ public class Mapper { private final EsqlFunctionRegistry functionRegistry; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index d9f073d952a37..57d8d9748aa01 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -50,7 +50,6 @@ import org.elasticsearch.xpack.esql.plan.physical.TopNExec; import org.elasticsearch.xpack.esql.session.EsqlConfiguration; import org.elasticsearch.xpack.esql.stats.SearchStats; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.util.ArrayList; import java.util.LinkedHashSet; @@ -221,7 +220,7 @@ static QueryBuilder detectFilter(PhysicalPlan plan, String fieldName, Predicate< * This specifically excludes spatial data types, which are not themselves sortable. */ public static ElementType toSortableElementType(DataType dataType) { - if (EsqlDataTypes.isSpatial(dataType)) { + if (DataType.isSpatial(dataType)) { return ElementType.UNKNOWN; } return toElementType(dataType); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java index c9e82c76367cc..4b6a38a3e8762 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java @@ -174,6 +174,11 @@ public class EsqlFeatures implements FeatureSpecification { */ public static final NodeFeature METRICS_SYNTAX = new NodeFeature("esql.metrics_syntax"); + /** + * Internal resolve_fields API for ES|QL + */ + public static final NodeFeature RESOLVE_FIELDS_API = new NodeFeature("esql.resolve_fields_api"); + private Set snapshotBuildFeatures() { assert Build.current().isSnapshot() : Build.current(); return Set.of(METRICS_SYNTAX); @@ -202,7 +207,8 @@ public Set getFeatures() { STRING_LITERAL_AUTO_CASTING_EXTENDED, METADATA_FIELDS, TIMESPAN_ABBREVIATIONS, - COUNTER_TYPES + COUNTER_TYPES, + RESOLVE_FIELDS_API ); if (Build.current().isSnapshot()) { return Collections.unmodifiableSet(Sets.union(features, snapshotBuildFeatures())); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java index 46fe229098a16..9809c361ea426 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; @@ -51,6 +52,7 @@ import org.elasticsearch.xpack.esql.action.EsqlAsyncGetResultAction; import org.elasticsearch.xpack.esql.action.EsqlQueryAction; import org.elasticsearch.xpack.esql.action.EsqlQueryRequestBuilder; +import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction; import org.elasticsearch.xpack.esql.action.RestEsqlAsyncQueryAction; import org.elasticsearch.xpack.esql.action.RestEsqlDeleteAsyncResultAction; import org.elasticsearch.xpack.esql.action.RestEsqlGetAsyncResultAction; @@ -78,6 +80,7 @@ import java.util.function.Supplier; public class EsqlPlugin extends Plugin implements ActionPlugin { + public static final FeatureFlag INLINESTATS_FEATURE_FLAG = new FeatureFlag("esql_inlinestats"); public static final String ESQL_WORKER_THREAD_POOL_NAME = "esql_worker"; @@ -144,7 +147,8 @@ public List> getSettings() { new ActionHandler<>(EsqlAsyncGetResultAction.INSTANCE, TransportEsqlAsyncGetResultsAction.class), new ActionHandler<>(EsqlStatsAction.INSTANCE, TransportEsqlStatsAction.class), new ActionHandler<>(XPackUsageFeatureAction.ESQL, EsqlUsageTransportAction.class), - new ActionHandler<>(XPackInfoFeatureAction.ESQL, EsqlInfoTransportAction.class) + new ActionHandler<>(XPackInfoFeatureAction.ESQL, EsqlInfoTransportAction.class), + new ActionHandler<>(EsqlResolveFieldsAction.TYPE, EsqlResolveFieldsAction.class) ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueMatchQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueMatchQuery.java new file mode 100644 index 0000000000000..386c983c8e6af --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueMatchQuery.java @@ -0,0 +1,343 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.querydsl.query; + +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.Terms; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.elasticsearch.index.fielddata.FieldData; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.LeafFieldData; +import org.elasticsearch.index.fielddata.LeafNumericFieldData; +import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +import java.io.IOException; +import java.util.Objects; + +/** + * Finds all fields with a single-value. If a field has a multi-value, it emits a {@link Warnings}. + */ +final class SingleValueMatchQuery extends Query { + + /** + * Choose a big enough value so this approximation never drives the iteration. + * This avoids reporting warnings when queries are not matching multi-values + */ + private static final int MULTI_VALUE_MATCH_COST = 1000; + private static final IllegalArgumentException MULTI_VALUE_EXCEPTION = new IllegalArgumentException( + "single-value function encountered multi-value" + ); + private final IndexFieldData fieldData; + private final Warnings warnings; + + SingleValueMatchQuery(IndexFieldData fieldData, Warnings warnings) { + this.fieldData = fieldData; + this.warnings = warnings; + } + + @Override + public String toString(String field) { + StringBuilder builder = new StringBuilder("single_value_match("); + if (false == this.fieldData.getFieldName().equals(field)) { + builder.append(this.fieldData.getFieldName()); + } + return builder.append(")").toString(); + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) { + return new ConstantScoreWeight(this, boost) { + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + final ScorerSupplier scorerSupplier = scorerSupplier(context); + if (scorerSupplier == null) { + return null; + } + return scorerSupplier.get(Long.MAX_VALUE); + } + + @Override + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + final LeafFieldData lfd = fieldData.load(context); + if (lfd == null) { + return null; + } + /* + * SortedBinaryDocValues are available for most fields, but they + * are made available by eagerly converting non-bytes values to + * utf-8 strings. The eager conversion is quite expensive. So + * we specialize on numeric fields and fields with ordinals to + * avoid that expense in at least that case. + * + * Also! Lucene's FieldExistsQuery only needs one scorer that can + * use all the docs values iterators at DocIdSetIterators. We + * can't do that because we need the check the number of fields. + */ + if (lfd instanceof LeafNumericFieldData n) { + return scorerSupplier(context, n.getLongValues(), this, boost, scoreMode); + } + if (lfd instanceof LeafOrdinalsFieldData o) { + return scorerSupplier(context, o.getOrdinalsValues(), this, boost, scoreMode); + } + return scorerSupplier(context, lfd.getBytesValues(), this, boost, scoreMode); + } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + // don't cache so we can emit warnings + return false; + } + + private ScorerSupplier scorerSupplier( + LeafReaderContext context, + SortedNumericDocValues sortedNumerics, + Weight weight, + float boost, + ScoreMode scoreMode + ) throws IOException { + final int maxDoc = context.reader().maxDoc(); + if (DocValues.unwrapSingleton(sortedNumerics) != null) { + // check for dense field + final PointValues points = context.reader().getPointValues(fieldData.getFieldName()); + if (points != null && points.getDocCount() == maxDoc) { + return new DocIdSetIteratorScorerSupplier(weight, boost, scoreMode, DocIdSetIterator.all(maxDoc)); + } else { + return new PredicateScorerSupplier( + weight, + boost, + scoreMode, + maxDoc, + MULTI_VALUE_MATCH_COST, + sortedNumerics::advanceExact + ); + } + } + final CheckedIntPredicate predicate = doc -> { + if (false == sortedNumerics.advanceExact(doc)) { + return false; + } + if (sortedNumerics.docValueCount() != 1) { + warnings.registerException(MULTI_VALUE_EXCEPTION); + return false; + } + return true; + }; + return new PredicateScorerSupplier(weight, boost, scoreMode, maxDoc, MULTI_VALUE_MATCH_COST, predicate); + } + + private ScorerSupplier scorerSupplier( + LeafReaderContext context, + SortedSetDocValues sortedSetDocValues, + Weight weight, + float boost, + ScoreMode scoreMode + ) throws IOException { + final int maxDoc = context.reader().maxDoc(); + if (DocValues.unwrapSingleton(sortedSetDocValues) != null) { + // check for dense field + final Terms terms = context.reader().terms(fieldData.getFieldName()); + if (terms != null && terms.getDocCount() == maxDoc) { + return new DocIdSetIteratorScorerSupplier(weight, boost, scoreMode, DocIdSetIterator.all(maxDoc)); + } else { + return new PredicateScorerSupplier( + weight, + boost, + scoreMode, + maxDoc, + MULTI_VALUE_MATCH_COST, + sortedSetDocValues::advanceExact + ); + } + } + final CheckedIntPredicate predicate = doc -> { + if (false == sortedSetDocValues.advanceExact(doc)) { + return false; + } + if (sortedSetDocValues.docValueCount() != 1) { + warnings.registerException(MULTI_VALUE_EXCEPTION); + return false; + } + return true; + }; + return new PredicateScorerSupplier(weight, boost, scoreMode, maxDoc, MULTI_VALUE_MATCH_COST, predicate); + } + + private ScorerSupplier scorerSupplier( + LeafReaderContext context, + SortedBinaryDocValues sortedBinaryDocValues, + Weight weight, + float boost, + ScoreMode scoreMode + ) { + final int maxDoc = context.reader().maxDoc(); + if (FieldData.unwrapSingleton(sortedBinaryDocValues) != null) { + return new PredicateScorerSupplier( + weight, + boost, + scoreMode, + maxDoc, + MULTI_VALUE_MATCH_COST, + sortedBinaryDocValues::advanceExact + ); + } + final CheckedIntPredicate predicate = doc -> { + if (false == sortedBinaryDocValues.advanceExact(doc)) { + return false; + } + if (sortedBinaryDocValues.docValueCount() != 1) { + warnings.registerException(MULTI_VALUE_EXCEPTION); + return false; + } + return true; + }; + return new PredicateScorerSupplier(weight, boost, scoreMode, maxDoc, MULTI_VALUE_MATCH_COST, predicate); + } + }; + } + + @Override + public Query rewrite(IndexSearcher indexSearcher) throws IOException { + for (LeafReaderContext context : indexSearcher.getIndexReader().leaves()) { + final LeafFieldData lfd = fieldData.load(context); + if (lfd instanceof LeafNumericFieldData) { + final PointValues pointValues = context.reader().getPointValues(fieldData.getFieldName()); + if (pointValues == null + || pointValues.getDocCount() != context.reader().maxDoc() + || pointValues.size() != pointValues.getDocCount()) { + return super.rewrite(indexSearcher); + } + } else if (lfd instanceof LeafOrdinalsFieldData) { + final Terms terms = context.reader().terms(fieldData.getFieldName()); + if (terms == null || terms.getDocCount() != context.reader().maxDoc() || terms.size() != terms.getDocCount()) { + return super.rewrite(indexSearcher); + } + } else { + return super.rewrite(indexSearcher); + } + } + return new MatchAllDocsQuery(); + } + + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(fieldData.getFieldName())) { + visitor.visitLeaf(this); + } + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (obj == null || obj.getClass() != getClass()) { + return false; + } + final SingleValueMatchQuery other = (SingleValueMatchQuery) obj; + return fieldData.getFieldName().equals(other.fieldData.getFieldName()); + } + + @Override + public int hashCode() { + return Objects.hash(classHash(), fieldData.getFieldName()); + } + + private static class DocIdSetIteratorScorerSupplier extends ScorerSupplier { + + private final Weight weight; + private final float score; + private final ScoreMode scoreMode; + private final DocIdSetIterator docIdSetIterator; + + private DocIdSetIteratorScorerSupplier(Weight weight, float score, ScoreMode scoreMode, DocIdSetIterator docIdSetIterator) { + this.weight = weight; + this.score = score; + this.scoreMode = scoreMode; + this.docIdSetIterator = docIdSetIterator; + } + + @Override + public Scorer get(long leadCost) { + return new ConstantScoreScorer(weight, score, scoreMode, docIdSetIterator); + } + + @Override + public long cost() { + return docIdSetIterator.cost(); + } + } + + private static class PredicateScorerSupplier extends ScorerSupplier { + + private final Weight weight; + private final float score; + private final ScoreMode scoreMode; + private final int maxDoc; + private final int matchCost; + private final CheckedIntPredicate predicate; + + private PredicateScorerSupplier( + Weight weight, + float score, + ScoreMode scoreMode, + int maxDoc, + int matchCost, + CheckedIntPredicate predicate + ) { + this.weight = weight; + this.score = score; + this.scoreMode = scoreMode; + this.maxDoc = maxDoc; + this.matchCost = matchCost; + this.predicate = predicate; + } + + @Override + public Scorer get(long leadCost) { + TwoPhaseIterator iterator = new TwoPhaseIterator(DocIdSetIterator.all(maxDoc)) { + @Override + public boolean matches() throws IOException { + return predicate.test(approximation.docID()); + } + + @Override + public float matchCost() { + return matchCost; + } + }; + return new ConstantScoreScorer(weight, score, scoreMode, iterator); + } + + @Override + public long cost() { + return maxDoc; + } + } + + @FunctionalInterface + private interface CheckedIntPredicate { + boolean test(int doc) throws IOException; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java index 4cd51b676fe89..07db69e6c5e51 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java @@ -7,32 +7,15 @@ package org.elasticsearch.xpack.esql.querydsl.query; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.PointValues; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.index.Terms; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; -import org.apache.lucene.search.QueryVisitor; -import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.TwoPhaseIterator; -import org.apache.lucene.search.Weight; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.LeafFieldData; -import org.elasticsearch.index.fielddata.LeafNumericFieldData; -import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData; -import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.MatchNoneQueryBuilder; @@ -71,8 +54,6 @@ public class SingleValueQuery extends Query { Builder::new ); - public static final String MULTI_VALUE_WARNING = "single-value function encountered multi-value"; - private final Query next; private final String field; @@ -84,7 +65,7 @@ public SingleValueQuery(Query next, String field) { @Override public Builder asBuilder() { - return new Builder(next.asBuilder(), field, new Stats(), next.source()); + return new Builder(next.asBuilder(), field, next.source()); } @Override @@ -114,13 +95,11 @@ public int hashCode() { public static class Builder extends AbstractQueryBuilder { private final QueryBuilder next; private final String field; - private final Stats stats; private final Source source; - Builder(QueryBuilder next, String field, Stats stats, Source source) { + Builder(QueryBuilder next, String field, Source source) { this.next = next; this.field = field; - this.stats = stats; this.source = source; } @@ -128,7 +107,6 @@ public static class Builder extends AbstractQueryBuilder { super(in); this.next = in.readNamedWriteable(QueryBuilder.class); this.field = in.readString(); - this.stats = new Stats(); if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { this.source = readSource(in); } else { @@ -181,28 +159,33 @@ public TransportVersion getMinimalSupportedVersion() { protected org.apache.lucene.search.Query doToQuery(SearchExecutionContext context) throws IOException { MappedFieldType ft = context.getFieldType(field); if (ft == null) { - stats.missingField++; return new MatchNoDocsQuery("missing field [" + field + "]"); } - return new LuceneQuery( - next.toQuery(context), + SingleValueMatchQuery singleValueQuery = new SingleValueMatchQuery( context.getForField(ft, MappedFieldType.FielddataOperation.SEARCH), - stats, new Warnings(source) ); + org.apache.lucene.search.Query rewrite = singleValueQuery.rewrite(context.searcher()); + if (rewrite instanceof MatchAllDocsQuery) { + // nothing to filter + return next.toQuery(context); + } + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(next.toQuery(context), BooleanClause.Occur.FILTER); + builder.add(rewrite, BooleanClause.Occur.FILTER); + return builder.build(); } @Override protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { QueryBuilder rewritten = next.rewrite(queryRewriteContext); if (rewritten instanceof MatchNoneQueryBuilder) { - stats.rewrittenToMatchNone++; return rewritten; } if (rewritten == next) { return this; } - return new Builder(rewritten, field, stats, source); + return new Builder(rewritten, field, source); } @Override @@ -214,526 +197,6 @@ protected boolean doEquals(Builder other) { protected int doHashCode() { return Objects.hash(next, field); } - - Stats stats() { - return stats; - } - } - - private static class LuceneQuery extends org.apache.lucene.search.Query { - final org.apache.lucene.search.Query next; - private final IndexFieldData fieldData; - // mutable object for collecting stats and warnings, not really part of the query - private final Stats stats; - private final Warnings warnings; - - LuceneQuery(org.apache.lucene.search.Query next, IndexFieldData fieldData, Stats stats, Warnings warnings) { - this.next = next; - this.fieldData = fieldData; - this.stats = stats; - this.warnings = warnings; - } - - @Override - public void visit(QueryVisitor visitor) { - if (visitor.acceptField(fieldData.getFieldName())) { - visitor.visitLeaf(next); - } - } - - @Override - public org.apache.lucene.search.Query rewrite(IndexReader reader) throws IOException { - org.apache.lucene.search.Query rewritten = next.rewrite(reader); - if (rewritten instanceof MatchNoDocsQuery) { - stats.rewrittenToMatchNone++; - return rewritten; - } - if (rewritten == next) { - return this; - } - return new LuceneQuery(rewritten, fieldData, stats, warnings); - } - - @Override - public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { - return new SingleValueWeight(this, next.createWeight(searcher, scoreMode, boost), fieldData, warnings); - } - - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; - } - if (obj == null || obj.getClass() != getClass()) { - return false; - } - SingleValueQuery.LuceneQuery other = (SingleValueQuery.LuceneQuery) obj; - return next.equals(other.next) && fieldData.getFieldName().equals(other.fieldData.getFieldName()); - } - - @Override - public int hashCode() { - return Objects.hash(classHash(), next, fieldData.getFieldName()); - } - - @Override - public String toString(String field) { - StringBuilder builder = new StringBuilder("single_value("); - if (false == this.fieldData.getFieldName().equals(field)) { - builder.append(this.fieldData.getFieldName()); - builder.append(":"); - } - builder.append(next); - return builder.append(")").toString(); - } - } - - private static class SingleValueWeight extends Weight { - private final Stats stats; - private final Weight next; - private final IndexFieldData fieldData; - private final Warnings warnings; - - private SingleValueWeight(SingleValueQuery.LuceneQuery query, Weight next, IndexFieldData fieldData, Warnings warnings) { - super(query); - this.stats = query.stats; - this.next = next; - this.fieldData = fieldData; - this.warnings = warnings; - } - - @Override - public Explanation explain(LeafReaderContext context, int doc) throws IOException { - Explanation nextExplanation = next.explain(context, doc); - if (false == nextExplanation.isMatch()) { - return Explanation.noMatch("next didn't match", nextExplanation); - } - LeafFieldData lfd = fieldData.load(context); - SortedBinaryDocValues values = lfd.getBytesValues(); - if (false == values.advanceExact(doc)) { - return Explanation.noMatch("no values in field", nextExplanation); - } - if (values.docValueCount() != 1) { - return Explanation.noMatch("field has too many values [" + values.docValueCount() + "]", nextExplanation); - } - return Explanation.match(nextExplanation.getValue(), "field has exactly 1 value", nextExplanation); - } - - @Override - public Scorer scorer(LeafReaderContext context) throws IOException { - Scorer nextScorer = next.scorer(context); - if (nextScorer == null) { - stats.noNextScorer++; - return null; - } - LeafFieldData lfd = fieldData.load(context); - /* - * SortedBinaryDocValues are available for most fields, but they - * are made available by eagerly converting non-bytes values to - * utf-8 strings. The eager conversion is quite expensive. So - * we specialize on numeric fields and fields with ordinals to - * avoid that expense in at least that case. - * - * Also! Lucene's FieldExistsQuery only needs one scorer that can - * use all the docs values iterators at DocIdSetIterators. We - * can't do that because we need the check the number of fields. - */ - if (lfd instanceof LeafNumericFieldData n) { - return scorer(context, nextScorer, n); - } - if (lfd instanceof LeafOrdinalsFieldData o) { - return scorer(context, nextScorer, o); - } - return scorer(nextScorer, lfd); - } - - private Scorer scorer(LeafReaderContext context, Scorer nextScorer, LeafNumericFieldData lfd) throws IOException { - SortedNumericDocValues sortedNumerics = lfd.getLongValues(); - if (DocValues.unwrapSingleton(sortedNumerics) != null) { - /* - * Segment contains only single valued fields. But it's possible - * that some fields have 0 values. The most surefire way to check - * is to look at the index for the data. If there isn't an index - * this isn't going to work - but if there is we can compare the - * number of documents in the index to the number of values in it - - * if they are the same we've got a dense singleton. - */ - PointValues points = context.reader().getPointValues(fieldData.getFieldName()); - if (points != null && points.getDocCount() == context.reader().maxDoc()) { - stats.numericSingle++; - return nextScorer; - } - } - TwoPhaseIterator nextIterator = nextScorer.twoPhaseIterator(); - if (nextIterator == null) { - stats.numericMultiNoApprox++; - return new SingleValueQueryScorer( - this, - nextScorer, - new TwoPhaseIteratorForSortedNumericsAndSinglePhaseQueries(nextScorer.iterator(), sortedNumerics, warnings) - ); - } - stats.numericMultiApprox++; - return new SingleValueQueryScorer( - this, - nextScorer, - new TwoPhaseIteratorForSortedNumericsAndTwoPhaseQueries(nextIterator, sortedNumerics, warnings) - ); - } - - private Scorer scorer(LeafReaderContext context, Scorer nextScorer, LeafOrdinalsFieldData lfd) throws IOException { - SortedSetDocValues sortedSet = lfd.getOrdinalsValues(); - if (DocValues.unwrapSingleton(sortedSet) != null) { - /* - * Segment contains only single valued fields. But it's possible - * that some fields have 0 values. The most surefire way to check - * is to look at the index for the data. If there isn't an index - * this isn't going to work - but if there is we can compare the - * number of documents in the index to the number of values in it - - * if they are the same we've got a dense singleton. - */ - Terms terms = context.reader().terms(fieldData.getFieldName()); - if (terms != null && terms.getDocCount() == context.reader().maxDoc()) { - stats.ordinalsSingle++; - return nextScorer; - } - } - TwoPhaseIterator nextIterator = nextScorer.twoPhaseIterator(); - if (nextIterator == null) { - stats.ordinalsMultiNoApprox++; - return new SingleValueQueryScorer( - this, - nextScorer, - new TwoPhaseIteratorForSortedSetAndSinglePhaseQueries(nextScorer.iterator(), sortedSet, warnings) - ); - } - stats.ordinalsMultiApprox++; - return new SingleValueQueryScorer( - this, - nextScorer, - new TwoPhaseIteratorForSortedSetAndTwoPhaseQueries(nextIterator, sortedSet, warnings) - ); - } - - private Scorer scorer(Scorer nextScorer, LeafFieldData lfd) { - SortedBinaryDocValues sortedBinary = lfd.getBytesValues(); - TwoPhaseIterator nextIterator = nextScorer.twoPhaseIterator(); - if (nextIterator == null) { - stats.bytesNoApprox++; - return new SingleValueQueryScorer( - this, - nextScorer, - new TwoPhaseIteratorForSortedBinaryAndSinglePhaseQueries(nextScorer.iterator(), sortedBinary, warnings) - ); - } - stats.bytesApprox++; - return new SingleValueQueryScorer( - this, - nextScorer, - new TwoPhaseIteratorForSortedBinaryAndTwoPhaseQueries(nextIterator, sortedBinary, warnings) - ); - } - - @Override - public boolean isCacheable(LeafReaderContext ctx) { - // we cannot cache this query because we loose the ability of emitting warnings - return false; - } - } - - private static class SingleValueQueryScorer extends Scorer { - private final Scorer next; - private final TwoPhaseIterator iterator; - - private SingleValueQueryScorer(Weight weight, Scorer next, TwoPhaseIterator iterator) { - super(weight); - this.next = next; - this.iterator = iterator; - } - - @Override - public DocIdSetIterator iterator() { - return TwoPhaseIterator.asDocIdSetIterator(iterator); - } - - @Override - public TwoPhaseIterator twoPhaseIterator() { - return iterator; - } - - @Override - public float getMaxScore(int upTo) throws IOException { - return next.getMaxScore(upTo); - } - - @Override - public float score() throws IOException { - return next.score(); - } - - @Override - public int docID() { - return next.docID(); - } - } - - /** - * The estimated number of comparisons to check if a {@link SortedNumericDocValues} - * has more than one value. There isn't a good way to get that number out of - * {@link SortedNumericDocValues} so this is a guess. - */ - private static final int SORTED_NUMERIC_MATCH_COST = 10; - - private static class TwoPhaseIteratorForSortedNumericsAndSinglePhaseQueries extends TwoPhaseIterator { - private final SortedNumericDocValues sortedNumerics; - private final Warnings warnings; - - private TwoPhaseIteratorForSortedNumericsAndSinglePhaseQueries( - DocIdSetIterator approximation, - SortedNumericDocValues sortedNumerics, - Warnings warning - ) { - super(approximation); - this.sortedNumerics = sortedNumerics; - this.warnings = warning; - } - - @Override - public boolean matches() throws IOException { - if (false == sortedNumerics.advanceExact(approximation.docID())) { - return false; - } - if (sortedNumerics.docValueCount() != 1) { - warnings.registerException(new IllegalArgumentException(MULTI_VALUE_WARNING)); - return false; - } - return true; - } - - @Override - public float matchCost() { - return SORTED_NUMERIC_MATCH_COST; - } - } - - private static class TwoPhaseIteratorForSortedNumericsAndTwoPhaseQueries extends TwoPhaseIterator { - private final SortedNumericDocValues sortedNumerics; - private final TwoPhaseIterator next; - private final Warnings warnings; - - private TwoPhaseIteratorForSortedNumericsAndTwoPhaseQueries( - TwoPhaseIterator next, - SortedNumericDocValues sortedNumerics, - Warnings warnings - ) { - super(next.approximation()); - this.sortedNumerics = sortedNumerics; - this.next = next; - this.warnings = warnings; - } - - @Override - public boolean matches() throws IOException { - if (false == sortedNumerics.advanceExact(approximation.docID())) { - return false; - } - if (sortedNumerics.docValueCount() != 1) { - warnings.registerException(new IllegalArgumentException(MULTI_VALUE_WARNING)); - return false; - } - return next.matches(); - } - - @Override - public float matchCost() { - return SORTED_NUMERIC_MATCH_COST + next.matchCost(); - } - } - - private static class TwoPhaseIteratorForSortedBinaryAndSinglePhaseQueries extends TwoPhaseIterator { - private final SortedBinaryDocValues sortedBinary; - private final Warnings warnings; - - private TwoPhaseIteratorForSortedBinaryAndSinglePhaseQueries( - DocIdSetIterator approximation, - SortedBinaryDocValues sortedBinary, - Warnings warnings - ) { - super(approximation); - this.sortedBinary = sortedBinary; - this.warnings = warnings; - } - - @Override - public boolean matches() throws IOException { - if (false == sortedBinary.advanceExact(approximation.docID())) { - return false; - } - if (sortedBinary.docValueCount() != 1) { - warnings.registerException(new IllegalArgumentException(MULTI_VALUE_WARNING)); - return false; - } - return true; - } - - @Override - public float matchCost() { - return SORTED_NUMERIC_MATCH_COST; - } - } - - private static class TwoPhaseIteratorForSortedSetAndTwoPhaseQueries extends TwoPhaseIterator { - private final SortedSetDocValues sortedSet; - private final TwoPhaseIterator next; - private final Warnings warnings; - - private TwoPhaseIteratorForSortedSetAndTwoPhaseQueries(TwoPhaseIterator next, SortedSetDocValues sortedSet, Warnings warnings) { - super(next.approximation()); - this.sortedSet = sortedSet; - this.next = next; - this.warnings = warnings; - } - - @Override - public boolean matches() throws IOException { - if (false == sortedSet.advanceExact(approximation.docID())) { - return false; - } - if (sortedSet.docValueCount() != 1) { - warnings.registerException(new IllegalArgumentException(MULTI_VALUE_WARNING)); - return false; - } - return next.matches(); - } - - @Override - public float matchCost() { - return SORTED_NUMERIC_MATCH_COST + next.matchCost(); - } } - private static class TwoPhaseIteratorForSortedSetAndSinglePhaseQueries extends TwoPhaseIterator { - private final SortedSetDocValues sortedSet; - private final Warnings warnings; - - private TwoPhaseIteratorForSortedSetAndSinglePhaseQueries( - DocIdSetIterator approximation, - SortedSetDocValues sortedSet, - Warnings warnings - ) { - super(approximation); - this.sortedSet = sortedSet; - this.warnings = warnings; - } - - @Override - public boolean matches() throws IOException { - if (false == sortedSet.advanceExact(approximation.docID())) { - return false; - } - if (sortedSet.docValueCount() != 1) { - warnings.registerException(new IllegalArgumentException(MULTI_VALUE_WARNING)); - return false; - } - return true; - } - - @Override - public float matchCost() { - return SORTED_NUMERIC_MATCH_COST; - } - } - - private static class TwoPhaseIteratorForSortedBinaryAndTwoPhaseQueries extends TwoPhaseIterator { - private final SortedBinaryDocValues sortedBinary; - private final TwoPhaseIterator next; - private final Warnings warnings; - - private TwoPhaseIteratorForSortedBinaryAndTwoPhaseQueries( - TwoPhaseIterator next, - SortedBinaryDocValues sortedBinary, - Warnings warnings - ) { - super(next.approximation()); - this.sortedBinary = sortedBinary; - this.next = next; - this.warnings = warnings; - } - - @Override - public boolean matches() throws IOException { - if (false == sortedBinary.advanceExact(approximation.docID())) { - return false; - } - if (sortedBinary.docValueCount() != 1) { - warnings.registerException(new IllegalArgumentException(MULTI_VALUE_WARNING)); - return false; - } - return next.matches(); - } - - @Override - public float matchCost() { - return SORTED_NUMERIC_MATCH_COST + next.matchCost(); - } - } - - static class Stats { - // TODO expose stats somehow - private int missingField; - private int rewrittenToMatchNone; - private int noNextScorer; - private int numericSingle; - private int numericMultiNoApprox; - private int numericMultiApprox; - private int ordinalsSingle; - private int ordinalsMultiNoApprox; - private int ordinalsMultiApprox; - private int bytesNoApprox; - private int bytesApprox; - - int missingField() { - return missingField; - } - - int rewrittenToMatchNone() { - return rewrittenToMatchNone; - } - - int noNextScorer() { - return noNextScorer; - } - - int numericSingle() { - return numericSingle; - } - - int numericMultiNoApprox() { - return numericMultiNoApprox; - } - - int numericMultiApprox() { - return numericMultiApprox; - } - - int ordinalsSingle() { - return ordinalsSingle; - } - - int ordinalsMultiNoApprox() { - return ordinalsMultiNoApprox; - } - - int ordinalsMultiApprox() { - return ordinalsMultiApprox; - } - - int bytesNoApprox() { - return bytesNoApprox; - } - - int bytesApprox() { - return bytesApprox; - } - } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java index 23de36d6d3d77..7a47b1d38f053 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java @@ -35,7 +35,6 @@ import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.util.Objects; @@ -58,7 +57,7 @@ public SpatialRelatesQuery(Source source, String field, ShapeField.QueryRelation @Override public QueryBuilder asBuilder() { - return EsqlDataTypes.isSpatialGeo(dataType) ? new GeoShapeQueryBuilder() : new CartesianShapeQueryBuilder(); + return DataType.isSpatialGeo(dataType) ? new GeoShapeQueryBuilder() : new CartesianShapeQueryBuilder(); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 8c831cc260e03..ff234c4a91572 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -10,7 +10,10 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.fieldcaps.FieldCapabilities; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.compute.operator.DriverProfile; +import org.elasticsearch.core.Releasables; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; @@ -46,6 +49,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.Keep; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.logical.Phased; import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; import org.elasticsearch.xpack.esql.plan.physical.EstimatesRowSize; @@ -125,14 +129,51 @@ public void execute( ); } + /** + * Execute an analyzed plan. Most code should prefer calling {@link #execute} but + * this is public for testing. See {@link Phased} for the sequence of operations. + */ public void executeAnalyzedPlan( EsqlQueryRequest request, BiConsumer> runPhase, LogicalPlan analyzedPlan, ActionListener listener ) { - // TODO phased execution lands here. - runPhase.accept(logicalPlanToPhysicalPlan(analyzedPlan, request), listener); + LogicalPlan firstPhase = Phased.extractFirstPhase(analyzedPlan); + if (firstPhase == null) { + runPhase.accept(logicalPlanToPhysicalPlan(analyzedPlan, request), listener); + } else { + executePhased(new ArrayList<>(), analyzedPlan, request, firstPhase, runPhase, listener); + } + } + + private void executePhased( + List profileAccumulator, + LogicalPlan mainPlan, + EsqlQueryRequest request, + LogicalPlan firstPhase, + BiConsumer> runPhase, + ActionListener listener + ) { + PhysicalPlan physicalPlan = logicalPlanToPhysicalPlan(firstPhase, request); + runPhase.accept(physicalPlan, listener.delegateFailureAndWrap((next, result) -> { + try { + profileAccumulator.addAll(result.profiles()); + LogicalPlan newMainPlan = Phased.applyResultsFromFirstPhase(mainPlan, physicalPlan.output(), result.pages()); + LogicalPlan newFirstPhase = Phased.extractFirstPhase(newMainPlan); + if (newFirstPhase == null) { + PhysicalPlan finalPhysicalPlan = logicalPlanToPhysicalPlan(newMainPlan, request); + runPhase.accept(finalPhysicalPlan, next.delegateFailureAndWrap((finalListener, finalResult) -> { + profileAccumulator.addAll(finalResult.profiles()); + finalListener.onResponse(new Result(finalResult.schema(), finalResult.pages(), profileAccumulator)); + })); + } else { + executePhased(profileAccumulator, newMainPlan, request, newFirstPhase, runPhase, next); + } + } finally { + Releasables.closeExpectNoException(Releasables.wrap(Iterators.map(result.pages().iterator(), p -> p::releaseBlocks))); + } + })); } private LogicalPlan parse(String query, QueryParams params) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index 5fd7f0c230463..8fe00b70f1ff6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction; import org.elasticsearch.xpack.esql.core.index.EsIndex; import org.elasticsearch.xpack.esql.core.index.IndexResolution; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -75,7 +76,8 @@ public IndexResolver(Client client, DataTypeRegistry typeRegistry) { * Resolves a pattern to one (potentially compound meaning that spawns multiple indices) mapping. */ public void resolveAsMergedMapping(String indexWildcard, Set fieldNames, ActionListener listener) { - client.fieldCaps( + client.execute( + EsqlResolveFieldsAction.TYPE, createFieldCapsRequest(indexWildcard, fieldNames), listener.delegateFailureAndWrap((l, response) -> l.onResponse(mergedMappings(indexWildcard, response))) ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index 23a94bde56b1e..08387a9a825a4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -68,7 +68,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; -import static org.elasticsearch.xpack.esql.core.type.DataType.isPrimitive; +import static org.elasticsearch.xpack.esql.core.type.DataType.isPrimitiveAndSupported; import static org.elasticsearch.xpack.esql.core.type.DataType.isString; import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.safeDoubleToLong; import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.safeToInt; @@ -115,7 +115,7 @@ public static boolean canConvert(DataType from, DataType to) { return true; } // only primitives are supported so far - return isPrimitive(from) && isPrimitive(to) && converterFor(from, to) != null; + return isPrimitiveAndSupported(from) && isPrimitiveAndSupported(to) && converterFor(from, to) != null; } public static Converter converterFor(DataType from, DataType to) { @@ -142,7 +142,7 @@ public static Converter converterFor(DataType from, DataType to) { if (to == DataType.BOOLEAN) { return EsqlConverter.STRING_TO_BOOLEAN; } - if (EsqlDataTypes.isSpatial(to)) { + if (DataType.isSpatial(to)) { return EsqlConverter.STRING_TO_SPATIAL; } if (to == DataType.TIME_DURATION) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java index 4ddef25584eea..836ce35fa8f7f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java @@ -17,10 +17,10 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD; import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION; import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTime; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isDateTimeOrTemporal; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isNullOrDatePeriod; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isNullOrTemporalAmount; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isNullOrTimeDuration; +import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTimeOrTemporal; +import static org.elasticsearch.xpack.esql.core.type.DataType.isNullOrDatePeriod; +import static org.elasticsearch.xpack.esql.core.type.DataType.isNullOrTemporalAmount; +import static org.elasticsearch.xpack.esql.core.type.DataType.isNullOrTimeDuration; public class EsqlDataTypeRegistry implements DataTypeRegistry { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java deleted file mode 100644 index 8a75d3f379dd3..0000000000000 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.type; - -import org.elasticsearch.xpack.esql.core.type.DataType; - -import java.util.Locale; - -import static org.elasticsearch.xpack.esql.core.type.DataType.BYTE; -import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD; -import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT; -import static org.elasticsearch.xpack.esql.core.type.DataType.HALF_FLOAT; -import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; -import static org.elasticsearch.xpack.esql.core.type.DataType.NESTED; -import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; -import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT; -import static org.elasticsearch.xpack.esql.core.type.DataType.PARTIAL_AGG; -import static org.elasticsearch.xpack.esql.core.type.DataType.SCALED_FLOAT; -import static org.elasticsearch.xpack.esql.core.type.DataType.SHORT; -import static org.elasticsearch.xpack.esql.core.type.DataType.SOURCE; -import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; -import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION; -import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; -import static org.elasticsearch.xpack.esql.core.type.DataType.isNull; - -public final class EsqlDataTypes { - - private EsqlDataTypes() {} - - public static DataType fromTypeName(String name) { - return DataType.fromTypeName(name.toLowerCase(Locale.ROOT)); - } - - public static boolean isString(DataType t) { - return t == KEYWORD || t == TEXT; - } - - public static boolean isPrimitive(DataType t) { - return t != OBJECT && t != NESTED; - } - - public static boolean isDateTimeOrTemporal(DataType t) { - return DataType.isDateTime(t) || isTemporalAmount(t); - } - - public static boolean isTemporalAmount(DataType t) { - return t == DataType.DATE_PERIOD || t == DataType.TIME_DURATION; - } - - public static boolean isNullOrTemporalAmount(DataType t) { - return isTemporalAmount(t) || isNull(t); - } - - public static boolean isNullOrDatePeriod(DataType t) { - return t == DataType.DATE_PERIOD || isNull(t); - } - - public static boolean isNullOrTimeDuration(DataType t) { - return t == DataType.TIME_DURATION || isNull(t); - } - - public static boolean isSpatial(DataType t) { - return t == DataType.GEO_POINT || t == DataType.CARTESIAN_POINT || t == DataType.GEO_SHAPE || t == DataType.CARTESIAN_SHAPE; - } - - public static boolean isSpatialGeo(DataType t) { - return t == DataType.GEO_POINT || t == DataType.GEO_SHAPE; - } - - public static boolean isSpatialPoint(DataType t) { - return t == DataType.GEO_POINT || t == DataType.CARTESIAN_POINT; - } - - /** - * Supported types that can be contained in a block. - */ - public static boolean isRepresentable(DataType t) { - return t != OBJECT - && t != NESTED - && t != UNSUPPORTED - && t != DATE_PERIOD - && t != TIME_DURATION - && t != BYTE - && t != SHORT - && t != FLOAT - && t != SCALED_FLOAT - && t != SOURCE - && t != HALF_FLOAT - && t != PARTIAL_AGG - && t.isCounter() == false; - } - - public static boolean areCompatible(DataType left, DataType right) { - if (left == right) { - return true; - } else { - return (left == NULL || right == NULL) || (isString(left) && isString(right)) || (left.isNumeric() && right.isNumeric()); - } - } -} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 20b4d3a503f0c..6a9b7a0e0089d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -157,6 +157,7 @@ public class CsvTests extends ESTestCase { private final String testName; private final Integer lineNumber; private final CsvSpecReader.CsvTestCase testCase; + private final String instructions; private final EsqlConfiguration configuration = EsqlTestUtils.configuration( new QueryPragmas(Settings.builder().put("page_size", randomPageSize()).build()) @@ -211,17 +212,25 @@ private int randomPageSize() { } } - public CsvTests(String fileName, String groupName, String testName, Integer lineNumber, CsvSpecReader.CsvTestCase testCase) { + public CsvTests( + String fileName, + String groupName, + String testName, + Integer lineNumber, + CsvSpecReader.CsvTestCase testCase, + String instructions + ) { this.fileName = fileName; this.groupName = groupName; this.testName = testName; this.lineNumber = lineNumber; this.testCase = testCase; + this.instructions = instructions; } public final void test() throws Throwable { try { - assumeTrue("Test " + testName + " is not enabled", isEnabled(testName, Version.CURRENT)); + assumeTrue("Test " + testName + " is not enabled", isEnabled(testName, instructions, Version.CURRENT)); /* * The csv tests support all but a few features. The unsupported features * are tested in integration tests. diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java index 8c5a5a4b3ba3b..552aa9443438f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.data.Block; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.ExistsQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; @@ -126,6 +127,7 @@ public static NamedWriteableRegistry writableRegistry() { entries.addAll(Expression.getNamedWriteables()); entries.addAll(EsqlScalarFunction.getNamedWriteables()); entries.addAll(AggregateFunction.getNamedWriteables()); + entries.addAll(Block.getNamedWriteables()); return new NamedWriteableRegistry(entries); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index cff4d274dc49c..86610ae923af6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -41,6 +41,7 @@ import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ParserConstructor; import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; @@ -63,9 +64,13 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; import static org.elasticsearch.xpack.esql.action.EsqlQueryResponse.DROP_NULL_COLUMNS_OPTION; -import static org.elasticsearch.xpack.esql.action.ResponseValueUtils.valuesToPage; import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN; import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToLong; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.longToUnsignedLong; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToIP; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToVersion; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; @@ -122,7 +127,10 @@ EsqlQueryResponse randomResponseAsync(boolean columnar, EsqlQueryResponse.Profil private ColumnInfoImpl randomColumnInfo() { DataType type = randomValueOtherThanMany( - t -> false == DataType.isPrimitive(t) || t == DataType.DATE_PERIOD || t == DataType.TIME_DURATION || t == DataType.PARTIAL_AGG, + t -> false == DataType.isPrimitiveAndSupported(t) + || t == DataType.DATE_PERIOD + || t == DataType.TIME_DURATION + || t == DataType.PARTIAL_AGG, () -> randomFrom(DataType.types()) ).widenSmallNumeric(); return new ColumnInfoImpl(randomAlphaOfLength(10), type.esType()); @@ -254,6 +262,13 @@ protected EsqlQueryResponse doParseInstance(XContentParser parser) { return ResponseBuilder.fromXContent(parser); } + /** + * Used to test round tripping through x-content. Unlike lots of other + * response objects, ESQL doesn't have production code that can parse + * the response because it doesn't need it. But we want to test random + * responses are valid. This helps with that by parsing it into a + * response. + */ public static class ResponseBuilder { private static final ParseField ID = new ParseField("id"); private static final ParseField IS_RUNNING = new ParseField("is_running"); @@ -625,4 +640,56 @@ static List columnValues(Iterator values) { values.forEachRemaining(l::add); return l; } + + /** + * Converts a list of values to Pages so that we can parse from xcontent, so we + * can test round tripping. This is functionally the inverse of {@link PositionToXContent}. + */ + static Page valuesToPage(BlockFactory blockFactory, List columns, List> values) { + List dataTypes = columns.stream().map(ColumnInfoImpl::type).toList(); + List results = dataTypes.stream() + .map(c -> PlannerUtils.toElementType(c).newBlockBuilder(values.size(), blockFactory)) + .toList(); + + for (List row : values) { + for (int c = 0; c < row.size(); c++) { + var builder = results.get(c); + var value = row.get(c); + switch (dataTypes.get(c)) { + case UNSIGNED_LONG -> ((LongBlock.Builder) builder).appendLong(longToUnsignedLong(((Number) value).longValue(), true)); + case LONG, COUNTER_LONG -> ((LongBlock.Builder) builder).appendLong(((Number) value).longValue()); + case INTEGER, COUNTER_INTEGER -> ((IntBlock.Builder) builder).appendInt(((Number) value).intValue()); + case DOUBLE, COUNTER_DOUBLE -> ((DoubleBlock.Builder) builder).appendDouble(((Number) value).doubleValue()); + case KEYWORD, TEXT, UNSUPPORTED -> ((BytesRefBlock.Builder) builder).appendBytesRef(new BytesRef(value.toString())); + case IP -> ((BytesRefBlock.Builder) builder).appendBytesRef(stringToIP(value.toString())); + case DATETIME -> { + long longVal = dateTimeToLong(value.toString()); + ((LongBlock.Builder) builder).appendLong(longVal); + } + case BOOLEAN -> ((BooleanBlock.Builder) builder).appendBoolean(((Boolean) value)); + case NULL -> builder.appendNull(); + case VERSION -> ((BytesRefBlock.Builder) builder).appendBytesRef(stringToVersion(new BytesRef(value.toString()))); + case SOURCE -> { + @SuppressWarnings("unchecked") + Map o = (Map) value; + try { + try (XContentBuilder sourceBuilder = JsonXContent.contentBuilder()) { + sourceBuilder.map(o); + ((BytesRefBlock.Builder) builder).appendBytesRef(BytesReference.bytes(sourceBuilder).toBytesRef()); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + case GEO_POINT, GEO_SHAPE, CARTESIAN_POINT, CARTESIAN_SHAPE -> { + // This just converts WKT to WKB, so does not need CRS knowledge, we could merge GEO and CARTESIAN here + BytesRef wkb = stringToSpatial(value.toString()); + ((BytesRefBlock.Builder) builder).appendBytesRef(wkb); + } + } + } + } + return new Page(results.stream().map(Block.Builder::build).toArray(Block[]::new)); + } + } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index d6cd4a5e84d49..73aa9ee73bfb4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -43,7 +43,6 @@ import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; -import org.elasticsearch.xpack.esql.plan.logical.EsqlUnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Filter; import org.elasticsearch.xpack.esql.plan.logical.Limit; @@ -51,6 +50,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Lookup; import org.elasticsearch.xpack.esql.plan.logical.OrderBy; import org.elasticsearch.xpack.esql.plan.logical.Row; +import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.elasticsearch.xpack.esql.session.IndexResolver; @@ -90,11 +90,13 @@ //@TestLogging(value = "org.elasticsearch.xpack.esql.analysis:TRACE", reason = "debug") public class AnalyzerTests extends ESTestCase { - private static final EsqlUnresolvedRelation UNRESOLVED_RELATION = new EsqlUnresolvedRelation( + private static final UnresolvedRelation UNRESOLVED_RELATION = new UnresolvedRelation( EMPTY, new TableIdentifier(EMPTY, null, "idx"), + false, List.of(), - IndexMode.STANDARD + IndexMode.STANDARD, + null ); private static final int MAX_LIMIT = EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY); @@ -1830,15 +1832,15 @@ public void testUnsupportedTypesInStats() { Found 8 problems line 2:12: argument of [avg(x)] must be [numeric except unsigned_long or counter types],\ found value [x] type [unsigned_long] - line 2:20: argument of [count_distinct(x)] must be [any exact type except unsigned_long or counter types],\ + line 2:20: argument of [count_distinct(x)] must be [any exact type except unsigned_long, _source, or counter types],\ found value [x] type [unsigned_long] - line 2:39: argument of [max(x)] must be [boolean, datetime or numeric except unsigned_long or counter types],\ + line 2:39: argument of [max(x)] must be [boolean, datetime, ip or numeric except unsigned_long or counter types],\ found value [max(x)] type [unsigned_long] line 2:47: argument of [median(x)] must be [numeric except unsigned_long or counter types],\ found value [x] type [unsigned_long] line 2:58: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter types],\ found value [x] type [unsigned_long] - line 2:88: argument of [min(x)] must be [boolean, datetime or numeric except unsigned_long or counter types],\ + line 2:88: argument of [min(x)] must be [boolean, datetime, ip or numeric except unsigned_long or counter types],\ found value [min(x)] type [unsigned_long] line 2:96: first argument of [percentile(x, 10)] must be [numeric except unsigned_long],\ found value [x] type [unsigned_long] @@ -1852,13 +1854,13 @@ public void testUnsupportedTypesInStats() { Found 7 problems line 2:10: argument of [avg(x)] must be [numeric except unsigned_long or counter types],\ found value [x] type [version] - line 2:18: argument of [max(x)] must be [boolean, datetime or numeric except unsigned_long or counter types],\ + line 2:18: argument of [max(x)] must be [boolean, datetime, ip or numeric except unsigned_long or counter types],\ found value [max(x)] type [version] line 2:26: argument of [median(x)] must be [numeric except unsigned_long or counter types],\ found value [x] type [version] line 2:37: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter types],\ found value [x] type [version] - line 2:67: argument of [min(x)] must be [boolean, datetime or numeric except unsigned_long or counter types],\ + line 2:67: argument of [min(x)] must be [boolean, datetime, ip or numeric except unsigned_long or counter types],\ found value [min(x)] type [version] line 2:75: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], found value [x] type [version] line 2:94: argument of [sum(x)] must be [numeric except unsigned_long or counter types], found value [x] type [version]"""); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 00d12240e67e5..f1ea1387c59e6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.parser.QueryParam; import org.elasticsearch.xpack.esql.parser.QueryParams; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.util.ArrayList; import java.util.List; @@ -313,7 +312,7 @@ public void testMixedNumericalNonConvertibleTypesInIn() { public void testUnsignedLongTypeMixInComparisons() { List types = DataType.types() .stream() - .filter(dt -> dt.isNumeric() && EsqlDataTypes.isRepresentable(dt) && dt != UNSIGNED_LONG) + .filter(dt -> dt.isNumeric() && DataType.isRepresentable(dt) && dt != UNSIGNED_LONG) .map(DataType::typeName) .toList(); for (var type : types) { @@ -351,7 +350,7 @@ public void testUnsignedLongTypeMixInComparisons() { public void testUnsignedLongTypeMixInArithmetics() { List types = DataType.types() .stream() - .filter(dt -> dt.isNumeric() && EsqlDataTypes.isRepresentable(dt) && dt != UNSIGNED_LONG) + .filter(dt -> dt.isNumeric() && DataType.isRepresentable(dt) && dt != UNSIGNED_LONG) .map(DataType::typeName) .toList(); for (var type : types) { @@ -494,7 +493,7 @@ public void testAggregateOnCounter() { error("FROM tests | STATS min(network.bytes_in)", tsdb), equalTo( "1:20: argument of [min(network.bytes_in)] must be" - + " [boolean, datetime or numeric except unsigned_long or counter types]," + + " [boolean, datetime, ip or numeric except unsigned_long or counter types]," + " found value [min(network.bytes_in)] type [counter_long]" ) ); @@ -503,7 +502,7 @@ public void testAggregateOnCounter() { error("FROM tests | STATS max(network.bytes_in)", tsdb), equalTo( "1:20: argument of [max(network.bytes_in)] must be" - + " [boolean, datetime or numeric except unsigned_long or counter types]," + + " [boolean, datetime, ip or numeric except unsigned_long or counter types]," + " found value [max(network.bytes_in)] type [counter_long]" ) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java index 792c6b5139796..25ff4f9c2122d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java @@ -10,8 +10,12 @@ import org.elasticsearch.compute.aggregation.Aggregator; import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.AggregatorMode; +import org.elasticsearch.compute.aggregation.GroupingAggregator; +import org.elasticsearch.compute.aggregation.SeenGroupIds; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.core.Releasables; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -54,7 +58,6 @@ public abstract class AbstractAggregationTestCase extends AbstractFunctionTestCa *

    */ protected static Iterable parameterSuppliersFromTypedDataWithDefaultChecks(List suppliers) { - // TODO: Add case with no input expecting null return parameterSuppliersFromTypedData(withNoRowsExpectingNull(randomizeBytesRefsOffset(suppliers))); } @@ -100,6 +103,12 @@ public void testAggregate() { resolveExpression(expression, this::aggregateSingleMode, this::evaluate); } + public void testGroupingAggregate() { + Expression expression = randomBoolean() ? buildDeepCopyOfFieldExpression(testCase) : buildFieldExpression(testCase); + + resolveExpression(expression, this::aggregateGroupingSingleMode, this::evaluate); + } + public void testAggregateIntermediate() { Expression expression = randomBoolean() ? buildDeepCopyOfFieldExpression(testCase) : buildFieldExpression(testCase); @@ -155,6 +164,39 @@ private void aggregateSingleMode(Expression expression) { } } + private void aggregateGroupingSingleMode(Expression expression) { + var pages = rows(testCase.getMultiRowFields()); + List results; + try { + assumeFalse("Grouping aggregations must receive data to check results", pages.isEmpty()); + + try (var aggregator = groupingAggregator(expression, initialInputChannels(), AggregatorMode.SINGLE)) { + var groupCount = randomIntBetween(1, 1000); + for (Page inputPage : pages) { + processPageGrouping(aggregator, inputPage, groupCount); + } + + results = extractResultsFromAggregator(aggregator, PlannerUtils.toElementType(testCase.expectedType()), groupCount); + } + } finally { + for (var page : pages) { + page.releaseBlocks(); + } + } + + for (var result : results) { + assertThat(result, not(equalTo(Double.NaN))); + assert testCase.getMatcher().matches(Double.POSITIVE_INFINITY) == false; + assertThat(result, not(equalTo(Double.POSITIVE_INFINITY))); + assert testCase.getMatcher().matches(Double.NEGATIVE_INFINITY) == false; + assertThat(result, not(equalTo(Double.NEGATIVE_INFINITY))); + assertThat(result, testCase.getMatcher()); + if (testCase.getExpectedWarnings() != null) { + assertWarnings(testCase.getExpectedWarnings()); + } + } + } + private void aggregateWithIntermediates(Expression expression) { int intermediateBlockOffset = randomIntBetween(0, 10); Block[] intermediateBlocks; @@ -293,6 +335,29 @@ private Object extractResultFromAggregator(Aggregator aggregator, ElementType ex } } + /** + * Returns a list of results, one for each group from 0 to {@code groupCount} + */ + private List extractResultsFromAggregator(GroupingAggregator aggregator, ElementType expectedElementType, int groupCount) { + var blocksArraySize = randomIntBetween(1, 10); + var resultBlockIndex = randomIntBetween(0, blocksArraySize - 1); + var blocks = new Block[blocksArraySize]; + try (var groups = IntVector.range(0, groupCount, driverContext().blockFactory())) { + aggregator.evaluate(blocks, resultBlockIndex, groups, driverContext()); + + var block = blocks[resultBlockIndex]; + + // For null blocks, the element type is NULL, so if the provided matcher matches, the type works too + assertThat(block.elementType(), is(oneOf(expectedElementType, ElementType.NULL))); + + return IntStream.range(resultBlockIndex, groupCount) + .mapToObj(position -> toJavaObject(blocks[resultBlockIndex], position)) + .toList(); + } finally { + Releasables.close(blocks); + } + } + private List initialInputChannels() { // TODO: Randomize channels // TODO: If surrogated, channels may change @@ -338,4 +403,61 @@ private Aggregator aggregator(Expression expression, List inputChannels return new Aggregator(aggregatorFunctionSupplier.aggregator(driverContext()), mode); } + + private GroupingAggregator groupingAggregator(Expression expression, List inputChannels, AggregatorMode mode) { + AggregatorFunctionSupplier aggregatorFunctionSupplier = ((ToAggregator) expression).supplier(inputChannels); + + return new GroupingAggregator(aggregatorFunctionSupplier.groupingAggregator(driverContext()), mode); + } + + /** + * Make a groupsId block with all the groups in the range for each row. + */ + private IntBlock makeGroupsVector(int groupStart, int groupEnd, int rowCount) { + try (var groupsBuilder = driverContext().blockFactory().newIntBlockBuilder(rowCount)) { + for (var i = 0; i < rowCount; i++) { + groupsBuilder.beginPositionEntry(); + for (int groupId = groupStart; groupId < groupEnd; groupId++) { + groupsBuilder.appendInt(groupId); + } + groupsBuilder.endPositionEntry(); + } + + return groupsBuilder.build(); + } + } + + /** + * Process the page with the aggregator. Adds all the values in all the groups in the range [0, {@code groupCount}). + *

    + * This method splits the data and groups in chunks, to test the aggregator capabilities. + *

    + */ + private void processPageGrouping(GroupingAggregator aggregator, Page inputPage, int groupCount) { + var groupSliceSize = 1; + // Add data to chunks of groups + for (int currentGroupOffset = 0; currentGroupOffset < groupCount;) { + var seenGroupIds = new SeenGroupIds.Range(0, currentGroupOffset + groupSliceSize); + var addInput = aggregator.prepareProcessPage(seenGroupIds, inputPage); + + var positionCount = inputPage.getPositionCount(); + var dataSliceSize = 1; + // Divide data in chunks + for (int currentDataOffset = 0; currentDataOffset < positionCount;) { + try (var groups = makeGroupsVector(currentGroupOffset, currentGroupOffset + groupSliceSize, dataSliceSize)) { + addInput.add(currentDataOffset, groups); + } + + currentDataOffset += dataSliceSize; + if (positionCount > currentDataOffset) { + dataSliceSize = randomIntBetween(1, Math.min(100, positionCount - currentDataOffset)); + } + } + + currentGroupOffset += groupSliceSize; + if (groupCount > currentGroupOffset) { + groupSliceSize = randomIntBetween(1, Math.min(100, groupCount - currentGroupOffset)); + } + } + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index 80dc2e434ab0f..0ec0a29dc530b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -40,12 +40,30 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; +import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull; +import org.elasticsearch.xpack.esql.core.session.Configuration; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.util.NumericUtils; import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.evaluator.EvalMapper; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Div; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mod; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mul; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Sub; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.esql.optimizer.FoldNull; import org.elasticsearch.xpack.esql.parser.ExpressionBuilder; import org.elasticsearch.xpack.esql.planner.Layout; @@ -56,6 +74,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.file.Files; @@ -76,10 +95,13 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static java.util.Map.entry; import static org.elasticsearch.compute.data.BlockUtils.toJavaObject; import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization; import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN; import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO; +import static org.hamcrest.Matchers.either; +import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; @@ -88,32 +110,55 @@ * Base class for function tests. */ public abstract class AbstractFunctionTestCase extends ESTestCase { + /** + * Operators are unregistered functions. + */ + private static final Map> OPERATORS = Map.ofEntries( + entry("in", In.class), + entry("like", WildcardLike.class), + entry("rlike", RLike.class), + entry("equals", Equals.class), + entry("not_equals", NotEquals.class), + entry("greater_than", GreaterThan.class), + entry("greater_than_or_equal", GreaterThanOrEqual.class), + entry("less_than", LessThan.class), + entry("less_than_or_equal", LessThanOrEqual.class), + entry("add", Add.class), + entry("sub", Sub.class), + entry("mul", Mul.class), + entry("div", Div.class), + entry("mod", Mod.class), + entry("neg", Neg.class), + entry("is_null", IsNull.class), + entry("is_not_null", IsNotNull.class) + ); + /** * Generate a random value of the appropriate type to fit into blocks of {@code e}. */ public static Literal randomLiteral(DataType type) { - return new Literal(Source.EMPTY, switch (type.typeName()) { - case "boolean" -> randomBoolean(); - case "byte" -> randomByte(); - case "short" -> randomShort(); - case "integer", "counter_integer" -> randomInt(); - case "unsigned_long", "long", "counter_long" -> randomLong(); - case "date_period" -> Period.of(randomIntBetween(-1000, 1000), randomIntBetween(-13, 13), randomIntBetween(-32, 32)); - case "datetime" -> randomMillisUpToYear9999(); - case "double", "scaled_float", "counter_double" -> randomDouble(); - case "float" -> randomFloat(); - case "half_float" -> HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(randomFloat())); - case "keyword" -> new BytesRef(randomAlphaOfLength(5)); - case "ip" -> new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean()))); - case "time_duration" -> Duration.ofMillis(randomLongBetween(-604800000L, 604800000L)); // plus/minus 7 days - case "text" -> new BytesRef(randomAlphaOfLength(50)); - case "version" -> randomVersion().toBytesRef(); - case "geo_point" -> GEO.asWkb(GeometryTestUtils.randomPoint()); - case "cartesian_point" -> CARTESIAN.asWkb(ShapeTestUtils.randomPoint()); - case "geo_shape" -> GEO.asWkb(GeometryTestUtils.randomGeometry(randomBoolean())); - case "cartesian_shape" -> CARTESIAN.asWkb(ShapeTestUtils.randomGeometry(randomBoolean())); - case "null" -> null; - case "_source" -> { + return new Literal(Source.EMPTY, switch (type) { + case BOOLEAN -> randomBoolean(); + case BYTE -> randomByte(); + case SHORT -> randomShort(); + case INTEGER, COUNTER_INTEGER -> randomInt(); + case UNSIGNED_LONG, LONG, COUNTER_LONG -> randomLong(); + case DATE_PERIOD -> Period.of(randomIntBetween(-1000, 1000), randomIntBetween(-13, 13), randomIntBetween(-32, 32)); + case DATETIME -> randomMillisUpToYear9999(); + case DOUBLE, SCALED_FLOAT, COUNTER_DOUBLE -> randomDouble(); + case FLOAT -> randomFloat(); + case HALF_FLOAT -> HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(randomFloat())); + case KEYWORD -> new BytesRef(randomAlphaOfLength(5)); + case IP -> new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean()))); + case TIME_DURATION -> Duration.ofMillis(randomLongBetween(-604800000L, 604800000L)); // plus/minus 7 days + case TEXT -> new BytesRef(randomAlphaOfLength(50)); + case VERSION -> randomVersion().toBytesRef(); + case GEO_POINT -> GEO.asWkb(GeometryTestUtils.randomPoint()); + case CARTESIAN_POINT -> CARTESIAN.asWkb(ShapeTestUtils.randomPoint()); + case GEO_SHAPE -> GEO.asWkb(GeometryTestUtils.randomGeometry(randomBoolean())); + case CARTESIAN_SHAPE -> CARTESIAN.asWkb(ShapeTestUtils.randomGeometry(randomBoolean())); + case NULL -> null; + case SOURCE -> { try { yield BytesReference.bytes( JsonXContent.contentBuilder().startObject().field(randomAlphaOfLength(3), randomAlphaOfLength(10)).endObject() @@ -122,7 +167,9 @@ public static Literal randomLiteral(DataType type) { throw new UncheckedIOException(e); } } - default -> throw new IllegalArgumentException("can't make random values for [" + type.typeName() + "]"); + case UNSUPPORTED, OBJECT, NESTED, DOC_DATA_TYPE, TSID_DATA_TYPE, PARTIAL_AGG -> throw new IllegalArgumentException( + "can't make random values for [" + type.typeName() + "]" + ); }, type); } @@ -303,18 +350,13 @@ protected static List randomizeBytesRefsOffset(List { - if (typedData.data() instanceof BytesRef bytesRef) { - var offset = randomIntBetween(0, 10); - var extraLength = randomIntBetween(0, 10); - var newBytesArray = randomByteArrayOfLength(bytesRef.length + offset + extraLength); - - System.arraycopy(bytesRef.bytes, bytesRef.offset, newBytesArray, offset, bytesRef.length); - - var newBytesRef = new BytesRef(newBytesArray, offset, bytesRef.length); - - return typedData.withData(newBytesRef); + if (typedData.isMultiRow()) { + return typedData.withData( + typedData.multiRowData().stream().map(AbstractFunctionTestCase::tryRandomizeBytesRefOffset).toList() + ); } - return typedData; + + return typedData.withData(tryRandomizeBytesRefOffset(typedData.data())); }).toList(); return new TestCaseSupplier.TestCase( @@ -330,6 +372,33 @@ protected static List randomizeBytesRefsOffset(List list) { + return list.stream().map(element -> { + if (element instanceof BytesRef bytesRef) { + return randomizeBytesRefOffset(bytesRef); + } + return element; + }).toList(); + } + + return value; + } + + private static BytesRef randomizeBytesRefOffset(BytesRef bytesRef) { + var offset = randomIntBetween(0, 10); + var extraLength = randomIntBetween(0, 10); + var newBytesArray = randomByteArrayOfLength(bytesRef.length + offset + extraLength); + + System.arraycopy(bytesRef.bytes, bytesRef.offset, newBytesArray, offset, bytesRef.length); + + return new BytesRef(newBytesArray, offset, bytesRef.length); + } + public void testSerializationOfSimple() { assertSerialization(buildFieldExpression(testCase)); } @@ -349,6 +418,12 @@ public static void testFunctionInfo() { List args = description.args(); assertTrue("expect description to be defined", description.description() != null && false == description.description().isEmpty()); + assertThat( + "descriptions should be complete sentences", + description.description(), + either(endsWith(".")) // A full sentence + .or(endsWith("∅")) // Math + ); List> typesFromSignature = new ArrayList<>(); Set returnFromSignature = new HashSet<>(); @@ -457,20 +532,8 @@ public static void renderDocs() throws IOException { return; } String name = functionName(); - if (binaryOperator(name) != null) { - renderTypes(List.of("lhs", "rhs")); - return; - } - if (unaryOperator(name) != null) { - renderTypes(List.of("v")); - return; - } - if (name.equalsIgnoreCase("rlike")) { - renderTypes(List.of("str", "pattern", "caseInsensitive")); - return; - } - if (name.equalsIgnoreCase("like")) { - renderTypes(List.of("str", "pattern")); + if (binaryOperator(name) != null || unaryOperator(name) != null || likeOrInOperator(name)) { + renderDocsForOperators(name); return; } FunctionDefinition definition = definition(name); @@ -481,7 +544,8 @@ public static void renderDocs() throws IOException { FunctionInfo info = EsqlFunctionRegistry.functionInfo(definition); renderDescription(description.description(), info.detailedDescription(), info.note()); boolean hasExamples = renderExamples(info); - renderFullLayout(name, hasExamples); + boolean hasAppendix = renderAppendix(info.appendix()); + renderFullLayout(name, hasExamples, hasAppendix); renderKibanaInlineDocs(name, info); List args = description.args(); if (name.equals("case")) { @@ -610,7 +674,19 @@ private static boolean renderExamples(FunctionInfo info) throws IOException { return true; } - private static void renderFullLayout(String name, boolean hasExamples) throws IOException { + private static boolean renderAppendix(String appendix) throws IOException { + if (appendix.isEmpty()) { + return false; + } + + String rendered = DOCS_WARNING + appendix + "\n"; + + LogManager.getLogger(getTestClass()).info("Writing appendix for [{}]:\n{}", functionName(), rendered); + writeToTempDir("appendix", rendered, "asciidoc"); + return true; + } + + private static void renderFullLayout(String name, boolean hasExamples, boolean hasAppendix) throws IOException { String rendered = DOCS_WARNING + """ [discrete] [[esql-$NAME$]] @@ -628,10 +704,48 @@ private static void renderFullLayout(String name, boolean hasExamples) throws IO if (hasExamples) { rendered += "include::../examples/" + name + ".asciidoc[]\n"; } + if (hasAppendix) { + rendered += "include::../appendix/" + name + ".asciidoc[]\n"; + } LogManager.getLogger(getTestClass()).info("Writing layout for [{}]:\n{}", functionName(), rendered); writeToTempDir("layout", rendered, "asciidoc"); } + private static Constructor constructorWithFunctionInfo(Class clazz) { + for (Constructor ctor : clazz.getConstructors()) { + FunctionInfo functionInfo = ctor.getAnnotation(FunctionInfo.class); + if (functionInfo != null) { + return ctor; + } + } + return null; + } + + private static void renderDocsForOperators(String name) throws IOException { + Constructor ctor = constructorWithFunctionInfo(OPERATORS.get(name)); + assert ctor != null; + FunctionInfo functionInfo = ctor.getAnnotation(FunctionInfo.class); + assert functionInfo != null; + renderKibanaInlineDocs(name, functionInfo); + + var params = ctor.getParameters(); + + List args = new ArrayList<>(params.length); + for (int i = 1; i < params.length; i++) { // skipping 1st argument, the source + if (Configuration.class.isAssignableFrom(params[i].getType()) == false) { + Param paramInfo = params[i].getAnnotation(Param.class); + String paramName = paramInfo == null ? params[i].getName() : paramInfo.name(); + String[] type = paramInfo == null ? new String[] { "?" } : paramInfo.type(); + String desc = paramInfo == null ? "" : paramInfo.description().replace('\n', ' '); + boolean optional = paramInfo == null ? false : paramInfo.optional(); + DataType targetDataType = EsqlFunctionRegistry.getTargetType(type); + args.add(new EsqlFunctionRegistry.ArgSignature(paramName, type, desc, optional, targetDataType)); + } + } + renderKibanaFunctionDefinition(name, functionInfo, args, likeOrInOperator(name)); + renderTypes(args.stream().map(EsqlFunctionRegistry.ArgSignature::name).toList()); + } + private static void renderKibanaInlineDocs(String name, FunctionInfo info) throws IOException { StringBuilder builder = new StringBuilder(); builder.append(""" @@ -806,6 +920,13 @@ private static String unaryOperator(String name) { }; } + /** + * If this tests is for a like or rlike operator return true, otherwise return {@code null}. + */ + private static boolean likeOrInOperator(String name) { + return name.equalsIgnoreCase("rlike") || name.equalsIgnoreCase("like") || name.equalsIgnoreCase("in"); + } + /** * Write some text to a tempdir so we can copy it to the docs later. *

    diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java index 1aa90d367099a..e50bc62b27265 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java @@ -30,16 +30,12 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.optimizer.FoldNull; import org.elasticsearch.xpack.esql.planner.PlannerUtils; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.hamcrest.Matcher; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -50,7 +46,7 @@ import java.util.stream.Stream; import static org.elasticsearch.compute.data.BlockUtils.toJavaObject; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isSpatial; +import static org.elasticsearch.xpack.esql.core.type.DataType.isSpatial; import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; @@ -74,10 +70,14 @@ public abstract class AbstractScalarFunctionTestCase extends AbstractFunctionTes */ protected static Iterable parameterSuppliersFromTypedDataWithDefaultChecks( boolean entirelyNullPreservesType, - List suppliers + List suppliers, + PositionalErrorMessageSupplier positionalErrorMessageSupplier ) { return parameterSuppliersFromTypedData( - errorsForCasesWithoutExamples(anyNullIsNull(entirelyNullPreservesType, randomizeBytesRefsOffset(suppliers))) + errorsForCasesWithoutExamples( + anyNullIsNull(entirelyNullPreservesType, randomizeBytesRefsOffset(suppliers)), + positionalErrorMessageSupplier + ) ); } @@ -89,7 +89,7 @@ public final void testEvaluate() { assertTypeResolutionFailure(expression); return; } - assumeTrue("Expected type must be representable to build an evaluator", EsqlDataTypes.isRepresentable(testCase.expectedType())); + assumeTrue("Expected type must be representable to build an evaluator", DataType.isRepresentable(testCase.expectedType())); logger.info( "Test Values: " + testCase.getData().stream().map(TestCaseSupplier.TypedData::toString).collect(Collectors.joining(",")) ); @@ -190,7 +190,7 @@ private void testEvaluateBlock(BlockFactory inputBlockFactory, DriverContext con return; } assumeTrue("Can't build evaluator", testCase.canBuildEvaluator()); - assumeTrue("Expected type must be representable to build an evaluator", EsqlDataTypes.isRepresentable(testCase.expectedType())); + assumeTrue("Expected type must be representable to build an evaluator", DataType.isRepresentable(testCase.expectedType())); int positions = between(1, 1024); List data = testCase.getData(); Page onePositionPage = row(testCase.getDataValues()); @@ -293,7 +293,7 @@ public final void testEvaluateInManyThreads() throws ExecutionException, Interru return; } assumeTrue("Can't build evaluator", testCase.canBuildEvaluator()); - assumeTrue("Expected type must be representable to build an evaluator", EsqlDataTypes.isRepresentable(testCase.expectedType())); + assumeTrue("Expected type must be representable to build an evaluator", DataType.isRepresentable(testCase.expectedType())); int count = 10_000; int threads = 5; var evalSupplier = evaluator(expression); @@ -480,8 +480,14 @@ protected static List anyNullIsNull( * Adds test cases containing unsupported parameter types that assert * that they throw type errors. */ - protected static List errorsForCasesWithoutExamples(List testCaseSuppliers) { - return errorsForCasesWithoutExamples(testCaseSuppliers, AbstractScalarFunctionTestCase::typeErrorMessage); + protected static List errorsForCasesWithoutExamples( + List testCaseSuppliers, + PositionalErrorMessageSupplier positionalErrorMessageSupplier + ) { + return errorsForCasesWithoutExamples( + testCaseSuppliers, + (i, v, t) -> AbstractScalarFunctionTestCase.typeErrorMessage(i, v, t, positionalErrorMessageSupplier) + ); } protected static List errorsForCasesWithoutExamples( @@ -515,10 +521,11 @@ protected static List errorsForCasesWithoutExamples( public static String errorMessageStringForBinaryOperators( boolean includeOrdinal, List> validPerPosition, - List types + List types, + PositionalErrorMessageSupplier positionalErrorMessageSupplier ) { try { - return typeErrorMessage(includeOrdinal, validPerPosition, types); + return typeErrorMessage(includeOrdinal, validPerPosition, types, positionalErrorMessageSupplier); } catch (IllegalStateException e) { // This means all the positional args were okay, so the expected error is from the combination if (types.get(0).equals(DataType.UNSIGNED_LONG)) { @@ -617,12 +624,33 @@ protected interface TypeErrorMessageSupplier { String apply(boolean includeOrdinal, List> validPerPosition, List types); } + @FunctionalInterface + protected interface PositionalErrorMessageSupplier { + /** + * This interface defines functions to supply error messages for incorrect types in specific positions. Functions which have + * the same type requirements for all positions can simplify this with a lambda returning a string constant. + * + * @param validForPosition - the set of {@link DataType}s that the test infrastructure believes to be allowable in the + * given position. + * @param position - the zero-index position in the list of parameters the function has detected the bad argument to be. + * @return The string describing the acceptable parameters for that position. Note that this function should not return + * the full error string; that will be constructed by the test. Just return the type string for that position. + */ + String apply(Set validForPosition, int position); + } + protected static TestCaseSupplier typeErrorSupplier( boolean includeOrdinal, List> validPerPosition, - List types + List types, + PositionalErrorMessageSupplier errorMessageSupplier ) { - return typeErrorSupplier(includeOrdinal, validPerPosition, types, AbstractScalarFunctionTestCase::typeErrorMessage); + return typeErrorSupplier( + includeOrdinal, + validPerPosition, + types, + (o, v, t) -> AbstractScalarFunctionTestCase.typeErrorMessage(o, v, t, errorMessageSupplier) + ); } /** @@ -647,7 +675,12 @@ protected static TestCaseSupplier typeErrorSupplier( /** * Build the expected error message for an invalid type signature. */ - protected static String typeErrorMessage(boolean includeOrdinal, List> validPerPosition, List types) { + protected static String typeErrorMessage( + boolean includeOrdinal, + List> validPerPosition, + List types, + PositionalErrorMessageSupplier expectedTypeSupplier + ) { int badArgPosition = -1; for (int i = 0; i < types.size(); i++) { if (validPerPosition.get(i).contains(types.get(i)) == false) { @@ -661,213 +694,13 @@ protected static String typeErrorMessage(boolean includeOrdinal, List, String> NAMED_EXPECTED_TYPES = Map.ofEntries( - Map.entry( - Set.of(DataType.DATE_PERIOD, DataType.DOUBLE, DataType.INTEGER, DataType.LONG, DataType.TIME_DURATION, DataType.NULL), - "numeric, date_period or time_duration" - ), - Map.entry(Set.of(DataType.DATETIME, DataType.NULL), "datetime"), - Map.entry(Set.of(DataType.DOUBLE, DataType.NULL), "double"), - Map.entry(Set.of(DataType.INTEGER, DataType.NULL), "integer"), - Map.entry(Set.of(DataType.IP, DataType.NULL), "ip"), - Map.entry(Set.of(DataType.LONG, DataType.INTEGER, DataType.UNSIGNED_LONG, DataType.DOUBLE, DataType.NULL), "numeric"), - Map.entry(Set.of(DataType.LONG, DataType.INTEGER, DataType.UNSIGNED_LONG, DataType.DOUBLE), "numeric"), - Map.entry(Set.of(DataType.KEYWORD, DataType.TEXT, DataType.VERSION, DataType.NULL), "string or version"), - Map.entry(Set.of(DataType.KEYWORD, DataType.TEXT, DataType.NULL), "string"), - Map.entry(Set.of(DataType.IP, DataType.KEYWORD, DataType.TEXT, DataType.NULL), "ip or string"), - Map.entry(Set.copyOf(Arrays.asList(representableTypes())), "representable"), - Map.entry(Set.copyOf(Arrays.asList(representableNonSpatialTypes())), "representableNonSpatial"), - Map.entry( - Set.of( - DataType.BOOLEAN, - DataType.DOUBLE, - DataType.INTEGER, - DataType.KEYWORD, - DataType.LONG, - DataType.TEXT, - DataType.UNSIGNED_LONG, - DataType.NULL - ), - "boolean or numeric or string" - ), - Map.entry( - Set.of( - DataType.DATETIME, - DataType.DOUBLE, - DataType.INTEGER, - DataType.KEYWORD, - DataType.LONG, - DataType.TEXT, - DataType.UNSIGNED_LONG, - DataType.NULL - ), - "datetime or numeric or string" - ), - // What Add accepts - Map.entry( - Set.of( - DataType.DATE_PERIOD, - DataType.DATETIME, - DataType.DOUBLE, - DataType.INTEGER, - DataType.LONG, - DataType.NULL, - DataType.TIME_DURATION, - DataType.UNSIGNED_LONG - ), - "datetime or numeric" - ), - Map.entry( - Set.of( - DataType.BOOLEAN, - DataType.DATETIME, - DataType.DOUBLE, - DataType.INTEGER, - DataType.KEYWORD, - DataType.LONG, - DataType.TEXT, - DataType.UNSIGNED_LONG, - DataType.NULL - ), - "boolean or datetime or numeric or string" - ), - // to_int - Map.entry( - Set.of( - DataType.BOOLEAN, - DataType.COUNTER_INTEGER, - DataType.DATETIME, - DataType.DOUBLE, - DataType.INTEGER, - DataType.KEYWORD, - DataType.LONG, - DataType.TEXT, - DataType.UNSIGNED_LONG, - DataType.NULL - ), - "boolean or counter_integer or datetime or numeric or string" - ), - // to_long - Map.entry( - Set.of( - DataType.BOOLEAN, - DataType.COUNTER_INTEGER, - DataType.COUNTER_LONG, - DataType.DATETIME, - DataType.DOUBLE, - DataType.INTEGER, - DataType.KEYWORD, - DataType.LONG, - DataType.TEXT, - DataType.UNSIGNED_LONG, - DataType.NULL - ), - "boolean or counter_integer or counter_long or datetime or numeric or string" - ), - // to_double - Map.entry( - Set.of( - DataType.BOOLEAN, - DataType.COUNTER_DOUBLE, - DataType.COUNTER_INTEGER, - DataType.COUNTER_LONG, - DataType.DATETIME, - DataType.DOUBLE, - DataType.INTEGER, - DataType.KEYWORD, - DataType.LONG, - DataType.TEXT, - DataType.UNSIGNED_LONG, - DataType.NULL - ), - "boolean or counter_double or counter_integer or counter_long or datetime or numeric or string" - ), - Map.entry( - Set.of( - DataType.BOOLEAN, - DataType.CARTESIAN_POINT, - DataType.DATETIME, - DataType.DOUBLE, - DataType.GEO_POINT, - DataType.INTEGER, - DataType.KEYWORD, - DataType.LONG, - DataType.TEXT, - DataType.UNSIGNED_LONG, - DataType.NULL - ), - "boolean or cartesian_point or datetime or geo_point or numeric or string" - ), - Map.entry( - Set.of( - DataType.DATETIME, - DataType.DOUBLE, - DataType.INTEGER, - DataType.IP, - DataType.KEYWORD, - DataType.LONG, - DataType.TEXT, - DataType.UNSIGNED_LONG, - DataType.VERSION, - DataType.NULL - ), - "datetime, double, integer, ip, keyword, long, text, unsigned_long or version" - ), - Map.entry( - Set.of( - DataType.BOOLEAN, - DataType.DATETIME, - DataType.DOUBLE, - DataType.GEO_POINT, - DataType.GEO_SHAPE, - DataType.INTEGER, - DataType.IP, - DataType.KEYWORD, - DataType.LONG, - DataType.TEXT, - DataType.UNSIGNED_LONG, - DataType.VERSION, - DataType.NULL - ), - "cartesian_point or datetime or geo_point or numeric or string" - ), - Map.entry(Set.of(DataType.GEO_POINT, DataType.KEYWORD, DataType.TEXT, DataType.NULL), "geo_point or string"), - Map.entry(Set.of(DataType.CARTESIAN_POINT, DataType.KEYWORD, DataType.TEXT, DataType.NULL), "cartesian_point or string"), - Map.entry( - Set.of(DataType.GEO_POINT, DataType.GEO_SHAPE, DataType.KEYWORD, DataType.TEXT, DataType.NULL), - "geo_point or geo_shape or string" - ), - Map.entry( - Set.of(DataType.CARTESIAN_POINT, DataType.CARTESIAN_SHAPE, DataType.KEYWORD, DataType.TEXT, DataType.NULL), - "cartesian_point or cartesian_shape or string" - ), - Map.entry(Set.of(DataType.GEO_POINT, DataType.CARTESIAN_POINT, DataType.NULL), "geo_point or cartesian_point"), - Map.entry(Set.of(DataType.DATE_PERIOD, DataType.TIME_DURATION, DataType.NULL), "dateperiod or timeduration") - ); - - // TODO: generate this message dynamically, a la AbstractConvertFunction#supportedTypesNames()? - private static String expectedType(Set validTypes) { - String named = NAMED_EXPECTED_TYPES.get(validTypes); - if (named == null) { - /* - * Note for anyone who's test lands here - it's likely that you - * don't have a test case covering explicit `null` arguments in - * this position. Generally you can get that with anyNullIsNull. - */ - throw new UnsupportedOperationException( - "can't guess expected types for " + validTypes.stream().sorted(Comparator.comparing(t -> t.typeName())).toList() - ); - } - return named; + return ordinal + "argument of [] must be [" + expectedTypeString + "], found value [" + name + "] type [" + name + "]"; } protected static Stream representable() { - return DataType.types().stream().filter(EsqlDataTypes::isRepresentable); + return DataType.types().stream().filter(DataType::isRepresentable); } protected static DataType[] representableTypes() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java index 68f5414302c9d..dd73e64fbd8da 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java @@ -7,6 +7,9 @@ package org.elasticsearch.xpack.esql.expression.function; +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -280,15 +283,9 @@ public static List dateCases(int minRows, int maxRows) { } public static List booleanCases(int minRows, int maxRows) { - List cases = new ArrayList<>(); - - cases.add(new TypedDataSupplier("", () -> randomList(minRows, maxRows, () -> true), DataType.BOOLEAN, false, true)); - - cases.add( - new TypedDataSupplier("", () -> randomList(minRows, maxRows, () -> false), DataType.BOOLEAN, false, true) - ); - - cases.add( + return List.of( + new TypedDataSupplier("", () -> randomList(minRows, maxRows, () -> true), DataType.BOOLEAN, false, true), + new TypedDataSupplier("", () -> randomList(minRows, maxRows, () -> false), DataType.BOOLEAN, false, true), new TypedDataSupplier( "", () -> randomList(minRows, maxRows, ESTestCase::randomBoolean), @@ -297,7 +294,31 @@ public static List booleanCases(int minRows, int maxRows) { true ) ); + } - return cases; + public static List ipCases(int minRows, int maxRows) { + return List.of( + new TypedDataSupplier( + "<127.0.0.1 ips>", + () -> randomList(minRows, maxRows, () -> new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1")))), + DataType.IP, + false, + true + ), + new TypedDataSupplier( + "", + () -> randomList(minRows, maxRows, () -> new BytesRef(InetAddressPoint.encode(ESTestCase.randomIp(true)))), + DataType.IP, + false, + true + ), + new TypedDataSupplier( + "", + () -> randomList(minRows, maxRows, () -> new BytesRef(InetAddressPoint.encode(ESTestCase.randomIp(false)))), + DataType.IP, + false, + true + ) + ); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index 6ece7151ccd7a..3130d852c1ab1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -21,7 +21,6 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.NumericUtils; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.versionfield.Version; import org.hamcrest.Matcher; @@ -991,6 +990,12 @@ public static List doubleCases(double min, double max, boolea return cases; } + /** + * Generate cases for {@link DataType#BOOLEAN}. + *

    + * For multi-row parameters, see {@link MultiRowTestCaseSupplier#booleanCases}. + *

    + */ public static List booleanCases() { return List.of( new TypedDataSupplier("", () -> true, DataType.BOOLEAN), @@ -1107,6 +1112,12 @@ public static List cartesianShapeCases(Supplier hasA ); } + /** + * Generate cases for {@link DataType#IP}. + *

    + * For multi-row parameters, see {@link MultiRowTestCaseSupplier#ipCases}. + *

    + */ public static List ipCases() { return List.of( new TypedDataSupplier( @@ -1296,7 +1307,7 @@ public static TestCase typeError(List data, String expectedTypeError) this.matcher = matcher; this.expectedWarnings = expectedWarnings; this.expectedTypeError = expectedTypeError; - this.canBuildEvaluator = data.stream().allMatch(d -> d.forceLiteral || EsqlDataTypes.isRepresentable(d.type)); + this.canBuildEvaluator = data.stream().allMatch(d -> d.forceLiteral || DataType.isRepresentable(d.type)); this.foldingExceptionClass = foldingExceptionClass; this.foldingExceptionMessage = foldingExceptionMessage; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java index 3fddaff226f3e..1d489e0146ad3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java @@ -10,6 +10,9 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -40,7 +43,8 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true), MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true), MultiRowTestCaseSupplier.dateCases(1, 1000), - MultiRowTestCaseSupplier.booleanCases(1, 1000) + MultiRowTestCaseSupplier.booleanCases(1, 1000), + MultiRowTestCaseSupplier.ipCases(1, 1000) ).flatMap(List::stream).map(MaxTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); suppliers.addAll( @@ -91,6 +95,26 @@ public static Iterable parameters() { equalTo(true) ) ), + new TestCaseSupplier( + List.of(DataType.IP), + () -> new TestCaseSupplier.TestCase( + List.of( + TestCaseSupplier.TypedData.multiRow( + List.of( + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1"))), + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))), + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::"))), + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("ffff::"))) + ), + DataType.IP, + "field" + ) + ), + "Max[field=Attribute[channel=0]]", + DataType.IP, + equalTo(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("ffff::")))) + ) + ), // Folding new TestCaseSupplier( @@ -137,6 +161,21 @@ public static Iterable parameters() { DataType.BOOLEAN, equalTo(true) ) + ), + new TestCaseSupplier( + List.of(DataType.IP), + () -> new TestCaseSupplier.TestCase( + List.of( + TestCaseSupplier.TypedData.multiRow( + List.of(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1")))), + DataType.IP, + "field" + ) + ), + "Max[field=Attribute[channel=0]]", + DataType.IP, + equalTo(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1")))) + ) ) ) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java index 6f59928059bec..b5fb5b2c1c414 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java @@ -10,6 +10,9 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -40,7 +43,8 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true), MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true), MultiRowTestCaseSupplier.dateCases(1, 1000), - MultiRowTestCaseSupplier.booleanCases(1, 1000) + MultiRowTestCaseSupplier.booleanCases(1, 1000), + MultiRowTestCaseSupplier.ipCases(1, 1000) ).flatMap(List::stream).map(MinTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); suppliers.addAll( @@ -91,6 +95,26 @@ public static Iterable parameters() { equalTo(false) ) ), + new TestCaseSupplier( + List.of(DataType.IP), + () -> new TestCaseSupplier.TestCase( + List.of( + TestCaseSupplier.TypedData.multiRow( + List.of( + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1"))), + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))), + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::"))), + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("ffff::"))) + ), + DataType.IP, + "field" + ) + ), + "Min[field=Attribute[channel=0]]", + DataType.IP, + equalTo(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::")))) + ) + ), // Folding new TestCaseSupplier( @@ -137,6 +161,21 @@ public static Iterable parameters() { DataType.BOOLEAN, equalTo(true) ) + ), + new TestCaseSupplier( + List.of(DataType.IP), + () -> new TestCaseSupplier.TestCase( + List.of( + TestCaseSupplier.TypedData.multiRow( + List.of(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1")))), + DataType.IP, + "field" + ) + ), + "Min[field=Attribute[channel=0]]", + DataType.IP, + equalTo(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1")))) + ) ) ) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PercentileTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PercentileTests.java new file mode 100644 index 0000000000000..5271431bd43b8 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PercentileTests.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.aggregate; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.search.aggregations.metrics.TDigestState; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase; +import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; +import java.util.stream.Stream; + +import static org.hamcrest.Matchers.equalTo; + +public class PercentileTests extends AbstractAggregationTestCase { + public PercentileTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + var suppliers = new ArrayList(); + + var fieldCases = Stream.of( + MultiRowTestCaseSupplier.intCases(1, 1000, Integer.MIN_VALUE, Integer.MAX_VALUE, true), + MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true), + MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true) + ).flatMap(List::stream).toList(); + + var percentileCases = Stream.of( + TestCaseSupplier.intCases(0, 100, true), + TestCaseSupplier.longCases(0, 100, true), + TestCaseSupplier.doubleCases(0, 100, true) + ).flatMap(List::stream).toList(); + + for (var field : fieldCases) { + for (var percentile : percentileCases) { + suppliers.add(makeSupplier(field, percentile)); + } + } + + return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers); + } + + @Override + protected Expression build(Source source, List args) { + return new Percentile(source, args.get(0), args.get(1)); + } + + private static TestCaseSupplier makeSupplier( + TestCaseSupplier.TypedDataSupplier fieldSupplier, + TestCaseSupplier.TypedDataSupplier percentileSupplier + ) { + return new TestCaseSupplier(List.of(fieldSupplier.type(), percentileSupplier.type()), () -> { + var fieldTypedData = fieldSupplier.get(); + var percentileTypedData = percentileSupplier.get().forceLiteral(); + + var percentile = ((Number) percentileTypedData.data()).intValue(); + + var digest = TDigestState.create(1000); + + for (var value : fieldTypedData.multiRowData()) { + digest.add(((Number) value).doubleValue()); + } + + var expected = digest.size() == 0 ? null : digest.quantile((double) percentile / 100); + + return new TestCaseSupplier.TestCase( + List.of(fieldTypedData, percentileTypedData), + "Percentile[number=Attribute[channel=0],percentile=Attribute[channel=1]]", + DataType.DOUBLE, + equalTo(expected) + ); + }); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SumTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SumTests.java new file mode 100644 index 0000000000000..4f14dafc8b30d --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SumTests.java @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.aggregate; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase; +import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; +import static org.hamcrest.Matchers.equalTo; + +public class SumTests extends AbstractAggregationTestCase { + public SumTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + var suppliers = new ArrayList(); + + Stream.of(MultiRowTestCaseSupplier.intCases(1, 1000, Integer.MIN_VALUE, Integer.MAX_VALUE, true) + // Longs currently throw on overflow. + // Restore after https://github.com/elastic/elasticsearch/issues/110437 + // MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true), + // Doubles currently return +/-Infinity on overflow. + // Restore after https://github.com/elastic/elasticsearch/issues/111026 + // MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true) + ).flatMap(List::stream).map(SumTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); + + suppliers.addAll( + List.of( + // Folding + new TestCaseSupplier( + List.of(DataType.INTEGER), + () -> new TestCaseSupplier.TestCase( + List.of(TestCaseSupplier.TypedData.multiRow(List.of(200), DataType.INTEGER, "field")), + "Sum[field=Attribute[channel=0]]", + DataType.LONG, + equalTo(200L) + ) + ), + new TestCaseSupplier( + List.of(DataType.LONG), + () -> new TestCaseSupplier.TestCase( + List.of(TestCaseSupplier.TypedData.multiRow(List.of(200L), DataType.LONG, "field")), + "Sum[field=Attribute[channel=0]]", + DataType.LONG, + equalTo(200L) + ) + ), + new TestCaseSupplier( + List.of(DataType.DOUBLE), + () -> new TestCaseSupplier.TestCase( + List.of(TestCaseSupplier.TypedData.multiRow(List.of(200.), DataType.DOUBLE, "field")), + "Sum[field=Attribute[channel=0]]", + DataType.DOUBLE, + equalTo(200.) + ) + ) + ) + ); + + return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers); + } + + @Override + protected Expression build(Source source, List args) { + return new Sum(source, args.get(0)); + } + + private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier) { + return new TestCaseSupplier(List.of(fieldSupplier.type()), () -> { + var fieldTypedData = fieldSupplier.get(); + + Object expected; + + try { + expected = switch (fieldTypedData.type().widenSmallNumeric()) { + case INTEGER -> fieldTypedData.multiRowData() + .stream() + .map(v -> (Integer) v) + .collect(Collectors.summarizingInt(Integer::intValue)) + .getSum(); + case LONG -> fieldTypedData.multiRowData() + .stream() + .map(v -> (Long) v) + .collect(Collectors.summarizingLong(Long::longValue)) + .getSum(); + case DOUBLE -> { + var value = fieldTypedData.multiRowData() + .stream() + .map(v -> (Double) v) + .collect(Collectors.summarizingDouble(Double::doubleValue)) + .getSum(); + + if (Double.isInfinite(value) || Double.isNaN(value)) { + yield null; + } + + yield value; + } + default -> throw new IllegalStateException("Unexpected value: " + fieldTypedData.type()); + }; + } catch (Exception e) { + expected = null; + } + + var dataType = fieldTypedData.type().isWholeNumber() == false || fieldTypedData.type() == UNSIGNED_LONG + ? DataType.DOUBLE + : DataType.LONG; + + return new TestCaseSupplier.TestCase(List.of(fieldTypedData), "Sum[field=Attribute[channel=0]]", dataType, equalTo(expected)); + }); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopTests.java index c0c23ce29301e..f64d6a200a031 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopTests.java @@ -10,7 +10,9 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -42,7 +44,9 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.intCases(1, 1000, Integer.MIN_VALUE, Integer.MAX_VALUE, true), MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true), MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true), - MultiRowTestCaseSupplier.dateCases(1, 1000) + MultiRowTestCaseSupplier.dateCases(1, 1000), + MultiRowTestCaseSupplier.booleanCases(1, 1000), + MultiRowTestCaseSupplier.ipCases(1, 1000) ) .flatMap(List::stream) .map(fieldCaseSupplier -> TopTests.makeSupplier(fieldCaseSupplier, limitCaseSupplier, order)) @@ -53,6 +57,19 @@ public static Iterable parameters() { suppliers.addAll( List.of( // Surrogates + new TestCaseSupplier( + List.of(DataType.BOOLEAN, DataType.INTEGER, DataType.KEYWORD), + () -> new TestCaseSupplier.TestCase( + List.of( + TestCaseSupplier.TypedData.multiRow(List.of(true, true, false), DataType.BOOLEAN, "field"), + new TestCaseSupplier.TypedData(1, DataType.INTEGER, "limit").forceLiteral(), + new TestCaseSupplier.TypedData(new BytesRef("desc"), DataType.KEYWORD, "order").forceLiteral() + ), + "Top[field=Attribute[channel=0], limit=Attribute[channel=1], order=Attribute[channel=2]]", + DataType.BOOLEAN, + equalTo(true) + ) + ), new TestCaseSupplier( List.of(DataType.INTEGER, DataType.INTEGER, DataType.KEYWORD), () -> new TestCaseSupplier.TestCase( @@ -105,8 +122,43 @@ public static Iterable parameters() { equalTo(200L) ) ), + new TestCaseSupplier( + List.of(DataType.IP), + () -> new TestCaseSupplier.TestCase( + List.of( + TestCaseSupplier.TypedData.multiRow( + List.of( + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1"))), + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))), + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::"))), + new BytesRef(InetAddressPoint.encode(InetAddresses.forString("ffff::"))) + ), + DataType.IP, + "field" + ), + new TestCaseSupplier.TypedData(1, DataType.INTEGER, "limit").forceLiteral(), + new TestCaseSupplier.TypedData(new BytesRef("desc"), DataType.KEYWORD, "order").forceLiteral() + ), + "Top[field=Attribute[channel=0], limit=Attribute[channel=1], order=Attribute[channel=2]]", + DataType.IP, + equalTo(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("ffff::")))) + ) + ), // Folding + new TestCaseSupplier( + List.of(DataType.BOOLEAN, DataType.INTEGER, DataType.KEYWORD), + () -> new TestCaseSupplier.TestCase( + List.of( + TestCaseSupplier.TypedData.multiRow(List.of(true), DataType.BOOLEAN, "field"), + new TestCaseSupplier.TypedData(1, DataType.INTEGER, "limit").forceLiteral(), + new TestCaseSupplier.TypedData(new BytesRef("desc"), DataType.KEYWORD, "order").forceLiteral() + ), + "Top[field=Attribute[channel=0], limit=Attribute[channel=1], order=Attribute[channel=2]]", + DataType.BOOLEAN, + equalTo(true) + ) + ), new TestCaseSupplier( List.of(DataType.INTEGER, DataType.INTEGER, DataType.KEYWORD), () -> new TestCaseSupplier.TestCase( @@ -159,6 +211,23 @@ public static Iterable parameters() { equalTo(200L) ) ), + new TestCaseSupplier( + List.of(DataType.IP), + () -> new TestCaseSupplier.TestCase( + List.of( + TestCaseSupplier.TypedData.multiRow( + List.of(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1")))), + DataType.IP, + "field" + ), + new TestCaseSupplier.TypedData(1, DataType.INTEGER, "limit").forceLiteral(), + new TestCaseSupplier.TypedData(new BytesRef("desc"), DataType.KEYWORD, "order").forceLiteral() + ), + "Top[field=Attribute[channel=0], limit=Attribute[channel=1], order=Attribute[channel=2]]", + DataType.IP, + equalTo(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1")))) + ) + ), // Resolution errors new TestCaseSupplier( @@ -222,7 +291,7 @@ private static TestCaseSupplier makeSupplier( TestCaseSupplier.TypedDataSupplier limitCaseSupplier, String order ) { - return new TestCaseSupplier(List.of(fieldSupplier.type(), DataType.INTEGER, DataType.KEYWORD), () -> { + return new TestCaseSupplier(fieldSupplier.name(), List.of(fieldSupplier.type(), DataType.INTEGER, DataType.KEYWORD), () -> { var fieldTypedData = fieldSupplier.get(); var limitTypedData = limitCaseSupplier.get().forceLiteral(); var limit = (int) limitTypedData.getValue(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java index e08da9850b555..f472e5ef5efd9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java @@ -55,7 +55,7 @@ public static Iterable parameters() { ); })); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java index 88ca7d0452b3e..b5ea1827926dd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java @@ -55,7 +55,7 @@ public static Iterable parameters() { ); })); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java index c5b9b2501aeae..73837c23ff640 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java @@ -80,7 +80,7 @@ public static Iterable parameters() { emptyList() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "boolean or numeric or string"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java index a59e7b0085e4c..9548b0476f3c2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java @@ -72,7 +72,7 @@ public static Iterable parameters() { ); } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "cartesian_point or string"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java index 973431d676b82..5784c3559b10e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java @@ -73,7 +73,7 @@ public static Iterable parameters() { ); } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "cartesian_point or cartesian_shape or string"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java index e512334391bed..7025c7df4ba39 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java @@ -162,7 +162,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "datetime or numeric or string"); } private static String randomDateString(long from, long to) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java index bd07141009d3e..4d52a470edb87 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java @@ -89,7 +89,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java index d4d20629da09e..d5153019c1e41 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java @@ -139,7 +139,11 @@ public static Iterable parameters() { List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + suppliers, + (v, p) -> "boolean or counter_double or counter_integer or counter_long or datetime or numeric or string" + ); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java index 7a3b83f3ab113..445c4b8f15d10 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java @@ -66,7 +66,7 @@ public static Iterable parameters() { ); } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "geo_point or string"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java index 831539852846c..38d871b17e8ba 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java @@ -66,7 +66,7 @@ public static Iterable parameters() { List.of() ); } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "geo_point or geo_shape or string"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java index ffa94548f0a23..66358b7ba7440 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java @@ -63,7 +63,7 @@ public static Iterable parameters() { ); // add null as parameter - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "ip or string"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java index 7984c1e04effc..eb81d48e0c5be 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java @@ -268,7 +268,11 @@ public static Iterable parameters() { List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + suppliers, + (v, p) -> "boolean or counter_integer or datetime or numeric or string" + ); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java index 27c69ae977f6b..6e931c802030f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java @@ -227,7 +227,11 @@ public static Iterable parameters() { l -> ((Integer) l).longValue(), List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + suppliers, + (v, p) -> "boolean or counter_integer or counter_long or datetime or numeric or string" + ); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java index 33e8eee7a8de4..6e99e720ad5ea 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java @@ -70,7 +70,7 @@ public static Iterable parameters() { List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java index 809b4ddaa78a4..e405fddae60dc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java @@ -130,7 +130,7 @@ public static Iterable parameters() { v -> new BytesRef(v.toString()), List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> ""); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java index a1fccac8edfd1..d8122aa73f81a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java @@ -244,7 +244,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "boolean or datetime or numeric or string"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java index 1c37afc1c0722..46a8086f9479c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java @@ -60,7 +60,7 @@ public static Iterable parameters() { ); } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string or version"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java index bce3b7efebbb6..5b835cd5e55b9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java @@ -84,7 +84,12 @@ public static Iterable parameters() { ) .withFoldingException(InvalidArgumentException.class, "invalid date field for []: not a unit") ) - ) + ), + (v, p) -> switch (p) { + case 0 -> "string"; + case 1 -> "datetime"; + default -> ""; + } ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java index b18748187709a..002e536bd19a7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java @@ -58,7 +58,12 @@ public static Iterable parameters() { equalTo(BytesRefs.toBytesRef("2023")) ) ) - ) + ), + (v, p) -> switch (p) { + case 0 -> "string"; + case 1 -> "datetime"; + default -> ""; + } ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java index f0aa766fb1bf9..8da01fc1989ba 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java @@ -127,7 +127,8 @@ public static Iterable parameters() { + "failed to parse date field [not a date] with format [yyyy-MM-dd]" ) ) - ) + ), + (v, p) -> "string" ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java index 17d8cd6a57223..48b23ed5c8840 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java @@ -54,7 +54,11 @@ public static Iterable parameters() { ofDuration(Duration.ofSeconds(30), ts, "2023-02-17T10:25:30.00Z"), randomSecond() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { + case 0 -> "dateperiod or timeduration"; + case 1 -> "datetime"; + default -> null; + }); } public void testCreateRoundingDuration() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java index 3cdc54f240a96..e777a0ce587e0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java @@ -84,7 +84,11 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { + case 0 -> "ip"; + case 1 -> "string"; + default -> ""; + }); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java index 298bcb3f49466..5209d042b6408 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java @@ -106,7 +106,11 @@ public static Iterable parameters() { }) ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { + case 0 -> "ip"; + case 1, 2 -> "integer"; + default -> ""; + }); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsTests.java index b5923c7a5b214..493e9e0e9d900 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsTests.java @@ -63,7 +63,7 @@ public static Iterable parameters() { equalTo(Math.abs(arg)) ); })); - return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "numeric"); } public AbsTests(@Name("TestCase") Supplier testCaseSupplier) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java index 7c5cd87dfee39..278c9123e30b1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java @@ -56,7 +56,7 @@ public static Iterable parameters() { ) ) ); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers)); + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java index 38e210d81e5fd..04fec5a20b438 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java @@ -56,7 +56,7 @@ public static Iterable parameters() { ) ) ); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers)); + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java index 1144919094812..c475a75699da7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java @@ -36,7 +36,7 @@ public static Iterable parameters() { Double.POSITIVE_INFINITY, List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanTests.java index c9f7a1baeadbe..b51154515de82 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanTests.java @@ -33,7 +33,7 @@ public static Iterable parameters() { Double.POSITIVE_INFINITY, List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java index f644d8bc72dce..bfe35a08b8ba1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java @@ -72,7 +72,7 @@ public static Iterable parameters() { ); suppliers = anyNullIsNull(true, suppliers); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers)); + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilTests.java index 1572b928a0d75..ddc099a2ad0b1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilTests.java @@ -66,7 +66,7 @@ public static Iterable parameters() { UNSIGNED_LONG_MAX, List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosTests.java index dc5eec4f90d32..47dc99f2c13f9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosTests.java @@ -33,7 +33,7 @@ public static Iterable parameters() { Double.POSITIVE_INFINITY, List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshTests.java index 79557b15be08a..ad4208420f481 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshTests.java @@ -61,7 +61,7 @@ public static Iterable parameters() { ) ) ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpSerializationTests.java new file mode 100644 index 0000000000000..be7478340d9ce --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpSerializationTests.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.math; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractUnaryScalarSerializationTests; + +public class ExpSerializationTests extends AbstractUnaryScalarSerializationTests { + @Override + protected Exp create(Source source, Expression child) { + return new Exp(source, child); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java new file mode 100644 index 0000000000000..d42f4ffde0609 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.math; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.NumericUtils; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +public class ExpTests extends AbstractScalarFunctionTestCase { + public ExpTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + // e^710 is Double.POSITIVE_INFINITY + private static final int MAX_EXP_VALUE = 709; + + @ParametersFactory + public static Iterable parameters() { + String read = "Attribute[channel=0]"; + List suppliers = new ArrayList<>(); + TestCaseSupplier.forUnaryInt( + suppliers, + "ExpIntEvaluator[val=" + read + "]", + DataType.DOUBLE, + i -> Math.exp(i), + Integer.MIN_VALUE, + MAX_EXP_VALUE, + List.of() + ); + + TestCaseSupplier.forUnaryLong( + suppliers, + "ExpLongEvaluator[val=" + read + "]", + DataType.DOUBLE, + l -> Math.exp(l), + Long.MIN_VALUE, + MAX_EXP_VALUE, + List.of() + ); + + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + "ExpUnsignedLongEvaluator[val=" + read + "]", + DataType.DOUBLE, + ul -> Math.exp(NumericUtils.unsignedLongToDouble(NumericUtils.asLongUnsigned(ul))), + BigInteger.ZERO, + BigInteger.valueOf(MAX_EXP_VALUE), + List.of() + ); + TestCaseSupplier.forUnaryDouble( + suppliers, + "ExpDoubleEvaluator[val=" + read + "]", + DataType.DOUBLE, + Math::exp, + -Double.MAX_VALUE, + MAX_EXP_VALUE, + List.of() + ); + + suppliers = anyNullIsNull(true, suppliers); + + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + } + + @Override + protected Expression build(Source source, List args) { + return new Exp(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorTests.java index 269dabcc6b6b8..1d35e034de908 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorTests.java @@ -50,7 +50,7 @@ public static Iterable parameters() { Double.POSITIVE_INFINITY, List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java index ca0c8718f5ac0..44ad4547481d6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java @@ -124,7 +124,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers)); + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java index 1c002e111e575..671cffe9e7f9e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java @@ -191,7 +191,7 @@ public static Iterable parameters() { suppliers = anyNullIsNull(true, suppliers); // Negative cases - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers)); + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java index bea0f399233fd..9d8b87bab8878 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java @@ -77,7 +77,7 @@ public static Iterable parameters() { ) ) ); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers)); + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java index 21b44134458b7..8c612e5e664e0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java @@ -72,7 +72,7 @@ public static Iterable parameters() { suppliers = anyNullIsNull(true, suppliers); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers)); + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java index 7a1190d86c2bf..990356f8df6de 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java @@ -33,7 +33,7 @@ public static Iterable parameters() { Double.POSITIVE_INFINITY, List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhTests.java index b83519c6d1299..d24dcd1f18f8f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhTests.java @@ -61,7 +61,7 @@ public static Iterable parameters() { ) ) ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java index 9c81bbdc3cd49..23f2adc6c02e0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java @@ -109,7 +109,7 @@ public static Iterable parameters() { "Line -1:-1: java.lang.ArithmeticException: Square root of negative" ) ); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers)); + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanTests.java index 369c33a1291f1..995894fec5259 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanTests.java @@ -33,7 +33,7 @@ public static Iterable parameters() { Double.POSITIVE_INFINITY, List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhTests.java index 14fdcdca2fe96..73a86fd5a114c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhTests.java @@ -33,7 +33,7 @@ public static Iterable parameters() { Double.POSITIVE_INFINITY, List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java index 212b66027d455..0adbe9164baee 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java @@ -20,7 +20,6 @@ import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.hamcrest.Matcher; import java.math.BigInteger; @@ -620,7 +619,7 @@ private static > void putInOrder(List mvData, Block.M protected final DataType[] representableNumerics() { // TODO numeric should only include representable numbers but that is a change for a followup - return DataType.types().stream().filter(DataType::isNumeric).filter(EsqlDataTypes::isRepresentable).toArray(DataType[]::new); + return DataType.types().stream().filter(DataType::isNumeric).filter(DataType::isRepresentable).toArray(DataType[]::new); } protected DataType expectedType(List argTypes) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgTests.java index 43c683040eac4..04baf82a461fe 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgTests.java @@ -55,7 +55,7 @@ public static Iterable parameters() { */ (size, data) -> avg.apply(size, data.mapToDouble(v -> unsignedLongToDouble(NumericUtils.asLongUnsigned(v)))) ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatTests.java index 0277093152cba..4467b49cd674a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.util.ArrayList; import java.util.List; @@ -33,11 +32,11 @@ public MvConcatTests(@Name("TestCase") Supplier testC public static Iterable parameters() { List suppliers = new ArrayList<>(); for (DataType fieldType : DataType.types()) { - if (EsqlDataTypes.isString(fieldType) == false) { + if (DataType.isString(fieldType) == false) { continue; } for (DataType delimType : DataType.types()) { - if (EsqlDataTypes.isString(delimType) == false) { + if (DataType.isString(delimType) == false) { continue; } for (int l = 1; l < 10; l++) { @@ -68,7 +67,7 @@ public static Iterable parameters() { } } } - return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountTests.java index 8c8772f8ed4e2..05b646610105c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountTests.java @@ -40,7 +40,7 @@ public static Iterable parameters() { cartesianPoints(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count()))); geoShape(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count()))); cartesianShape(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count()))); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> ""); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstTests.java index 6e143d9175f41..df122056b85d3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstTests.java @@ -41,7 +41,7 @@ public static Iterable parameters() { cartesianPoints(cases, "mv_first", "MvFirst", DataType.CARTESIAN_POINT, (size, values) -> equalTo(values.findFirst().get())); geoShape(cases, "mv_first", "MvFirst", DataType.GEO_SHAPE, (size, values) -> equalTo(values.findFirst().get())); cartesianShape(cases, "mv_first", "MvFirst", DataType.CARTESIAN_SHAPE, (size, values) -> equalTo(values.findFirst().get())); - return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases); + return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases, (v, p) -> ""); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastTests.java index 83d94f2cc9884..e9d73d6458055 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastTests.java @@ -41,7 +41,7 @@ public static Iterable parameters() { cartesianPoints(cases, "mv_last", "MvLast", DataType.CARTESIAN_POINT, (size, values) -> equalTo(values.reduce((f, s) -> s).get())); geoShape(cases, "mv_last", "MvLast", DataType.GEO_SHAPE, (size, values) -> equalTo(values.reduce((f, s) -> s).get())); cartesianShape(cases, "mv_last", "MvLast", DataType.CARTESIAN_SHAPE, (size, values) -> equalTo(values.reduce((f, s) -> s).get())); - return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases); + return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxTests.java index 63530234e53fa..db96bf328c92d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxTests.java @@ -38,7 +38,7 @@ public static Iterable parameters() { longs(cases, "mv_max", "MvMax", (size, values) -> equalTo(values.max().getAsLong())); unsignedLongs(cases, "mv_max", "MvMax", (size, values) -> equalTo(values.reduce(BigInteger::max).get())); dateTimes(cases, "mv_max", "MvMax", (size, values) -> equalTo(values.max().getAsLong())); - return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases); + return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases, (v, p) -> "representableNonSpatial"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianTests.java index f44f5d44e3f62..58d596fe29452 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianTests.java @@ -92,7 +92,7 @@ public static Iterable parameters() { ) ) ); - return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases); + return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinTests.java index 5be67548f784e..b966600b5e265 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinTests.java @@ -38,7 +38,7 @@ public static Iterable parameters() { longs(cases, "mv_min", "MvMin", (size, values) -> equalTo(values.min().getAsLong())); unsignedLongs(cases, "mv_min", "MvMin", (size, values) -> equalTo(values.reduce(BigInteger::min).get())); dateTimes(cases, "mv_min", "MvMin", (size, values) -> equalTo(values.min().getAsLong())); - return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases); + return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases, (v, p) -> "representableNonSpatial"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipTests.java index e9f0fd5b51516..86366e7660b90 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipTests.java @@ -52,7 +52,7 @@ public static Iterable parameters() { } } - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers)); + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "string")); } private static TestCaseSupplier supplier(DataType leftType, DataType rightType, DataType delimType) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java index b99b47b6f505a..d37c32c76b450 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java @@ -18,7 +18,6 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.hamcrest.Matcher; import java.util.ArrayList; @@ -36,7 +35,7 @@ public IsNotNullTests(@Name("TestCase") Supplier test public static Iterable parameters() { List suppliers = new ArrayList<>(); for (DataType type : DataType.types()) { - if (false == EsqlDataTypes.isRepresentable(type)) { + if (false == DataType.isRepresentable(type)) { continue; } if (type != DataType.NULL) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java index 7abfad39967a5..300f619dafa25 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java @@ -18,7 +18,6 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.hamcrest.Matcher; import java.util.ArrayList; @@ -36,7 +35,7 @@ public IsNullTests(@Name("TestCase") Supplier testCas public static Iterable parameters() { List suppliers = new ArrayList<>(); for (DataType type : DataType.types()) { - if (false == EsqlDataTypes.isRepresentable(type)) { + if (false == DataType.isRepresentable(type)) { continue; } if (type != DataType.NULL) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java index a30cce9f765ed..0729d4854f65f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java @@ -27,10 +27,10 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.elasticsearch.xpack.esql.core.type.DataType.isSpatial; +import static org.elasticsearch.xpack.esql.core.type.DataType.isSpatialGeo; +import static org.elasticsearch.xpack.esql.core.type.DataType.isString; import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction.compatibleTypeNames; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isSpatial; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isSpatialGeo; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isString; import static org.hamcrest.Matchers.equalTo; public abstract class BinarySpatialFunctionTestCase extends AbstractScalarFunctionTestCase { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXTests.java index 71e73398ddcd4..96cddfdd64099 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXTests.java @@ -36,7 +36,7 @@ public static Iterable parameters() { final List suppliers = new ArrayList<>(); TestCaseSupplier.forUnaryGeoPoint(suppliers, expectedEvaluator, DOUBLE, StXTests::valueOf, List.of()); TestCaseSupplier.forUnaryCartesianPoint(suppliers, expectedEvaluator, DOUBLE, StXTests::valueOf, List.of()); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "geo_point or cartesian_point"); } private static double valueOf(BytesRef wkb) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYTests.java index a30ae924754d6..165dbb2c0ab77 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYTests.java @@ -36,7 +36,7 @@ public static Iterable parameters() { final List suppliers = new ArrayList<>(); TestCaseSupplier.forUnaryGeoPoint(suppliers, expectedEvaluator, DOUBLE, StYTests::valueOf, List.of()); TestCaseSupplier.forUnaryCartesianPoint(suppliers, expectedEvaluator, DOUBLE, StYTests::valueOf, List.of()); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "geo_point or cartesian_point"); } private static double valueOf(BytesRef wkb) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java index a92f3ffb49533..f77a892d8682e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java @@ -67,7 +67,7 @@ static Iterable parameters(String name, boolean trimLeading, boolean t })); } } - return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string"); } private static TestCaseSupplier.TestCase testCase(String name, DataType type, String data, String expected) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java index c398faacb90d0..bbe92ae4a6618 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java @@ -20,7 +20,6 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.util.ArrayList; import java.util.HashMap; @@ -52,11 +51,11 @@ public static Iterable parameters() { Set supported = Set.of(DataType.NULL, DataType.KEYWORD, DataType.TEXT); List> supportedPerPosition = List.of(supported, supported); for (DataType lhs : DataType.types()) { - if (lhs == DataType.NULL || EsqlDataTypes.isRepresentable(lhs) == false) { + if (lhs == DataType.NULL || DataType.isRepresentable(lhs) == false) { continue; } for (DataType rhs : DataType.types()) { - if (rhs == DataType.NULL || EsqlDataTypes.isRepresentable(rhs) == false) { + if (rhs == DataType.NULL || DataType.isRepresentable(rhs) == false) { continue; } boolean lhsIsString = lhs == DataType.KEYWORD || lhs == DataType.TEXT; @@ -65,7 +64,7 @@ public static Iterable parameters() { continue; } - suppliers.add(typeErrorSupplier(false, supportedPerPosition, List.of(lhs, rhs))); + suppliers.add(typeErrorSupplier(false, supportedPerPosition, List.of(lhs, rhs), (v, p) -> "string")); } } return parameterSuppliersFromTypedData(suppliers); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java index 88ee7881e128a..627c46da025ea 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java @@ -167,7 +167,11 @@ public static Iterable parameters() { ); })); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { + case 0 -> "string"; + case 1 -> "integer"; + default -> ""; + }); } private static String unicodeLeftSubstring(String str, int length) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java index a1451b6bedf7a..6ae5a9d961398 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java @@ -49,7 +49,7 @@ public static Iterable parameters() { cases.addAll(makeTestCases("6 bytes, 2 code points", () -> "❗️", 2)); cases.addAll(makeTestCases("100 random alpha", () -> randomAlphaOfLength(100), 100)); cases.addAll(makeTestCases("100 random code points", () -> randomUnicodeOfCodepointLength(100), 100)); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "string"); } private static List makeTestCases(String title, Supplier text, int expectedLength) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java index 13d8edf489a66..207125bed2a19 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java @@ -78,7 +78,12 @@ public static Iterable parameters() { } } - suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers)); + suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), (v, p) -> { + if (p == 0 || p == 1) { + return "string"; + } + return "integer"; + }); // Here follows some non-randomized examples that we want to cover on every run suppliers.add(supplier("a tiger", "a t", null, 1)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java index 0074f83b3bbce..683fc2f4cbfb3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.util.ArrayList; import java.util.List; @@ -73,7 +72,7 @@ static Iterable parameters(Function escapeString, Supp if (type == DataType.KEYWORD || type == DataType.TEXT || type == DataType.NULL) { continue; } - if (EsqlDataTypes.isRepresentable(type) == false) { + if (DataType.isRepresentable(type) == false) { continue; } cases.add( @@ -140,6 +139,18 @@ private static void cases(List cases, String title, Supplier { + TextAndPattern v = textAndPattern.get(); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(v.text), type, "e"), + new TestCaseSupplier.TypedData(new BytesRef(v.pattern), type, "pattern").forceLiteral() + ), + startsWith("AutomataMatchEvaluator[input=Attribute[channel=0], pattern=digraph Automaton {\n"), + DataType.BOOLEAN, + equalTo(expected) + ); + })); } } @@ -152,10 +163,13 @@ protected void assertSimpleWithNulls(List data, Block value, int nullBlo protected Expression build(Source source, List args) { Expression expression = args.get(0); Literal pattern = (Literal) args.get(1); - Literal caseInsensitive = (Literal) args.get(2); + Literal caseInsensitive = args.size() > 2 ? (Literal) args.get(2) : null; String patternString = ((BytesRef) pattern.fold()).utf8ToString(); - boolean caseInsensitiveBool = (boolean) caseInsensitive.fold(); + boolean caseInsensitiveBool = caseInsensitive != null ? (boolean) caseInsensitive.fold() : false; logger.info("pattern={} caseInsensitive={}", patternString, caseInsensitiveBool); - return new RLike(source, expression, new RLikePattern(patternString), caseInsensitiveBool); + + return caseInsensitiveBool + ? new RLike(source, expression, new RLikePattern(patternString), true) + : new RLike(source, expression, new RLikePattern(patternString)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java index 8d0368d1c618f..4d97a2f629c23 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java @@ -107,7 +107,11 @@ public static Iterable parameters() { })); cases = anyNullIsNull(true, cases); - cases = errorsForCasesWithoutExamples(cases); + cases = errorsForCasesWithoutExamples(cases, (v, p) -> switch (p) { + case 0 -> "string"; + case 1 -> "integer"; + default -> ""; + }); return parameterSuppliersFromTypedData(cases); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java index fe77b9dcdb075..bf1325854f1a2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java @@ -103,7 +103,7 @@ public static Iterable parameters() { "Unclosed character class near index 0\n[\n^".replaceAll("\n", System.lineSeparator()) ); })); - return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string"); } private static TestCaseSupplier fixedCase(String name, String str, String oldStr, String newStr, String result) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java index cc98edb85f547..a1ef77a62b67c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java @@ -166,7 +166,11 @@ public static Iterable parameters() { equalTo(new BytesRef(unicodeRightSubstring(text, length))) ); })); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { + case 0 -> "string"; + case 1 -> "integer"; + default -> throw new IllegalStateException("bad parameter number"); + }); } private static String unicodeRightSubstring(String str, int length) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java index dd28b43bd66ed..b5560f37914a9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java @@ -64,7 +64,7 @@ public static Iterable parameters() { })); } } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java index 1c49d3b408ad6..6b934aae775df 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java @@ -86,7 +86,12 @@ public static Iterable parameters() { equalTo(new BytesRef("")) ); }) - ) + ), + (v, p) -> switch (p) { + case 0 -> "string"; + case 1, 2 -> "integer"; + default -> ""; + } ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java index abb419e1e4a81..ffde67d44090d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java @@ -47,7 +47,7 @@ public static Iterable parameters() { suppliers.add(supplier("text unicode", DataType.TEXT, () -> randomUnicodeOfLengthBetween(1, 10))); // add null as parameter - return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string"); } public void testRandomLocale() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java index f101cacd73dc5..19170a542ca3a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java @@ -47,7 +47,7 @@ public static Iterable parameters() { suppliers.add(supplier("text unicode", DataType.TEXT, () -> randomUnicodeOfLengthBetween(1, 10))); // add null as parameter - return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string"); } public void testRandomLocale() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java index 974c8703b2a09..69f01af0c03b1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java @@ -26,8 +26,8 @@ import static org.elasticsearch.compute.data.BlockUtils.toJavaObject; import static org.elasticsearch.xpack.esql.core.type.DataType.isNull; +import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable; import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.commonType; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isRepresentable; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractArithmeticTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractArithmeticTestCase.java index 141fc24e73e18..05e823c1649cd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractArithmeticTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractArithmeticTestCase.java @@ -11,7 +11,6 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.esql.expression.predicate.operator.AbstractBinaryOperatorTestCase; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.hamcrest.Matcher; import java.util.List; @@ -71,7 +70,7 @@ protected Matcher resultsMatcher(List typedD @Override protected boolean supportsType(DataType type) { - return type.isNumeric() && EsqlDataTypes.isRepresentable(type); + return type.isNumeric() && DataType.isRepresentable(type); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractDateTimeArithmeticTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractDateTimeArithmeticTestCase.java index 8a27a289bb77f..0c29eb5b8cae0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractDateTimeArithmeticTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractDateTimeArithmeticTestCase.java @@ -9,7 +9,6 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.hamcrest.Matcher; import java.time.Duration; @@ -20,8 +19,8 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTime; import static org.elasticsearch.xpack.esql.core.type.DataType.isNull; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isNullOrTemporalAmount; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isTemporalAmount; +import static org.elasticsearch.xpack.esql.core.type.DataType.isNullOrTemporalAmount; +import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount; import static org.hamcrest.Matchers.equalTo; public abstract class AbstractDateTimeArithmeticTestCase extends AbstractArithmeticTestCase { @@ -57,7 +56,7 @@ protected Matcher resultMatcher(List data, DataType dataType) { @Override protected final boolean supportsType(DataType type) { - return EsqlDataTypes.isDateTimeOrTemporal(type) || super.supportsType(type); + return DataType.isDateTimeOrTemporal(type) || super.supportsType(type); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java index c8a2511e34211..609effbbc31b7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java @@ -73,6 +73,38 @@ public static Iterable parameters() { ) ); + // Double overflows + suppliers.addAll( + List.of( + new TestCaseSupplier( + List.of(DataType.DOUBLE, DataType.DOUBLE), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(Double.MAX_VALUE, DataType.DOUBLE, "lhs"), + new TestCaseSupplier.TypedData(Double.MAX_VALUE, DataType.DOUBLE, "rhs") + ), + "AddDoublesEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", + DataType.DOUBLE, + equalTo(null) + ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") + .withWarning("Line -1:-1: java.lang.ArithmeticException: not a finite double number: Infinity") + ), + new TestCaseSupplier( + List.of(DataType.DOUBLE, DataType.DOUBLE), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(-Double.MAX_VALUE, DataType.DOUBLE, "lhs"), + new TestCaseSupplier.TypedData(-Double.MAX_VALUE, DataType.DOUBLE, "rhs") + ), + "AddDoublesEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", + DataType.DOUBLE, + equalTo(null) + ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") + .withWarning("Line -1:-1: java.lang.ArithmeticException: not a finite double number: -Infinity") + ) + ) + ); + // Unsigned Long cases // TODO: These should be integrated into the type cross product above, but are currently broken // see https://github.com/elastic/elasticsearch/issues/102935 @@ -253,7 +285,7 @@ public static Iterable parameters() { private static String addErrorMessageString(boolean includeOrdinal, List> validPerPosition, List types) { try { - return typeErrorMessage(includeOrdinal, validPerPosition, types); + return typeErrorMessage(includeOrdinal, validPerPosition, types, (a, b) -> "datetime or numeric"); } catch (IllegalStateException e) { // This means all the positional args were okay, so the expected error is from the combination return "[+] has arguments with incompatible types [" + types.get(0).typeName() + "] and [" + types.get(1).typeName() + "]"; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java index 7bc5b24651218..c6b607ded7999 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java @@ -168,7 +168,7 @@ public static Iterable parameters() { private static String divErrorMessageString(boolean includeOrdinal, List> validPerPosition, List types) { try { - return typeErrorMessage(includeOrdinal, validPerPosition, types); + return typeErrorMessage(includeOrdinal, validPerPosition, types, (a, b) -> "numeric"); } catch (IllegalStateException e) { // This means all the positional args were okay, so the expected error is from the combination return "[/] has arguments with incompatible types [" + types.get(0).typeName() + "] and [" + types.get(1).typeName() + "]"; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java index 133324bafd134..65335cdeeb4e1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java @@ -155,7 +155,7 @@ public static Iterable parameters() { private static String modErrorMessageString(boolean includeOrdinal, List> validPerPosition, List types) { try { - return typeErrorMessage(includeOrdinal, validPerPosition, types); + return typeErrorMessage(includeOrdinal, validPerPosition, types, (a, b) -> "numeric"); } catch (IllegalStateException e) { // This means all the positional args were okay, so the expected error is from the combination return "[%] has arguments with incompatible types [" + types.get(0).typeName() + "] and [" + types.get(1).typeName() + "]"; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java index 7472636611063..54786d11918b5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java @@ -56,7 +56,8 @@ public static Iterable parameters() { ) ); - suppliers.add(new TestCaseSupplier("Double * Double", List.of(DataType.DOUBLE, DataType.DOUBLE), () -> { + // Double + suppliers.addAll(List.of(new TestCaseSupplier("Double * Double", List.of(DataType.DOUBLE, DataType.DOUBLE), () -> { double rhs = randomDouble(); double lhs = randomDouble(); return new TestCaseSupplier.TestCase( @@ -68,7 +69,37 @@ public static Iterable parameters() { DataType.DOUBLE, equalTo(lhs * rhs) ); - })); + }), + + // Overflows + new TestCaseSupplier( + List.of(DataType.DOUBLE, DataType.DOUBLE), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(Double.MAX_VALUE, DataType.DOUBLE, "lhs"), + new TestCaseSupplier.TypedData(Double.MAX_VALUE, DataType.DOUBLE, "rhs") + ), + "MulDoublesEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", + DataType.DOUBLE, + equalTo(null) + ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") + .withWarning("Line -1:-1: java.lang.ArithmeticException: not a finite double number: Infinity") + ), + new TestCaseSupplier( + List.of(DataType.DOUBLE, DataType.DOUBLE), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(-Double.MAX_VALUE, DataType.DOUBLE, "lhs"), + new TestCaseSupplier.TypedData(Double.MAX_VALUE, DataType.DOUBLE, "rhs") + ), + "MulDoublesEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", + DataType.DOUBLE, + equalTo(null) + ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") + .withWarning("Line -1:-1: java.lang.ArithmeticException: not a finite double number: -Infinity") + ) + )); + suppliers.add( arithmeticExceptionOverflowCase( DataType.INTEGER, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java index 7eadd74eaeb9e..2adefff2823ed 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java @@ -110,7 +110,7 @@ public static Iterable parameters() { equalTo(arg.negated()) ); }))); - return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "numeric, date_period or time_duration"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java index 9dc024ac1e8ff..e57b523542d37 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java @@ -79,6 +79,38 @@ public static Iterable parameters() { ); }) */ + // Double overflows + suppliers.addAll( + List.of( + new TestCaseSupplier( + List.of(DataType.DOUBLE, DataType.DOUBLE), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(Double.MAX_VALUE, DataType.DOUBLE, "lhs"), + new TestCaseSupplier.TypedData(-Double.MAX_VALUE, DataType.DOUBLE, "rhs") + ), + "SubDoublesEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", + DataType.DOUBLE, + equalTo(null) + ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") + .withWarning("Line -1:-1: java.lang.ArithmeticException: not a finite double number: Infinity") + ), + new TestCaseSupplier( + List.of(DataType.DOUBLE, DataType.DOUBLE), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(-Double.MAX_VALUE, DataType.DOUBLE, "lhs"), + new TestCaseSupplier.TypedData(Double.MAX_VALUE, DataType.DOUBLE, "rhs") + ), + "SubDoublesEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", + DataType.DOUBLE, + equalTo(null) + ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") + .withWarning("Line -1:-1: java.lang.ArithmeticException: not a finite double number: -Infinity") + ) + ) + ); + suppliers.add(new TestCaseSupplier("Datetime - Period", () -> { long lhs = (Long) randomLiteral(DataType.DATETIME).value(); Period rhs = (Period) randomLiteral(DataType.DATE_PERIOD).value(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java index d3539f4a56fe9..f3eddc8aed08a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java @@ -198,7 +198,7 @@ public static Iterable parameters() { return parameterSuppliersFromTypedData( errorsForCasesWithoutExamples( anyNullIsNull(true, suppliers), - AbstractScalarFunctionTestCase::errorMessageStringForBinaryOperators + (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators(o, v, t, (l, p) -> "") ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java index b2174f7be1593..5435a7f629d43 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java @@ -133,7 +133,12 @@ public static Iterable parameters() { return parameterSuppliersFromTypedData( errorsForCasesWithoutExamples( anyNullIsNull(true, suppliers), - AbstractScalarFunctionTestCase::errorMessageStringForBinaryOperators + (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators( + o, + v, + t, + (l, p) -> "datetime, double, integer, ip, keyword, long, text, unsigned_long or version" + ) ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java index edb276e16dd99..75c22c34623b9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java @@ -133,7 +133,12 @@ public static Iterable parameters() { return parameterSuppliersFromTypedData( errorsForCasesWithoutExamples( anyNullIsNull(true, suppliers), - AbstractScalarFunctionTestCase::errorMessageStringForBinaryOperators + (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators( + o, + v, + t, + (l, p) -> "datetime, double, integer, ip, keyword, long, text, unsigned_long or version" + ) ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java index 224b1fdba3f2e..620f9b7675506 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java @@ -6,16 +6,40 @@ */ package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; -import org.elasticsearch.test.ESTestCase; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geo.ShapeTestUtils; +import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; +import java.util.function.Supplier; import static org.elasticsearch.xpack.esql.EsqlTestUtils.of; import static org.elasticsearch.xpack.esql.core.expression.Literal.NULL; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; +import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT; +import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.matchesPattern; -public class InTests extends ESTestCase { +public class InTests extends AbstractFunctionTestCase { + public InTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } private static final Literal ONE = L(1); private static final Literal TWO = L(2); @@ -23,12 +47,12 @@ public class InTests extends ESTestCase { public void testInWithContainedValue() { In in = new In(EMPTY, TWO, Arrays.asList(ONE, TWO, THREE)); - assertTrue(in.fold()); + assertTrue((Boolean) in.fold()); } public void testInWithNotContainedValue() { In in = new In(EMPTY, THREE, Arrays.asList(ONE, TWO)); - assertFalse(in.fold()); + assertFalse((Boolean) in.fold()); } public void testHandleNullOnLeftValue() { @@ -41,7 +65,7 @@ public void testHandleNullOnLeftValue() { public void testHandleNullsOnRightValue() { In in = new In(EMPTY, THREE, Arrays.asList(ONE, NULL, THREE)); - assertTrue(in.fold()); + assertTrue((Boolean) in.fold()); in = new In(EMPTY, ONE, Arrays.asList(TWO, NULL, THREE)); assertNull(in.fold()); } @@ -49,4 +73,246 @@ public void testHandleNullsOnRightValue() { private static Literal L(Object value) { return of(EMPTY, value); } + + @ParametersFactory + public static Iterable parameters() { + List suppliers = new ArrayList<>(); + for (int i : new int[] { 1, 3 }) { + booleans(suppliers, i); + numerics(suppliers, i); + bytesRefs(suppliers, i); + } + return parameterSuppliersFromTypedData(suppliers); + } + + private static void booleans(List suppliers, int items) { + suppliers.add(new TestCaseSupplier("boolean", List.of(DataType.BOOLEAN, DataType.BOOLEAN), () -> { + List inlist = randomList(items, items, () -> randomBoolean()); + boolean field = randomBoolean(); + List args = new ArrayList<>(inlist.size() + 1); + for (Boolean i : inlist) { + args.add(new TestCaseSupplier.TypedData(i, DataType.BOOLEAN, "inlist" + i)); + } + args.add(new TestCaseSupplier.TypedData(field, DataType.BOOLEAN, "field")); + return new TestCaseSupplier.TestCase( + args, + matchesPattern("InBooleanEvaluator.*"), + DataType.BOOLEAN, + equalTo(inlist.contains(field)) + ); + })); + } + + private static void numerics(List suppliers, int items) { + suppliers.add(new TestCaseSupplier("integer", List.of(DataType.INTEGER, DataType.INTEGER), () -> { + List inlist = randomList(items, items, () -> randomInt()); + int field = inlist.get(inlist.size() - 1); + List args = new ArrayList<>(inlist.size() + 1); + for (Integer i : inlist) { + args.add(new TestCaseSupplier.TypedData(i, DataType.INTEGER, "inlist" + i)); + } + args.add(new TestCaseSupplier.TypedData(field, DataType.INTEGER, "field")); + return new TestCaseSupplier.TestCase( + args, + matchesPattern("InIntEvaluator.*"), + DataType.BOOLEAN, + equalTo(inlist.contains(field)) + ); + })); + + suppliers.add(new TestCaseSupplier("long", List.of(DataType.LONG, DataType.LONG), () -> { + List inlist = randomList(items, items, () -> randomLong()); + long field = randomLong(); + List args = new ArrayList<>(inlist.size() + 1); + for (Long i : inlist) { + args.add(new TestCaseSupplier.TypedData(i, DataType.LONG, "inlist" + i)); + } + args.add(new TestCaseSupplier.TypedData(field, DataType.LONG, "field")); + return new TestCaseSupplier.TestCase( + args, + matchesPattern("InLongEvaluator.*"), + DataType.BOOLEAN, + equalTo(inlist.contains(field)) + ); + })); + + suppliers.add(new TestCaseSupplier("double", List.of(DataType.DOUBLE, DataType.DOUBLE), () -> { + List inlist = randomList(items, items, () -> randomDouble()); + double field = inlist.get(0); + List args = new ArrayList<>(inlist.size() + 1); + for (Double i : inlist) { + args.add(new TestCaseSupplier.TypedData(i, DataType.DOUBLE, "inlist" + i)); + } + args.add(new TestCaseSupplier.TypedData(field, DataType.DOUBLE, "field")); + return new TestCaseSupplier.TestCase( + args, + matchesPattern("InDoubleEvaluator.*"), + DataType.BOOLEAN, + equalTo(inlist.contains(field)) + ); + })); + } + + private static void bytesRefs(List suppliers, int items) { + suppliers.add(new TestCaseSupplier("keyword", List.of(DataType.KEYWORD, DataType.KEYWORD), () -> { + List inlist = randomList(items, items, () -> randomLiteral(DataType.KEYWORD).value()); + Object field = inlist.get(inlist.size() - 1); + List args = new ArrayList<>(inlist.size() + 1); + for (Object i : inlist) { + args.add(new TestCaseSupplier.TypedData(i, DataType.KEYWORD, "inlist" + i)); + } + args.add(new TestCaseSupplier.TypedData(field, DataType.KEYWORD, "field")); + return new TestCaseSupplier.TestCase( + args, + matchesPattern("InBytesRefEvaluator.*"), + DataType.BOOLEAN, + equalTo(inlist.contains(field)) + ); + })); + + suppliers.add(new TestCaseSupplier("text", List.of(DataType.TEXT, DataType.TEXT), () -> { + List inlist = randomList(items, items, () -> randomLiteral(DataType.TEXT).value()); + Object field = inlist.get(0); + List args = new ArrayList<>(inlist.size() + 1); + for (Object i : inlist) { + args.add(new TestCaseSupplier.TypedData(i, DataType.TEXT, "inlist" + i)); + } + args.add(new TestCaseSupplier.TypedData(field, DataType.TEXT, "field")); + return new TestCaseSupplier.TestCase( + args, + matchesPattern("InBytesRefEvaluator.*"), + DataType.BOOLEAN, + equalTo(inlist.contains(field)) + ); + })); + + for (DataType type1 : new DataType[] { DataType.KEYWORD, DataType.TEXT }) { + for (DataType type2 : new DataType[] { DataType.KEYWORD, DataType.TEXT }) { + if (type1 == type2 || items > 1) continue; + suppliers.add(new TestCaseSupplier(type1 + " " + type2, List.of(type1, type2), () -> { + List inlist = randomList(items, items, () -> randomLiteral(type1).value()); + Object field = randomLiteral(type2).value(); + List args = new ArrayList<>(inlist.size() + 1); + for (Object i : inlist) { + args.add(new TestCaseSupplier.TypedData(i, type1, "inlist" + i)); + } + args.add(new TestCaseSupplier.TypedData(field, type2, "field")); + return new TestCaseSupplier.TestCase( + args, + matchesPattern("InBytesRefEvaluator.*"), + DataType.BOOLEAN, + equalTo(inlist.contains(field)) + ); + })); + } + } + suppliers.add(new TestCaseSupplier("ip", List.of(DataType.IP, DataType.IP), () -> { + List inlist = randomList(items, items, () -> randomLiteral(DataType.IP).value()); + Object field = randomLiteral(DataType.IP).value(); + List args = new ArrayList<>(inlist.size() + 1); + for (Object i : inlist) { + args.add(new TestCaseSupplier.TypedData(i, DataType.IP, "inlist" + i)); + } + args.add(new TestCaseSupplier.TypedData(field, DataType.IP, "field")); + return new TestCaseSupplier.TestCase( + args, + matchesPattern("InBytesRefEvaluator.*"), + DataType.BOOLEAN, + equalTo(inlist.contains(field)) + ); + })); + + suppliers.add(new TestCaseSupplier("version", List.of(DataType.VERSION, DataType.VERSION), () -> { + List inlist = randomList(items, items, () -> randomLiteral(DataType.VERSION).value()); + Object field = randomLiteral(DataType.VERSION).value(); + List args = new ArrayList<>(inlist.size() + 1); + for (Object i : inlist) { + args.add(new TestCaseSupplier.TypedData(i, DataType.VERSION, "inlist" + i)); + } + args.add(new TestCaseSupplier.TypedData(field, DataType.VERSION, "field")); + return new TestCaseSupplier.TestCase( + args, + matchesPattern("InBytesRefEvaluator.*"), + DataType.BOOLEAN, + equalTo(inlist.contains(field)) + ); + })); + + suppliers.add(new TestCaseSupplier("geo_point", List.of(GEO_POINT, GEO_POINT), () -> { + List inlist = randomList(items, items, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomPoint()))); + Object field = inlist.get(0); + List args = new ArrayList<>(inlist.size() + 1); + for (Object i : inlist) { + args.add(new TestCaseSupplier.TypedData(i, GEO_POINT, "inlist" + i)); + } + args.add(new TestCaseSupplier.TypedData(field, GEO_POINT, "field")); + return new TestCaseSupplier.TestCase( + args, + matchesPattern("InBytesRefEvaluator.*"), + DataType.BOOLEAN, + equalTo(inlist.contains(field)) + ); + })); + + suppliers.add(new TestCaseSupplier("geo_shape", List.of(GEO_SHAPE, GEO_SHAPE), () -> { + List inlist = randomList( + items, + items, + () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomGeometry(randomBoolean()))) + ); + Object field = inlist.get(inlist.size() - 1); + List args = new ArrayList<>(inlist.size() + 1); + for (Object i : inlist) { + args.add(new TestCaseSupplier.TypedData(i, GEO_SHAPE, "inlist" + i)); + } + args.add(new TestCaseSupplier.TypedData(field, GEO_SHAPE, "field")); + return new TestCaseSupplier.TestCase( + args, + matchesPattern("InBytesRefEvaluator.*"), + DataType.BOOLEAN, + equalTo(inlist.contains(field)) + ); + })); + + suppliers.add(new TestCaseSupplier("cartesian_point", List.of(CARTESIAN_POINT, CARTESIAN_POINT), () -> { + List inlist = randomList(items, items, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomPoint()))); + Object field = new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomPoint())); + List args = new ArrayList<>(inlist.size() + 1); + for (Object i : inlist) { + args.add(new TestCaseSupplier.TypedData(i, CARTESIAN_POINT, "inlist" + i)); + } + args.add(new TestCaseSupplier.TypedData(field, CARTESIAN_POINT, "field")); + return new TestCaseSupplier.TestCase( + args, + matchesPattern("InBytesRefEvaluator.*"), + DataType.BOOLEAN, + equalTo(inlist.contains(field)) + ); + })); + + suppliers.add(new TestCaseSupplier("cartesian_shape", List.of(CARTESIAN_SHAPE, CARTESIAN_SHAPE), () -> { + List inlist = randomList( + items, + items, + () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean()))) + ); + Object field = new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean()))); + List args = new ArrayList<>(inlist.size() + 1); + for (Object i : inlist) { + args.add(new TestCaseSupplier.TypedData(i, CARTESIAN_SHAPE, "inlist" + i)); + } + args.add(new TestCaseSupplier.TypedData(field, CARTESIAN_SHAPE, "field")); + return new TestCaseSupplier.TestCase( + args, + matchesPattern("InBytesRefEvaluator.*"), + DataType.BOOLEAN, + equalTo(inlist.contains(field)) + ); + })); + } + + @Override + protected Expression build(Source source, List args) { + return new In(source, args.get(args.size() - 1), args.subList(0, args.size() - 1)); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java index d89421f579b08..b65c6a753e14d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java @@ -133,7 +133,12 @@ public static Iterable parameters() { return parameterSuppliersFromTypedData( errorsForCasesWithoutExamples( anyNullIsNull(true, suppliers), - AbstractScalarFunctionTestCase::errorMessageStringForBinaryOperators + (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators( + o, + v, + t, + (l, p) -> "datetime, double, integer, ip, keyword, long, text, unsigned_long or version" + ) ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java index 9487d774ff221..88c79d506e0c7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java @@ -133,7 +133,12 @@ public static Iterable parameters() { return parameterSuppliersFromTypedData( errorsForCasesWithoutExamples( anyNullIsNull(true, suppliers), - AbstractScalarFunctionTestCase::errorMessageStringForBinaryOperators + (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators( + o, + v, + t, + (l, p) -> "datetime, double, integer, ip, keyword, long, text, unsigned_long or version" + ) ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java index e7d8c680ba5cc..73a20da22c7a7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java @@ -192,7 +192,7 @@ public static Iterable parameters() { return parameterSuppliersFromTypedData( errorsForCasesWithoutExamples( anyNullIsNull(true, suppliers), - AbstractScalarFunctionTestCase::errorMessageStringForBinaryOperators + (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators(o, v, t, (l, p) -> "") ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java index 55691526ea428..5cce61484ef87 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java @@ -48,6 +48,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Filter; import org.elasticsearch.xpack.esql.plan.logical.Grok; +import org.elasticsearch.xpack.esql.plan.logical.InlineStats; import org.elasticsearch.xpack.esql.plan.logical.Limit; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.Lookup; @@ -146,6 +147,7 @@ public void testPhysicalPlanEntries() { Eval.class, Filter.class, Grok.class, + InlineStats.class, Join.class, Limit.class, LocalRelation.class, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index dea3a974fbd5a..43c3cb92dff66 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.compute.aggregation.QuantileStates; import org.elasticsearch.core.Tuple; +import org.elasticsearch.dissect.DissectParser; import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.EsqlTestUtils; @@ -21,6 +22,7 @@ import org.elasticsearch.xpack.esql.analysis.AnalyzerContext; import org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils; import org.elasticsearch.xpack.esql.analysis.EnrichResolution; +import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.AttributeSet; @@ -66,6 +68,7 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.Values; import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToLong; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToString; import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateExtract; @@ -107,11 +110,16 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.esql.optimizer.rules.LiteralsOnTheRight; +import org.elasticsearch.xpack.esql.optimizer.rules.OptimizerRules; import org.elasticsearch.xpack.esql.optimizer.rules.PushDownAndCombineFilters; import org.elasticsearch.xpack.esql.optimizer.rules.PushDownAndCombineLimits; +import org.elasticsearch.xpack.esql.optimizer.rules.PushDownEnrich; +import org.elasticsearch.xpack.esql.optimizer.rules.PushDownEval; +import org.elasticsearch.xpack.esql.optimizer.rules.PushDownRegexExtract; import org.elasticsearch.xpack.esql.optimizer.rules.SplitInWithFoldableValue; import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.parser.ParsingException; +import org.elasticsearch.xpack.esql.plan.GeneratingPlan; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Enrich; @@ -140,6 +148,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.BiFunction; import java.util.function.Function; import static java.util.Arrays.asList; @@ -157,6 +166,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.localSource; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.referenceAttribute; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.analysis.Analyzer.NO_FIELDS; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyze; @@ -188,6 +198,7 @@ import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.everyItem; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; @@ -1021,7 +1032,7 @@ public void testPushDownDissectPastProject() { var keep = as(plan, Project.class); var dissect = as(keep.child(), Dissect.class); - assertThat(dissect.extractedFields(), contains(new ReferenceAttribute(Source.EMPTY, "y", DataType.KEYWORD))); + assertThat(dissect.extractedFields(), contains(referenceAttribute("y", DataType.KEYWORD))); } public void testPushDownGrokPastProject() { @@ -1034,7 +1045,7 @@ public void testPushDownGrokPastProject() { var keep = as(plan, Project.class); var grok = as(keep.child(), Grok.class); - assertThat(grok.extractedFields(), contains(new ReferenceAttribute(Source.EMPTY, "y", DataType.KEYWORD))); + assertThat(grok.extractedFields(), contains(referenceAttribute("y", DataType.KEYWORD))); } public void testPushDownFilterPastProjectUsingEval() { @@ -4254,6 +4265,210 @@ public void testPushdownWithOverwrittenName() { } } + record PushdownShadowingGeneratingPlanTestCase( + BiFunction applyLogicalPlan, + OptimizerRules.OptimizerRule rule + ) {}; + + static PushdownShadowingGeneratingPlanTestCase[] PUSHDOWN_SHADOWING_GENERATING_PLAN_TEST_CASES = { + // | EVAL y = to_integer(x), y = y + 1 + new PushdownShadowingGeneratingPlanTestCase((plan, attr) -> { + Alias y1 = new Alias(EMPTY, "y", new ToInteger(EMPTY, attr)); + Alias y2 = new Alias(EMPTY, "y", new Add(EMPTY, y1.toAttribute(), new Literal(EMPTY, 1, INTEGER))); + return new Eval(EMPTY, plan, List.of(y1, y2)); + }, new PushDownEval()), + // | DISSECT x "%{y} %{y}" + new PushdownShadowingGeneratingPlanTestCase( + (plan, attr) -> new Dissect( + EMPTY, + plan, + attr, + new Dissect.Parser("%{y} %{y}", ",", new DissectParser("%{y} %{y}", ",")), + List.of(new ReferenceAttribute(EMPTY, "y", KEYWORD), new ReferenceAttribute(EMPTY, "y", KEYWORD)) + ), + new PushDownRegexExtract() + ), + // | GROK x "%{WORD:y} %{WORD:y}" + new PushdownShadowingGeneratingPlanTestCase( + (plan, attr) -> new Grok(EMPTY, plan, attr, Grok.pattern(EMPTY, "%{WORD:y} %{WORD:y}")), + new PushDownRegexExtract() + ), + // | ENRICH some_policy ON x WITH y = some_enrich_idx_field, y = some_other_enrich_idx_field + new PushdownShadowingGeneratingPlanTestCase( + (plan, attr) -> new Enrich( + EMPTY, + plan, + Enrich.Mode.ANY, + new Literal(EMPTY, "some_policy", KEYWORD), + attr, + null, + Map.of(), + List.of( + new Alias(EMPTY, "y", new ReferenceAttribute(EMPTY, "some_enrich_idx_field", KEYWORD)), + new Alias(EMPTY, "y", new ReferenceAttribute(EMPTY, "some_other_enrich_idx_field", KEYWORD)) + ) + ), + new PushDownEnrich() + ) }; + + /** + * Consider + * + * Eval[[TO_INTEGER(x{r}#2) AS y, y{r}#4 + 1[INTEGER] AS y]] + * \_Project[[y{r}#3, x{r}#2]] + * \_Row[[1[INTEGER] AS x, 2[INTEGER] AS y]] + * + * We can freely push down the Eval without renaming, but need to update the Project's references. + * + * Project[[x{r}#2, y{r}#6 AS y]] + * \_Eval[[TO_INTEGER(x{r}#2) AS y, y{r}#4 + 1[INTEGER] AS y]] + * \_Row[[1[INTEGER] AS x, 2[INTEGER] AS y]] + * + * And similarly for dissect, grok and enrich. + */ + public void testPushShadowingGeneratingPlanPastProject() { + Alias x = new Alias(EMPTY, "x", new Literal(EMPTY, "1", KEYWORD)); + Alias y = new Alias(EMPTY, "y", new Literal(EMPTY, "2", KEYWORD)); + LogicalPlan initialRow = new Row(EMPTY, List.of(x, y)); + LogicalPlan initialProject = new Project(EMPTY, initialRow, List.of(y.toAttribute(), x.toAttribute())); + + for (PushdownShadowingGeneratingPlanTestCase testCase : PUSHDOWN_SHADOWING_GENERATING_PLAN_TEST_CASES) { + LogicalPlan initialPlan = testCase.applyLogicalPlan.apply(initialProject, x.toAttribute()); + @SuppressWarnings("unchecked") + List initialGeneratedExprs = ((GeneratingPlan) initialPlan).generatedAttributes(); + LogicalPlan optimizedPlan = testCase.rule.apply(initialPlan); + + Failures inconsistencies = LogicalVerifier.INSTANCE.verify(optimizedPlan); + assertFalse(inconsistencies.hasFailures()); + + Project project = as(optimizedPlan, Project.class); + LogicalPlan pushedDownGeneratingPlan = project.child(); + + List projections = project.projections(); + @SuppressWarnings("unchecked") + List newGeneratedExprs = ((GeneratingPlan) pushedDownGeneratingPlan).generatedAttributes(); + assertEquals(newGeneratedExprs, initialGeneratedExprs); + // The rightmost generated attribute makes it into the final output as "y". + Attribute rightmostGenerated = newGeneratedExprs.get(newGeneratedExprs.size() - 1); + + assertThat(Expressions.names(projections), contains("x", "y")); + assertThat(projections, everyItem(instanceOf(ReferenceAttribute.class))); + ReferenceAttribute yShadowed = as(projections.get(1), ReferenceAttribute.class); + assertTrue(yShadowed.semanticEquals(rightmostGenerated)); + } + } + + /** + * Consider + * + * Eval[[TO_INTEGER(x{r}#2) AS y, y{r}#4 + 1[INTEGER] AS y]] + * \_Project[[x{r}#2, y{r}#3, y{r}#3 AS z]] + * \_Row[[1[INTEGER] AS x, 2[INTEGER] AS y]] + * + * To push down the Eval, we must not shadow the reference y{r}#3, so we rename. + * + * Project[[x{r}#2, y{r}#3 AS z, $$y$temp_name$10{r}#12 AS y]] + * Eval[[TO_INTEGER(x{r}#2) AS $$y$temp_name$10, $$y$temp_name$10{r}#11 + 1[INTEGER] AS $$y$temp_name$10]] + * \_Row[[1[INTEGER] AS x, 2[INTEGER] AS y]] + * + * And similarly for dissect, grok and enrich. + */ + public void testPushShadowingGeneratingPlanPastRenamingProject() { + Alias x = new Alias(EMPTY, "x", new Literal(EMPTY, "1", KEYWORD)); + Alias y = new Alias(EMPTY, "y", new Literal(EMPTY, "2", KEYWORD)); + LogicalPlan initialRow = new Row(EMPTY, List.of(x, y)); + LogicalPlan initialProject = new Project( + EMPTY, + initialRow, + List.of(x.toAttribute(), y.toAttribute(), new Alias(EMPTY, "z", y.toAttribute())) + ); + + for (PushdownShadowingGeneratingPlanTestCase testCase : PUSHDOWN_SHADOWING_GENERATING_PLAN_TEST_CASES) { + LogicalPlan initialPlan = testCase.applyLogicalPlan.apply(initialProject, x.toAttribute()); + @SuppressWarnings("unchecked") + List initialGeneratedExprs = ((GeneratingPlan) initialPlan).generatedAttributes(); + LogicalPlan optimizedPlan = testCase.rule.apply(initialPlan); + + Failures inconsistencies = LogicalVerifier.INSTANCE.verify(optimizedPlan); + assertFalse(inconsistencies.hasFailures()); + + Project project = as(optimizedPlan, Project.class); + LogicalPlan pushedDownGeneratingPlan = project.child(); + + List projections = project.projections(); + @SuppressWarnings("unchecked") + List newGeneratedExprs = ((GeneratingPlan) pushedDownGeneratingPlan).generatedAttributes(); + List newNames = Expressions.names(newGeneratedExprs); + assertThat(newNames.size(), equalTo(initialGeneratedExprs.size())); + assertThat(newNames, everyItem(startsWith("$$y$temp_name$"))); + // The rightmost generated attribute makes it into the final output as "y". + Attribute rightmostGeneratedWithNewName = newGeneratedExprs.get(newGeneratedExprs.size() - 1); + + assertThat(Expressions.names(projections), contains("x", "z", "y")); + assertThat(projections.get(0), instanceOf(ReferenceAttribute.class)); + Alias zAlias = as(projections.get(1), Alias.class); + ReferenceAttribute yRenamed = as(zAlias.child(), ReferenceAttribute.class); + assertEquals(yRenamed.name(), "y"); + Alias yAlias = as(projections.get(2), Alias.class); + ReferenceAttribute yTempRenamed = as(yAlias.child(), ReferenceAttribute.class); + assertTrue(yTempRenamed.semanticEquals(rightmostGeneratedWithNewName)); + } + } + + /** + * Consider + * + * Eval[[TO_INTEGER(x{r}#2) AS y, y{r}#3 + 1[INTEGER] AS y]] + * \_Project[[y{r}#1, y{r}#1 AS x]] + * \_Row[[2[INTEGER] AS y]] + * + * To push down the Eval, we must not shadow the reference y{r}#1, so we rename. + * Additionally, the rename "y AS x" needs to be propagated into the Eval. + * + * Project[[y{r}#1 AS x, $$y$temp_name$10{r}#12 AS y]] + * Eval[[TO_INTEGER(y{r}#1) AS $$y$temp_name$10, $$y$temp_name$10{r}#11 + 1[INTEGER] AS $$y$temp_name$10]] + * \_Row[[2[INTEGER] AS y]] + * + * And similarly for dissect, grok and enrich. + */ + public void testPushShadowingGeneratingPlanPastRenamingProjectWithResolution() { + Alias y = new Alias(EMPTY, "y", new Literal(EMPTY, "2", KEYWORD)); + Alias yAliased = new Alias(EMPTY, "x", y.toAttribute()); + LogicalPlan initialRow = new Row(EMPTY, List.of(y)); + LogicalPlan initialProject = new Project(EMPTY, initialRow, List.of(y.toAttribute(), yAliased)); + + for (PushdownShadowingGeneratingPlanTestCase testCase : PUSHDOWN_SHADOWING_GENERATING_PLAN_TEST_CASES) { + LogicalPlan initialPlan = testCase.applyLogicalPlan.apply(initialProject, yAliased.toAttribute()); + @SuppressWarnings("unchecked") + List initialGeneratedExprs = ((GeneratingPlan) initialPlan).generatedAttributes(); + LogicalPlan optimizedPlan = testCase.rule.apply(initialPlan); + + // This ensures that our generating plan doesn't use invalid references, resp. that any rename from the Project has + // been propagated into the generating plan. + Failures inconsistencies = LogicalVerifier.INSTANCE.verify(optimizedPlan); + assertFalse(inconsistencies.hasFailures()); + + Project project = as(optimizedPlan, Project.class); + LogicalPlan pushedDownGeneratingPlan = project.child(); + + List projections = project.projections(); + @SuppressWarnings("unchecked") + List newGeneratedExprs = ((GeneratingPlan) pushedDownGeneratingPlan).generatedAttributes(); + List newNames = Expressions.names(newGeneratedExprs); + assertThat(newNames.size(), equalTo(initialGeneratedExprs.size())); + assertThat(newNames, everyItem(startsWith("$$y$temp_name$"))); + // The rightmost generated attribute makes it into the final output as "y". + Attribute rightmostGeneratedWithNewName = newGeneratedExprs.get(newGeneratedExprs.size() - 1); + + assertThat(Expressions.names(projections), contains("x", "y")); + Alias yRenamed = as(projections.get(0), Alias.class); + assertTrue(yRenamed.child().semanticEquals(y.toAttribute())); + Alias yTempRenamed = as(projections.get(1), Alias.class); + ReferenceAttribute yTemp = as(yTempRenamed.child(), ReferenceAttribute.class); + assertTrue(yTemp.semanticEquals(rightmostGeneratedWithNewName)); + } + } + /** * Expects * Project[[min{r}#4, languages{f}#11]] @@ -5507,9 +5722,11 @@ METRICS k8s count(to_long(network.total_bytes_in)) BY bucket(@timestamp, 1 minut EsRelation relation = as(eval.child(), EsRelation.class); assertThat(relation.indexMode(), equalTo(IndexMode.STANDARD)); } - for (int i = 1; i < plans.size(); i++) { - assertThat(plans.get(i), equalTo(plans.get(0))); - } + // TODO: Unmute this part + // https://github.com/elastic/elasticsearch/issues/110827 + // for (int i = 1; i < plans.size(); i++) { + // assertThat(plans.get(i), equalTo(plans.get(0))); + // } } public void testRateInStats() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/BooleanSimplificationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/BooleanSimplificationTests.java index 03cd5921a80e2..6864bf4b9ebef 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/BooleanSimplificationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/BooleanSimplificationTests.java @@ -21,7 +21,7 @@ public class BooleanSimplificationTests extends ESTestCase { new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 0); public void testBoolSimplifyOr() { - OptimizerRules.BooleanSimplification simplification = new OptimizerRules.BooleanSimplification(); + BooleanSimplification simplification = new BooleanSimplification(); assertEquals(TRUE, simplification.rule(new Or(EMPTY, TRUE, TRUE))); assertEquals(TRUE, simplification.rule(new Or(EMPTY, TRUE, DUMMY_EXPRESSION))); @@ -33,7 +33,7 @@ public void testBoolSimplifyOr() { } public void testBoolSimplifyAnd() { - OptimizerRules.BooleanSimplification simplification = new OptimizerRules.BooleanSimplification(); + BooleanSimplification simplification = new BooleanSimplification(); assertEquals(TRUE, simplification.rule(new And(EMPTY, TRUE, TRUE))); assertEquals(DUMMY_EXPRESSION, simplification.rule(new And(EMPTY, TRUE, DUMMY_EXPRESSION))); @@ -45,7 +45,7 @@ public void testBoolSimplifyAnd() { } public void testBoolCommonFactorExtraction() { - OptimizerRules.BooleanSimplification simplification = new OptimizerRules.BooleanSimplification(); + BooleanSimplification simplification = new BooleanSimplification(); Expression a1 = new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 1); Expression a2 = new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 1); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/FoldNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/FoldNullTests.java index db5d42f8bb810..c9302937b1391 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/FoldNullTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/FoldNullTests.java @@ -26,13 +26,13 @@ public class FoldNullTests extends ESTestCase { public void testNullFoldingIsNull() { - OptimizerRules.FoldNull foldNull = new OptimizerRules.FoldNull(); + FoldNull foldNull = new FoldNull(); assertEquals(true, foldNull.rule(new IsNull(EMPTY, NULL)).fold()); assertEquals(false, foldNull.rule(new IsNull(EMPTY, TRUE)).fold()); } public void testGenericNullableExpression() { - OptimizerRules.FoldNull rule = new OptimizerRules.FoldNull(); + FoldNull rule = new FoldNull(); // arithmetic assertNullLiteral(rule.rule(new Add(EMPTY, getFieldAttribute(), NULL))); // comparison @@ -42,7 +42,7 @@ public void testGenericNullableExpression() { } public void testNullFoldingDoesNotApplyOnLogicalExpressions() { - OptimizerRules.FoldNull rule = new OptimizerRules.FoldNull(); + FoldNull rule = new FoldNull(); Or or = new Or(EMPTY, NULL, TRUE); assertEquals(or, rule.rule(or)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/PropagateNullableTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/PropagateNullableTests.java index 23c0886f1a7d3..29d5bb4cab907 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/PropagateNullableTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/PropagateNullableTests.java @@ -44,7 +44,7 @@ public void testIsNullAndNotNull() { FieldAttribute fa = getFieldAttribute(); And and = new And(EMPTY, new IsNull(EMPTY, fa), new IsNotNull(EMPTY, fa)); - assertEquals(FALSE, new OptimizerRules.PropagateNullable().rule(and)); + assertEquals(FALSE, new PropagateNullable().rule(and)); } // a IS NULL AND b IS NOT NULL AND c IS NULL AND d IS NOT NULL AND e IS NULL AND a IS NOT NULL => false @@ -57,7 +57,7 @@ public void testIsNullAndNotNullMultiField() { And and = new And(EMPTY, andOne, new And(EMPTY, andThree, andTwo)); - assertEquals(FALSE, new OptimizerRules.PropagateNullable().rule(and)); + assertEquals(FALSE, new PropagateNullable().rule(and)); } // a IS NULL AND a > 1 => a IS NULL AND false @@ -66,7 +66,7 @@ public void testIsNullAndComparison() { IsNull isNull = new IsNull(EMPTY, fa); And and = new And(EMPTY, isNull, greaterThanOf(fa, ONE)); - assertEquals(new And(EMPTY, isNull, nullOf(BOOLEAN)), new OptimizerRules.PropagateNullable().rule(and)); + assertEquals(new And(EMPTY, isNull, nullOf(BOOLEAN)), new PropagateNullable().rule(and)); } // a IS NULL AND b < 1 AND c < 1 AND a < 1 => a IS NULL AND b < 1 AND c < 1 => a IS NULL AND b < 1 AND c < 1 @@ -78,7 +78,7 @@ public void testIsNullAndMultipleComparison() { And and = new And(EMPTY, isNull, nestedAnd); And top = new And(EMPTY, and, lessThanOf(fa, ONE)); - Expression optimized = new OptimizerRules.PropagateNullable().rule(top); + Expression optimized = new PropagateNullable().rule(top); Expression expected = new And(EMPTY, and, nullOf(BOOLEAN)); assertEquals(Predicates.splitAnd(expected), Predicates.splitAnd(optimized)); } @@ -96,7 +96,7 @@ public void testIsNullAndDeeplyNestedExpression() { Expression kept = new And(EMPTY, isNull, lessThanOf(getFieldAttribute("b"), THREE)); And and = new And(EMPTY, nullified, kept); - Expression optimized = new OptimizerRules.PropagateNullable().rule(and); + Expression optimized = new PropagateNullable().rule(and); Expression expected = new And(EMPTY, new And(EMPTY, nullOf(BOOLEAN), nullOf(BOOLEAN)), kept); assertEquals(Predicates.splitAnd(expected), Predicates.splitAnd(optimized)); @@ -109,13 +109,13 @@ public void testIsNullInDisjunction() { Or or = new Or(EMPTY, new IsNull(EMPTY, fa), new IsNotNull(EMPTY, fa)); Filter dummy = new Filter(EMPTY, relation(), or); - LogicalPlan transformed = new OptimizerRules.PropagateNullable().apply(dummy); + LogicalPlan transformed = new PropagateNullable().apply(dummy); assertSame(dummy, transformed); assertEquals(or, ((Filter) transformed).condition()); or = new Or(EMPTY, new IsNull(EMPTY, fa), greaterThanOf(fa, ONE)); dummy = new Filter(EMPTY, relation(), or); - transformed = new OptimizerRules.PropagateNullable().apply(dummy); + transformed = new PropagateNullable().apply(dummy); assertSame(dummy, transformed); assertEquals(or, ((Filter) transformed).condition()); } @@ -128,7 +128,6 @@ public void testIsNullDisjunction() { Or or = new Or(EMPTY, isNull, greaterThanOf(fa, THREE)); And and = new And(EMPTY, new Add(EMPTY, fa, ONE), or); - assertEquals(and, new OptimizerRules.PropagateNullable().rule(and)); + assertEquals(and, new PropagateNullable().rule(and)); } - } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java index d575ba1fcb55a..697dfcf0a8e6b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.core.expression.Literal; -import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; @@ -55,10 +54,6 @@ static UnresolvedAttribute attribute(String name) { return new UnresolvedAttribute(EMPTY, name); } - static ReferenceAttribute referenceAttribute(String name, DataType type) { - return new ReferenceAttribute(EMPTY, name, type); - } - static Literal integer(int i) { return new Literal(EMPTY, i, DataType.INTEGER); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 2f76cb2049820..35688175f372a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -12,15 +12,18 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.xpack.esql.core.capabilities.UnresolvedException; import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.Order; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.core.plan.TableIdentifier; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; @@ -33,8 +36,6 @@ import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Enrich; -import org.elasticsearch.xpack.esql.plan.logical.EsqlAggregate; -import org.elasticsearch.xpack.esql.plan.logical.EsqlUnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Explain; import org.elasticsearch.xpack.esql.plan.logical.Filter; @@ -47,12 +48,14 @@ import org.elasticsearch.xpack.esql.plan.logical.OrderBy; import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.Row; +import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; import java.util.List; import java.util.Map; import java.util.function.Function; import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.referenceAttribute; import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE; import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; @@ -103,6 +106,17 @@ public void testRowCommandHugeInt() { ); } + public void testRowCommandHugeNegativeInt() { + assertEquals( + new Row(EMPTY, List.of(new Alias(EMPTY, "c", literalDouble(-92233720368547758080d)))), + statement("row c = -92233720368547758080") + ); + assertEquals( + new Row(EMPTY, List.of(new Alias(EMPTY, "c", literalDouble(-18446744073709551616d)))), + statement("row c = -18446744073709551616") + ); + } + public void testRowCommandDouble() { assertEquals(new Row(EMPTY, List.of(new Alias(EMPTY, "c", literalDouble(1.0)))), statement("row c = 1.0")); } @@ -233,7 +247,7 @@ public void testEvalImplicitNames() { public void testStatsWithGroups() { assertEquals( - new EsqlAggregate( + new Aggregate( EMPTY, PROCESSING_CMD_INPUT, Aggregate.AggregateType.STANDARD, @@ -250,7 +264,7 @@ public void testStatsWithGroups() { public void testStatsWithoutGroups() { assertEquals( - new EsqlAggregate( + new Aggregate( EMPTY, PROCESSING_CMD_INPUT, Aggregate.AggregateType.STANDARD, @@ -266,13 +280,7 @@ public void testStatsWithoutGroups() { public void testStatsWithoutAggs() throws Exception { assertEquals( - new EsqlAggregate( - EMPTY, - PROCESSING_CMD_INPUT, - Aggregate.AggregateType.STANDARD, - List.of(attribute("a")), - List.of(attribute("a")) - ), + new Aggregate(EMPTY, PROCESSING_CMD_INPUT, Aggregate.AggregateType.STANDARD, List.of(attribute("a")), List.of(attribute("a"))), processingCommand("stats by a") ); } @@ -303,6 +311,7 @@ public void testAggsWithGroupKeyAsAgg() throws Exception { } public void testInlineStatsWithGroups() { + assumeTrue("INLINESTATS requires snapshot builds", Build.current().isSnapshot()); assertEquals( new InlineStats( EMPTY, @@ -319,6 +328,7 @@ public void testInlineStatsWithGroups() { } public void testInlineStatsWithoutGroups() { + assumeTrue("INLINESTATS requires snapshot builds", Build.current().isSnapshot()); assertEquals( new InlineStats( EMPTY, @@ -509,7 +519,7 @@ public void testBasicLimitCommand() { assertThat(limit.children().size(), equalTo(1)); assertThat(limit.children().get(0), instanceOf(Filter.class)); assertThat(limit.children().get(0).children().size(), equalTo(1)); - assertThat(limit.children().get(0).children().get(0), instanceOf(EsqlUnresolvedRelation.class)); + assertThat(limit.children().get(0).children().get(0), instanceOf(UnresolvedRelation.class)); } public void testLimitConstraints() { @@ -559,7 +569,7 @@ public void testBasicSortCommand() { assertThat(orderBy.children().size(), equalTo(1)); assertThat(orderBy.children().get(0), instanceOf(Filter.class)); assertThat(orderBy.children().get(0).children().size(), equalTo(1)); - assertThat(orderBy.children().get(0).children().get(0), instanceOf(EsqlUnresolvedRelation.class)); + assertThat(orderBy.children().get(0).children().get(0), instanceOf(UnresolvedRelation.class)); } public void testSubquery() { @@ -1065,7 +1075,7 @@ public void testParamInWhere() { Filter w = (Filter) limit.children().get(0); assertThat(((Literal) w.condition().children().get(1)).value(), equalTo(5)); assertThat(limit.children().get(0).children().size(), equalTo(1)); - assertThat(limit.children().get(0).children().get(0), instanceOf(EsqlUnresolvedRelation.class)); + assertThat(limit.children().get(0).children().get(0), instanceOf(UnresolvedRelation.class)); plan = statement("from test | where x < ?n1 | limit 10", new QueryParams(List.of(new QueryParam("n1", 5, INTEGER)))); assertThat(plan, instanceOf(Limit.class)); @@ -1077,7 +1087,7 @@ public void testParamInWhere() { w = (Filter) limit.children().get(0); assertThat(((Literal) w.condition().children().get(1)).value(), equalTo(5)); assertThat(limit.children().get(0).children().size(), equalTo(1)); - assertThat(limit.children().get(0).children().get(0), instanceOf(EsqlUnresolvedRelation.class)); + assertThat(limit.children().get(0).children().get(0), instanceOf(UnresolvedRelation.class)); plan = statement("from test | where x < ?1 | limit 10", new QueryParams(List.of(new QueryParam(null, 5, INTEGER)))); assertThat(plan, instanceOf(Limit.class)); @@ -1089,7 +1099,7 @@ public void testParamInWhere() { w = (Filter) limit.children().get(0); assertThat(((Literal) w.condition().children().get(1)).value(), equalTo(5)); assertThat(limit.children().get(0).children().size(), equalTo(1)); - assertThat(limit.children().get(0).children().get(0), instanceOf(EsqlUnresolvedRelation.class)); + assertThat(limit.children().get(0).children().get(0), instanceOf(UnresolvedRelation.class)); } public void testParamInEval() { @@ -1111,7 +1121,7 @@ public void testParamInEval() { Filter f = (Filter) eval.children().get(0); assertThat(((Literal) f.condition().children().get(1)).value(), equalTo(5)); assertThat(f.children().size(), equalTo(1)); - assertThat(f.children().get(0), instanceOf(EsqlUnresolvedRelation.class)); + assertThat(f.children().get(0), instanceOf(UnresolvedRelation.class)); plan = statement( "from test | where x < ?n1 | eval y = ?n2 + ?n3 | limit 10", @@ -1131,7 +1141,7 @@ public void testParamInEval() { f = (Filter) eval.children().get(0); assertThat(((Literal) f.condition().children().get(1)).value(), equalTo(5)); assertThat(f.children().size(), equalTo(1)); - assertThat(f.children().get(0), instanceOf(EsqlUnresolvedRelation.class)); + assertThat(f.children().get(0), instanceOf(UnresolvedRelation.class)); plan = statement( "from test | where x < ?1 | eval y = ?2 + ?1 | limit 10", @@ -1149,7 +1159,7 @@ public void testParamInEval() { f = (Filter) eval.children().get(0); assertThat(((Literal) f.condition().children().get(1)).value(), equalTo(5)); assertThat(f.children().size(), equalTo(1)); - assertThat(f.children().get(0), instanceOf(EsqlUnresolvedRelation.class)); + assertThat(f.children().get(0), instanceOf(UnresolvedRelation.class)); } public void testParamInAggFunction() { @@ -1164,8 +1174,8 @@ public void testParamInAggFunction() { ) ) ); - assertThat(plan, instanceOf(EsqlAggregate.class)); - EsqlAggregate agg = (EsqlAggregate) plan; + assertThat(plan, instanceOf(Aggregate.class)); + Aggregate agg = (Aggregate) plan; assertThat(((Literal) agg.aggregates().get(0).children().get(0).children().get(0)).value(), equalTo("*")); assertThat(agg.child(), instanceOf(Eval.class)); assertThat(agg.children().size(), equalTo(1)); @@ -1176,7 +1186,7 @@ public void testParamInAggFunction() { Filter f = (Filter) eval.children().get(0); assertThat(((Literal) f.condition().children().get(1)).value(), equalTo(5)); assertThat(f.children().size(), equalTo(1)); - assertThat(f.children().get(0), instanceOf(EsqlUnresolvedRelation.class)); + assertThat(f.children().get(0), instanceOf(UnresolvedRelation.class)); plan = statement( "from test | where x < ?n1 | eval y = ?n2 + ?n3 | stats count(?n4) by z", @@ -1189,8 +1199,8 @@ public void testParamInAggFunction() { ) ) ); - assertThat(plan, instanceOf(EsqlAggregate.class)); - agg = (EsqlAggregate) plan; + assertThat(plan, instanceOf(Aggregate.class)); + agg = (Aggregate) plan; assertThat(((Literal) agg.aggregates().get(0).children().get(0).children().get(0)).value(), equalTo("*")); assertThat(agg.child(), instanceOf(Eval.class)); assertThat(agg.children().size(), equalTo(1)); @@ -1201,7 +1211,7 @@ public void testParamInAggFunction() { f = (Filter) eval.children().get(0); assertThat(((Literal) f.condition().children().get(1)).value(), equalTo(5)); assertThat(f.children().size(), equalTo(1)); - assertThat(f.children().get(0), instanceOf(EsqlUnresolvedRelation.class)); + assertThat(f.children().get(0), instanceOf(UnresolvedRelation.class)); plan = statement( "from test | where x < ?1 | eval y = ?2 + ?1 | stats count(?3) by z", @@ -1209,8 +1219,8 @@ public void testParamInAggFunction() { List.of(new QueryParam(null, 5, INTEGER), new QueryParam(null, -1, INTEGER), new QueryParam(null, "*", KEYWORD)) ) ); - assertThat(plan, instanceOf(EsqlAggregate.class)); - agg = (EsqlAggregate) plan; + assertThat(plan, instanceOf(Aggregate.class)); + agg = (Aggregate) plan; assertThat(((Literal) agg.aggregates().get(0).children().get(0).children().get(0)).value(), equalTo("*")); assertThat(agg.child(), instanceOf(Eval.class)); assertThat(agg.children().size(), equalTo(1)); @@ -1221,7 +1231,7 @@ public void testParamInAggFunction() { f = (Filter) eval.children().get(0); assertThat(((Literal) f.condition().children().get(1)).value(), equalTo(5)); assertThat(f.children().size(), equalTo(1)); - assertThat(f.children().get(0), instanceOf(EsqlUnresolvedRelation.class)); + assertThat(f.children().get(0), instanceOf(UnresolvedRelation.class)); } public void testParamMixed() { @@ -1292,8 +1302,8 @@ private void assertStringAsIndexPattern(String string, String statement) { return; } LogicalPlan from = statement(statement); - assertThat(from, instanceOf(EsqlUnresolvedRelation.class)); - EsqlUnresolvedRelation table = (EsqlUnresolvedRelation) from; + assertThat(from, instanceOf(UnresolvedRelation.class)); + UnresolvedRelation table = (UnresolvedRelation) from; assertThat(table.table().index(), is(string)); } @@ -1387,45 +1397,23 @@ public void testInlineConvertUnsupportedType() { public void testMetricsWithoutStats() { assumeTrue("requires snapshot build", Build.current().isSnapshot()); - assertStatement( - "METRICS foo", - new EsqlUnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, "foo"), List.of(), IndexMode.TIME_SERIES) - ); - assertStatement( - "METRICS foo,bar", - new EsqlUnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, "foo,bar"), List.of(), IndexMode.TIME_SERIES) - ); - assertStatement( - "METRICS foo*,bar", - new EsqlUnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, "foo*,bar"), List.of(), IndexMode.TIME_SERIES) - ); - assertStatement( - "METRICS foo-*,bar", - new EsqlUnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, "foo-*,bar"), List.of(), IndexMode.TIME_SERIES) - ); - assertStatement( - "METRICS foo-*,bar+*", - new EsqlUnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, "foo-*,bar+*"), List.of(), IndexMode.TIME_SERIES) - ); + assertStatement("METRICS foo", unresolvedRelation("foo")); + assertStatement("METRICS foo,bar", unresolvedRelation("foo,bar")); + assertStatement("METRICS foo*,bar", unresolvedRelation("foo*,bar")); + assertStatement("METRICS foo-*,bar", unresolvedRelation("foo-*,bar")); + assertStatement("METRICS foo-*,bar+*", unresolvedRelation("foo-*,bar+*")); } public void testMetricsIdentifiers() { assumeTrue("requires snapshot build", Build.current().isSnapshot()); - Map patterns = Map.of( - "metrics foo,test-*", - "foo,test-*", - "metrics 123-test@foo_bar+baz1", - "123-test@foo_bar+baz1", - "metrics foo, test,xyz", - "foo,test,xyz", - "metrics >", - ">" + Map patterns = Map.ofEntries( + Map.entry("metrics foo,test-*", "foo,test-*"), + Map.entry("metrics 123-test@foo_bar+baz1", "123-test@foo_bar+baz1"), + Map.entry("metrics foo, test,xyz", "foo,test,xyz"), + Map.entry("metrics >", ">") ); for (Map.Entry e : patterns.entrySet()) { - assertStatement( - e.getKey(), - new EsqlUnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, e.getValue()), List.of(), IndexMode.TIME_SERIES) - ); + assertStatement(e.getKey(), unresolvedRelation(e.getValue())); } } @@ -1433,9 +1421,9 @@ public void testSimpleMetricsWithStats() { assumeTrue("requires snapshot build", Build.current().isSnapshot()); assertStatement( "METRICS foo load=avg(cpu) BY ts", - new EsqlAggregate( + new Aggregate( EMPTY, - new EsqlUnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, "foo"), List.of(), IndexMode.TIME_SERIES), + unresolvedTSRelation("foo"), Aggregate.AggregateType.METRICS, List.of(attribute("ts")), List.of(new Alias(EMPTY, "load", new UnresolvedFunction(EMPTY, "avg", DEFAULT, List.of(attribute("cpu")))), attribute("ts")) @@ -1443,9 +1431,9 @@ public void testSimpleMetricsWithStats() { ); assertStatement( "METRICS foo,bar load=avg(cpu) BY ts", - new EsqlAggregate( + new Aggregate( EMPTY, - new EsqlUnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, "foo,bar"), List.of(), IndexMode.TIME_SERIES), + unresolvedTSRelation("foo,bar"), Aggregate.AggregateType.METRICS, List.of(attribute("ts")), List.of(new Alias(EMPTY, "load", new UnresolvedFunction(EMPTY, "avg", DEFAULT, List.of(attribute("cpu")))), attribute("ts")) @@ -1453,9 +1441,9 @@ public void testSimpleMetricsWithStats() { ); assertStatement( "METRICS foo,bar load=avg(cpu),max(rate(requests)) BY ts", - new EsqlAggregate( + new Aggregate( EMPTY, - new EsqlUnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, "foo,bar"), List.of(), IndexMode.TIME_SERIES), + unresolvedTSRelation("foo,bar"), Aggregate.AggregateType.METRICS, List.of(attribute("ts")), List.of( @@ -1476,9 +1464,9 @@ public void testSimpleMetricsWithStats() { ); assertStatement( "METRICS foo* count(errors)", - new EsqlAggregate( + new Aggregate( EMPTY, - new EsqlUnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, "foo*"), List.of(), IndexMode.TIME_SERIES), + unresolvedTSRelation("foo*"), Aggregate.AggregateType.METRICS, List.of(), List.of(new Alias(EMPTY, "count(errors)", new UnresolvedFunction(EMPTY, "count", DEFAULT, List.of(attribute("errors"))))) @@ -1486,9 +1474,9 @@ public void testSimpleMetricsWithStats() { ); assertStatement( "METRICS foo* a(b)", - new EsqlAggregate( + new Aggregate( EMPTY, - new EsqlUnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, "foo*"), List.of(), IndexMode.TIME_SERIES), + unresolvedTSRelation("foo*"), Aggregate.AggregateType.METRICS, List.of(), List.of(new Alias(EMPTY, "a(b)", new UnresolvedFunction(EMPTY, "a", DEFAULT, List.of(attribute("b"))))) @@ -1496,9 +1484,9 @@ public void testSimpleMetricsWithStats() { ); assertStatement( "METRICS foo* a(b)", - new EsqlAggregate( + new Aggregate( EMPTY, - new EsqlUnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, "foo*"), List.of(), IndexMode.TIME_SERIES), + unresolvedTSRelation("foo*"), Aggregate.AggregateType.METRICS, List.of(), List.of(new Alias(EMPTY, "a(b)", new UnresolvedFunction(EMPTY, "a", DEFAULT, List.of(attribute("b"))))) @@ -1506,9 +1494,9 @@ public void testSimpleMetricsWithStats() { ); assertStatement( "METRICS foo* a1(b2)", - new EsqlAggregate( + new Aggregate( EMPTY, - new EsqlUnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, "foo*"), List.of(), IndexMode.TIME_SERIES), + unresolvedTSRelation("foo*"), Aggregate.AggregateType.METRICS, List.of(), List.of(new Alias(EMPTY, "a1(b2)", new UnresolvedFunction(EMPTY, "a1", DEFAULT, List.of(attribute("b2"))))) @@ -1516,9 +1504,9 @@ public void testSimpleMetricsWithStats() { ); assertStatement( "METRICS foo*,bar* b = min(a) by c, d.e", - new EsqlAggregate( + new Aggregate( EMPTY, - new EsqlUnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, "foo*,bar*"), List.of(), IndexMode.TIME_SERIES), + unresolvedTSRelation("foo*,bar*"), Aggregate.AggregateType.METRICS, List.of(attribute("c"), attribute("d.e")), List.of( @@ -1530,6 +1518,15 @@ public void testSimpleMetricsWithStats() { ); } + private LogicalPlan unresolvedRelation(String index) { + return new UnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, index), false, List.of(), IndexMode.STANDARD, null); + } + + private LogicalPlan unresolvedTSRelation(String index) { + List metadata = List.of(new MetadataAttribute(EMPTY, MetadataAttribute.TSID_FIELD, DataType.KEYWORD, false)); + return new UnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, index), false, metadata, IndexMode.TIME_SERIES, null); + } + public void testMetricWithGroupKeyAsAgg() { assumeTrue("requires snapshot build", Build.current().isSnapshot()); var queries = List.of("METRICS foo a BY a"); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/PhasedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/PhasedTests.java new file mode 100644 index 0000000000000..4bd8dd3a0e96f --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/PhasedTests.java @@ -0,0 +1,146 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.plan.logical; + +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; +import org.elasticsearch.xpack.esql.core.index.EsIndex; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; + +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; + +public class PhasedTests extends ESTestCase { + public void testZeroLayers() { + EsRelation relation = new EsRelation(Source.synthetic("relation"), new EsIndex("foo", Map.of()), IndexMode.STANDARD, false); + relation.setAnalyzed(); + assertThat(Phased.extractFirstPhase(relation), nullValue()); + } + + public void testOneLayer() { + EsRelation relation = new EsRelation(Source.synthetic("relation"), new EsIndex("foo", Map.of()), IndexMode.STANDARD, false); + LogicalPlan orig = new Dummy(Source.synthetic("orig"), relation); + orig.setAnalyzed(); + assertThat(Phased.extractFirstPhase(orig), sameInstance(relation)); + LogicalPlan finalPhase = Phased.applyResultsFromFirstPhase( + orig, + List.of(new ReferenceAttribute(Source.EMPTY, "foo", DataType.KEYWORD)), + List.of() + ); + assertThat( + finalPhase, + equalTo(new Row(orig.source(), List.of(new Alias(orig.source(), "foo", new Literal(orig.source(), "foo", DataType.KEYWORD))))) + ); + assertThat(Phased.extractFirstPhase(finalPhase), nullValue()); + } + + public void testTwoLayer() { + EsRelation relation = new EsRelation(Source.synthetic("relation"), new EsIndex("foo", Map.of()), IndexMode.STANDARD, false); + LogicalPlan inner = new Dummy(Source.synthetic("inner"), relation); + LogicalPlan orig = new Dummy(Source.synthetic("outer"), inner); + orig.setAnalyzed(); + assertThat( + "extractFirstPhase should call #firstPhase on the earliest child in the plan", + Phased.extractFirstPhase(orig), + sameInstance(relation) + ); + LogicalPlan secondPhase = Phased.applyResultsFromFirstPhase( + orig, + List.of(new ReferenceAttribute(Source.EMPTY, "foo", DataType.KEYWORD)), + List.of() + ); + assertThat( + "applyResultsFromFirstPhase should call #nextPhase one th earliest child in the plan", + secondPhase, + equalTo( + new Dummy( + Source.synthetic("outer"), + new Row(orig.source(), List.of(new Alias(orig.source(), "foo", new Literal(orig.source(), "foo", DataType.KEYWORD)))) + ) + ) + ); + + assertThat(Phased.extractFirstPhase(secondPhase), sameInstance(secondPhase.children().get(0))); + LogicalPlan finalPhase = Phased.applyResultsFromFirstPhase( + secondPhase, + List.of(new ReferenceAttribute(Source.EMPTY, "foo", DataType.KEYWORD)), + List.of() + ); + assertThat( + finalPhase, + equalTo(new Row(orig.source(), List.of(new Alias(orig.source(), "foo", new Literal(orig.source(), "foo", DataType.KEYWORD))))) + ); + + assertThat(Phased.extractFirstPhase(finalPhase), nullValue()); + } + + public class Dummy extends UnaryPlan implements Phased { + Dummy(Source source, LogicalPlan child) { + super(source, child); + } + + @Override + public boolean expressionsResolved() { + throw new UnsupportedOperationException(); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Dummy::new, child()); + } + + @Override + public int hashCode() { + return child().hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof Dummy == false) { + return false; + } + Dummy other = (Dummy) obj; + return child().equals(other.child()); + } + + @Override + public UnaryPlan replaceChild(LogicalPlan newChild) { + return new Dummy(source(), newChild); + } + + @Override + public List output() { + return child().output(); + } + + @Override + public LogicalPlan firstPhase() { + return child(); + } + + @Override + public LogicalPlan nextPhase(List schema, List firstPhaseResult) { + // Replace myself with a dummy "row" command + return new Row( + source(), + schema.stream().map(a -> new Alias(source(), a.name(), new Literal(source(), a.name(), DataType.KEYWORD))).toList() + ); + } + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java index 26db9975b30d8..5954836096ffa 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java @@ -74,7 +74,7 @@ public void testBinaryComparisons() { assertQueryTranslation(""" FROM test | WHERE "2007-12-03T10:15:30+01:00" > date""", containsString(""" - "esql_single_value":{"field":"date","next":{"range":{"date":{"lt":1196673330000,"time_zone":"Z",""")); + "esql_single_value":{"field":"date","next":{"range":{"date":{"lt":"2007-12-03T09:15:30.000Z","time_zone":"Z",""")); assertQueryTranslation(""" FROM test | WHERE 2147483648::unsigned_long > unsigned_long""", containsString(""" @@ -94,7 +94,7 @@ public void testBinaryComparisons() { assertQueryTranslation(""" FROM test | WHERE "2007-12-03T10:15:30+01:00" == date""", containsString(""" - "esql_single_value":{"field":"date","next":{"term":{"date":{"value":1196673330000}""")); + "esql_single_value":{"field":"date","next":{"term":{"date":{"value":"2007-12-03T09:15:30.000Z""")); assertQueryTranslation(""" FROM test | WHERE ip != "127.0.0.1\"""", containsString(""" @@ -135,12 +135,12 @@ public void testRanges() { assertQueryTranslation(""" FROM test | WHERE "2007-12-03T10:15:30+01:00" < date AND date < "2024-01-01T10:15:30+01:00\"""", matchesRegex(""" .*must.*esql_single_value":\\{"field":"date\"""" + """ - .*"range":\\{"date":\\{"gt":1196673330000,.*"range":\\{"date":\\{"lt":1704100530000.*""")); + .*"range":\\{"date":\\{"gt":\"2007-12-03T09:15:30.000Z\",.*"range":\\{"date":\\{"lt":\"2024-01-01T09:15:30.000Z\".*""")); assertQueryTranslation(""" FROM test | WHERE "2007-12-03T10:15:30+01:00" <= date AND date <= "2024-01-01T10:15:30+01:00\"""", matchesRegex(""" .*must.*esql_single_value":\\{"field":"date\"""" + """ - .*"range":\\{"date":\\{"gte":1196673330000,.*"range":\\{"date":\\{"lte":1704100530000.*""")); + .*"range":\\{"date":\\{"gte":\"2007-12-03T09:15:30.000Z\",.*"range":\\{"date":\\{"lte":\"2024-01-01T09:15:30.000Z\".*""")); assertQueryTranslation(""" FROM test | WHERE 2147483648::unsigned_long < unsigned_long AND unsigned_long < 2147483650::unsigned_long""", matchesRegex(""" diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java index b08a2798bc509..0cd1fa11a7499 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java @@ -39,7 +39,6 @@ import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec; import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner.LocalExecutionPlannerContext; import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner.PhysicalOperation; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.util.List; import java.util.Random; @@ -323,7 +322,7 @@ private Block extractBlockForColumn( } private boolean shouldMapToDocValues(DataType dataType, MappedFieldType.FieldExtractPreference extractPreference) { - return extractPreference == DOC_VALUES && EsqlDataTypes.isSpatialPoint(dataType); + return extractPreference == DOC_VALUES && DataType.isSpatialPoint(dataType); } private static class TestBlockCopier { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueMathQueryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueMathQueryTests.java new file mode 100644 index 0000000000000..f49dfe67e591a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueMathQueryTests.java @@ -0,0 +1,203 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.querydsl.query; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.document.DoubleField; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KeywordField; +import org.apache.lucene.document.LongField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.sameInstance; + +public class SingleValueMathQueryTests extends MapperServiceTestCase { + interface Setup { + XContentBuilder mapping(XContentBuilder builder) throws IOException; + + List> build(RandomIndexWriter iw) throws IOException; + + void assertRewrite(IndexSearcher indexSearcher, Query query) throws IOException; + } + + @ParametersFactory + public static List params() { + List params = new ArrayList<>(); + for (String fieldType : new String[] { "long", "integer", "short", "byte", "double", "float", "keyword" }) { + for (boolean multivaluedField : new boolean[] { true, false }) { + for (boolean allowEmpty : new boolean[] { true, false }) { + params.add(new Object[] { new StandardSetup(fieldType, multivaluedField, allowEmpty, 100) }); + } + } + } + return params; + } + + private final Setup setup; + + public SingleValueMathQueryTests(Setup setup) { + this.setup = setup; + } + + public void testQuery() throws IOException { + MapperService mapper = createMapperService(mapping(setup::mapping)); + try (Directory d = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), d)) { + List> fieldValues = setup.build(iw); + try (IndexReader reader = iw.getReader()) { + SearchExecutionContext ctx = createSearchExecutionContext(mapper, new IndexSearcher(reader)); + Query query = new SingleValueMatchQuery( + ctx.getForField(mapper.fieldType("foo"), MappedFieldType.FielddataOperation.SEARCH), + new Warnings(Source.EMPTY) + ); + runCase(fieldValues, ctx.searcher().count(query)); + setup.assertRewrite(ctx.searcher(), query); + } + } + } + + public void testEmpty() throws IOException { + MapperService mapper = createMapperService(mapping(setup::mapping)); + try (Directory d = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), d)) { + try (IndexReader reader = iw.getReader()) { + SearchExecutionContext ctx = createSearchExecutionContext(mapper, new IndexSearcher(reader)); + Query query = new SingleValueMatchQuery( + ctx.getForField(mapper.fieldType("foo"), MappedFieldType.FielddataOperation.SEARCH), + new Warnings(Source.EMPTY) + ); + runCase(List.of(), ctx.searcher().count(query)); + } + } + } + + private void runCase(List> fieldValues, int count) { + int expected = 0; + int mvCountInRange = 0; + for (int i = 0; i < fieldValues.size(); i++) { + int valuesCount = fieldValues.get(i).size(); + if (valuesCount == 1) { + expected++; + } else if (valuesCount > 1) { + mvCountInRange++; + } + } + assertThat(count, equalTo(expected)); + // the SingleValueQuery.TwoPhaseIteratorForSortedNumericsAndTwoPhaseQueries can scan all docs - and generate warnings - even if + // inner query matches none, so warn if MVs have been encountered within given range, OR if a full scan is required + if (mvCountInRange > 0) { + assertWarnings( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.IllegalArgumentException: single-value function encountered multi-value" + ); + } + } + + private record StandardSetup(String fieldType, boolean multivaluedField, boolean empty, int count) implements Setup { + @Override + public XContentBuilder mapping(XContentBuilder builder) throws IOException { + return builder.startObject("foo").field("type", fieldType).endObject(); + } + + @Override + public List> build(RandomIndexWriter iw) throws IOException { + List> fieldValues = new ArrayList<>(100); + for (int i = 0; i < count; i++) { + List values = values(i); + fieldValues.add(values); + iw.addDocument(docFor(values)); + } + return fieldValues; + } + + @Override + public void assertRewrite(IndexSearcher indexSearcher, Query query) throws IOException { + if (empty == false && multivaluedField == false) { + assertThat(query.rewrite(indexSearcher), instanceOf(MatchAllDocsQuery.class)); + } else { + assertThat(query.rewrite(indexSearcher), sameInstance(query)); + } + } + + private List values(int i) { + // i == 10 forces at least one multivalued field when we're configured for multivalued fields + boolean makeMultivalued = multivaluedField && (i == 10 || randomBoolean()); + if (makeMultivalued) { + int count = between(2, 10); + Set set = new HashSet<>(count); + while (set.size() < count) { + set.add(randomValue()); + } + return List.copyOf(set); + } + // i == 0 forces at least one empty field when we're configured for empty fields + if (empty && (i == 0 || randomBoolean())) { + return List.of(); + } + return List.of(randomValue()); + } + + private Object randomValue() { + return switch (fieldType) { + case "long" -> randomLong(); + case "integer" -> randomInt(); + case "short" -> randomShort(); + case "byte" -> randomByte(); + case "double" -> randomDouble(); + case "float" -> randomFloat(); + case "keyword" -> randomAlphaOfLength(5); + default -> throw new UnsupportedOperationException(); + }; + } + + private List docFor(Iterable values) { + List fields = new ArrayList<>(); + switch (fieldType) { + case "long", "integer", "short", "byte" -> { + for (Object v : values) { + long l = ((Number) v).longValue(); + fields.add(new LongField("foo", l, Field.Store.NO)); + } + } + case "double", "float" -> { + for (Object v : values) { + double d = ((Number) v).doubleValue(); + fields.add(new DoubleField("foo", d, Field.Store.NO)); + } + } + case "keyword" -> { + for (Object v : values) { + fields.add(new KeywordField("foo", v.toString(), Field.Store.NO)); + } + } + default -> throw new UnsupportedOperationException(); + } + return fields; + } + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuerySerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuerySerializationTests.java index 34c66675fccdd..a3bf34ad38b8e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuerySerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuerySerializationTests.java @@ -19,7 +19,7 @@ public class SingleValueQuerySerializationTests extends AbstractWireSerializingTestCase { @Override protected SingleValueQuery.Builder createTestInstance() { - return new SingleValueQuery.Builder(randomQuery(), randomFieldName(), new SingleValueQuery.Stats(), Source.EMPTY); + return new SingleValueQuery.Builder(randomQuery(), randomFieldName(), Source.EMPTY); } private QueryBuilder randomQuery() { @@ -36,13 +36,11 @@ protected SingleValueQuery.Builder mutateInstance(SingleValueQuery.Builder insta case 0 -> new SingleValueQuery.Builder( randomValueOtherThan(instance.next(), this::randomQuery), instance.field(), - new SingleValueQuery.Stats(), Source.EMPTY ); case 1 -> new SingleValueQuery.Builder( instance.next(), randomValueOtherThan(instance.field(), this::randomFieldName), - new SingleValueQuery.Stats(), Source.EMPTY ); default -> throw new IllegalArgumentException(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java index f26e819685789..2ba397a3cb3de 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java @@ -40,17 +40,12 @@ import java.util.Set; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.instanceOf; public class SingleValueQueryTests extends MapperServiceTestCase { interface Setup { XContentBuilder mapping(XContentBuilder builder) throws IOException; List> build(RandomIndexWriter iw) throws IOException; - - void assertStats(SingleValueQuery.Builder builder, YesNoSometimes subHasTwoPhase); } @ParametersFactory @@ -74,47 +69,31 @@ public SingleValueQueryTests(Setup setup) { } public void testMatchAll() throws IOException { - testCase(new SingleValueQuery(new MatchAll(Source.EMPTY), "foo").asBuilder(), YesNoSometimes.NO, YesNoSometimes.NO, this::runCase); + testCase(new SingleValueQuery(new MatchAll(Source.EMPTY), "foo").asBuilder(), this::runCase); } public void testMatchSome() throws IOException { int max = between(1, 100); testCase( - new SingleValueQuery.Builder(new RangeQueryBuilder("i").lt(max), "foo", new SingleValueQuery.Stats(), Source.EMPTY), - YesNoSometimes.SOMETIMES, - YesNoSometimes.NO, - (fieldValues, count) -> runCase(fieldValues, count, null, max, false) + new SingleValueQuery.Builder(new RangeQueryBuilder("i").lt(max), "foo", Source.EMPTY), + (fieldValues, count) -> runCase(fieldValues, count, null, max) ); } public void testSubPhrase() throws IOException { - testCase( - new SingleValueQuery.Builder( - new MatchPhraseQueryBuilder("str", "fox jumped"), - "foo", - new SingleValueQuery.Stats(), - Source.EMPTY - ), - YesNoSometimes.NO, - YesNoSometimes.YES, - this::runCase - ); + testCase(new SingleValueQuery.Builder(new MatchPhraseQueryBuilder("str", "fox jumped"), "foo", Source.EMPTY), this::runCase); } public void testMatchNone() throws IOException { testCase( - new SingleValueQuery.Builder(new MatchNoneQueryBuilder(), "foo", new SingleValueQuery.Stats(), Source.EMPTY), - YesNoSometimes.YES, - YesNoSometimes.NO, + new SingleValueQuery.Builder(new MatchNoneQueryBuilder(), "foo", Source.EMPTY), (fieldValues, count) -> assertThat(count, equalTo(0)) ); } public void testRewritesToMatchNone() throws IOException { testCase( - new SingleValueQuery.Builder(new TermQueryBuilder("missing", 0), "foo", new SingleValueQuery.Stats(), Source.EMPTY), - YesNoSometimes.YES, - YesNoSometimes.NO, + new SingleValueQuery.Builder(new TermQueryBuilder("missing", 0), "foo", Source.EMPTY), (fieldValues, count) -> assertThat(count, equalTo(0)) ); } @@ -122,8 +101,6 @@ public void testRewritesToMatchNone() throws IOException { public void testNotMatchAll() throws IOException { testCase( new SingleValueQuery(new MatchAll(Source.EMPTY), "foo").negate(Source.EMPTY).asBuilder(), - YesNoSometimes.YES, - YesNoSometimes.NO, (fieldValues, count) -> assertThat(count, equalTo(0)) ); } @@ -131,8 +108,6 @@ public void testNotMatchAll() throws IOException { public void testNotMatchNone() throws IOException { testCase( new SingleValueQuery(new MatchAll(Source.EMPTY).negate(Source.EMPTY), "foo").negate(Source.EMPTY).asBuilder(), - YesNoSometimes.NO, - YesNoSometimes.NO, this::runCase ); } @@ -141,9 +116,7 @@ public void testNotMatchSome() throws IOException { int max = between(1, 100); testCase( new SingleValueQuery(new RangeQuery(Source.EMPTY, "i", null, false, max, false, null), "foo").negate(Source.EMPTY).asBuilder(), - YesNoSometimes.SOMETIMES, - YesNoSometimes.SOMETIMES, - (fieldValues, count) -> runCase(fieldValues, count, max, 100, true) + (fieldValues, count) -> runCase(fieldValues, count, max, 100) ); } @@ -159,10 +132,8 @@ interface TestCase { * @param count The count of the docs the query matched. * @param docsStart The start of the slice in fieldValues we want to consider. If `null`, the start will be 0. * @param docsStop The end of the slice in fieldValues we want to consider. If `null`, the end will be the fieldValues size. - * @param scanForMVs Should the check for Warnings scan the entire fieldValues? This will override the docsStart:docsStop interval, - * which is needed for some cases. */ - private void runCase(List> fieldValues, int count, Integer docsStart, Integer docsStop, boolean scanForMVs) { + private void runCase(List> fieldValues, int count, Integer docsStart, Integer docsStop) { int expected = 0; int min = docsStart != null ? docsStart : 0; int max = docsStop != null ? docsStop : fieldValues.size(); @@ -177,9 +148,8 @@ private void runCase(List> fieldValues, int count, Integer docsStar } assertThat(count, equalTo(expected)); - // the SingleValueQuery.TwoPhaseIteratorForSortedNumericsAndTwoPhaseQueries can scan all docs - and generate warnings - even if - // inner query matches none, so warn if MVs have been encountered within given range, OR if a full scan is required - if (mvCountInRange > 0 || (scanForMVs && fieldValues.stream().anyMatch(x -> x.size() > 1))) { + // we should only have warnings if we have matched a multi-value + if (mvCountInRange > 0) { assertWarnings( "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", "Line -1:-1: java.lang.IllegalArgumentException: single-value function encountered multi-value" @@ -188,21 +158,10 @@ private void runCase(List> fieldValues, int count, Integer docsStar } private void runCase(List> fieldValues, int count) { - runCase(fieldValues, count, null, null, false); - } - - enum YesNoSometimes { - YES, - NO, - SOMETIMES; + runCase(fieldValues, count, null, null); } - private void testCase( - SingleValueQuery.Builder builder, - YesNoSometimes rewritesToMatchNone, - YesNoSometimes subHasTwoPhase, - TestCase testCase - ) throws IOException { + private void testCase(SingleValueQuery.Builder builder, TestCase testCase) throws IOException { MapperService mapper = createMapperService(mapping(setup::mapping)); try (Directory d = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), d)) { List> fieldValues = setup.build(iw); @@ -211,25 +170,6 @@ private void testCase( QueryBuilder rewritten = builder.rewrite(ctx); Query query = rewritten.toQuery(ctx); testCase.run(fieldValues, ctx.searcher().count(query)); - if (rewritesToMatchNone == YesNoSometimes.YES) { - assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class)); - assertThat(builder.stats().missingField(), equalTo(0)); - assertThat(builder.stats().rewrittenToMatchNone(), equalTo(1)); - assertThat(builder.stats().numericSingle(), equalTo(0)); - assertThat(builder.stats().numericMultiNoApprox(), equalTo(0)); - assertThat(builder.stats().numericMultiApprox(), equalTo(0)); - assertThat(builder.stats().ordinalsSingle(), equalTo(0)); - assertThat(builder.stats().ordinalsMultiNoApprox(), equalTo(0)); - assertThat(builder.stats().ordinalsMultiApprox(), equalTo(0)); - assertThat(builder.stats().bytesApprox(), equalTo(0)); - assertThat(builder.stats().bytesNoApprox(), equalTo(0)); - } else { - assertThat(builder.stats().rewrittenToMatchNone(), equalTo(0)); - setup.assertStats(builder, subHasTwoPhase); - } - if (rewritesToMatchNone != YesNoSometimes.SOMETIMES) { - assertThat(builder.stats().noNextScorer(), equalTo(0)); - } assertEqualsAndHashcodeStable(query, rewritten.toQuery(ctx)); } } @@ -316,73 +256,6 @@ private List docFor(int i, Iterable values) { } return fields; } - - @Override - public void assertStats(SingleValueQuery.Builder builder, YesNoSometimes subHasTwoPhase) { - assertThat(builder.stats().missingField(), equalTo(0)); - switch (fieldType) { - case "long", "integer", "short", "byte", "double", "float" -> { - assertThat(builder.stats().ordinalsSingle(), equalTo(0)); - assertThat(builder.stats().ordinalsMultiNoApprox(), equalTo(0)); - assertThat(builder.stats().ordinalsMultiApprox(), equalTo(0)); - assertThat(builder.stats().bytesApprox(), equalTo(0)); - assertThat(builder.stats().bytesNoApprox(), equalTo(0)); - - if (multivaluedField || empty) { - assertThat(builder.stats().numericSingle(), greaterThanOrEqualTo(0)); - switch (subHasTwoPhase) { - case YES -> { - assertThat(builder.stats().numericMultiNoApprox(), equalTo(0)); - assertThat(builder.stats().numericMultiApprox(), greaterThan(0)); - } - case NO -> { - assertThat(builder.stats().numericMultiNoApprox(), greaterThan(0)); - assertThat(builder.stats().numericMultiApprox(), equalTo(0)); - } - case SOMETIMES -> { - assertThat(builder.stats().numericMultiNoApprox() + builder.stats().numericMultiApprox(), greaterThan(0)); - assertThat(builder.stats().numericMultiNoApprox(), greaterThanOrEqualTo(0)); - assertThat(builder.stats().numericMultiApprox(), greaterThanOrEqualTo(0)); - } - } - } else { - assertThat(builder.stats().numericSingle(), greaterThan(0)); - assertThat(builder.stats().numericMultiNoApprox(), equalTo(0)); - assertThat(builder.stats().numericMultiApprox(), equalTo(0)); - } - } - case "keyword" -> { - assertThat(builder.stats().numericSingle(), equalTo(0)); - assertThat(builder.stats().numericMultiNoApprox(), equalTo(0)); - assertThat(builder.stats().numericMultiApprox(), equalTo(0)); - assertThat(builder.stats().bytesApprox(), equalTo(0)); - assertThat(builder.stats().bytesNoApprox(), equalTo(0)); - if (multivaluedField || empty) { - assertThat(builder.stats().ordinalsSingle(), greaterThanOrEqualTo(0)); - switch (subHasTwoPhase) { - case YES -> { - assertThat(builder.stats().ordinalsMultiNoApprox(), equalTo(0)); - assertThat(builder.stats().ordinalsMultiApprox(), greaterThan(0)); - } - case NO -> { - assertThat(builder.stats().ordinalsMultiNoApprox(), greaterThan(0)); - assertThat(builder.stats().ordinalsMultiApprox(), equalTo(0)); - } - case SOMETIMES -> { - assertThat(builder.stats().ordinalsMultiNoApprox() + builder.stats().ordinalsMultiApprox(), greaterThan(0)); - assertThat(builder.stats().ordinalsMultiNoApprox(), greaterThanOrEqualTo(0)); - assertThat(builder.stats().ordinalsMultiApprox(), greaterThanOrEqualTo(0)); - } - } - } else { - assertThat(builder.stats().ordinalsSingle(), greaterThan(0)); - assertThat(builder.stats().ordinalsMultiNoApprox(), equalTo(0)); - assertThat(builder.stats().ordinalsMultiApprox(), equalTo(0)); - } - } - default -> throw new UnsupportedOperationException(); - } - } } private record FieldMissingSetup() implements Setup { @@ -403,18 +276,5 @@ public List> build(RandomIndexWriter iw) throws IOException { } return fieldValues; } - - @Override - public void assertStats(SingleValueQuery.Builder builder, YesNoSometimes subHasTwoPhase) { - assertThat(builder.stats().missingField(), equalTo(1)); - assertThat(builder.stats().numericSingle(), equalTo(0)); - assertThat(builder.stats().numericMultiNoApprox(), equalTo(0)); - assertThat(builder.stats().numericMultiApprox(), equalTo(0)); - assertThat(builder.stats().ordinalsSingle(), equalTo(0)); - assertThat(builder.stats().ordinalsMultiNoApprox(), equalTo(0)); - assertThat(builder.stats().ordinalsMultiApprox(), equalTo(0)); - assertThat(builder.stats().bytesApprox(), equalTo(0)); - assertThat(builder.stats().bytesNoApprox(), equalTo(0)); - } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java index 427c30311df0b..a4d75c1eb85c3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; +import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction; import org.elasticsearch.xpack.esql.analysis.EnrichResolution; import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolver; import org.elasticsearch.xpack.esql.execution.PlanExecutor; @@ -39,6 +40,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.hamcrest.Matchers.instanceOf; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -80,21 +82,21 @@ public void testFailedMetric() { when(fieldCapabilitiesResponse.get()).thenReturn(fields(indices)); doAnswer((Answer) invocation -> { @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; + ActionListener listener = (ActionListener) invocation.getArguments()[2]; // simulate a valid field_caps response so we can parse and correctly analyze de query listener.onResponse(fieldCapabilitiesResponse); return null; - }).when(qlClient).fieldCaps(any(), any()); + }).when(qlClient).execute(eq(EsqlResolveFieldsAction.TYPE), any(), any()); Client esqlClient = mock(Client.class); IndexResolver indexResolver = new IndexResolver(esqlClient, EsqlDataTypeRegistry.INSTANCE); doAnswer((Answer) invocation -> { @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; + ActionListener listener = (ActionListener) invocation.getArguments()[2]; // simulate a valid field_caps response so we can parse and correctly analyze de query listener.onResponse(new FieldCapabilitiesResponse(indexFieldCapabilities(indices), List.of())); return null; - }).when(esqlClient).fieldCaps(any(), any()); + }).when(esqlClient).execute(eq(EsqlResolveFieldsAction.TYPE), any(), any()); var planExecutor = new PlanExecutor(indexResolver); var enrichResolver = mockEnrichResolver(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java index fa20cfdec0ca0..6c6c8ccb665a5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.dissect.DissectParser; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.core.capabilities.UnresolvedException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; @@ -28,6 +29,7 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.FullTextPredicate; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.Like; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.LikePattern; +import org.elasticsearch.xpack.esql.core.session.Configuration; import org.elasticsearch.xpack.esql.core.tree.AbstractNodeTestCase; import org.elasticsearch.xpack.esql.core.tree.Node; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; @@ -44,6 +46,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Grok; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.logical.PhasedTests; import org.elasticsearch.xpack.esql.plan.logical.join.JoinType; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec; import org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.Stat; @@ -113,7 +116,7 @@ public class EsqlNodeSubclassTests> extends NodeS private static final Predicate CLASSNAME_FILTER = className -> { boolean esqlCore = className.startsWith("org.elasticsearch.xpack.esql.core") != false; boolean esqlProper = className.startsWith("org.elasticsearch.xpack.esql") != false; - return esqlCore || esqlProper; + return (esqlCore || esqlProper) && className.equals(PhasedTests.Dummy.class.getName()) == false; }; /** @@ -124,7 +127,7 @@ public class EsqlNodeSubclassTests> extends NodeS @SuppressWarnings("rawtypes") public static List nodeSubclasses() throws IOException { return subclassesOf(Node.class, CLASSNAME_FILTER).stream() - .filter(c -> testClassFor(c) == null) + .filter(c -> testClassFor(c) == null || c != PhasedTests.Dummy.class) .map(c -> new Object[] { c }) .toList(); } @@ -477,6 +480,9 @@ public void accept(Page page) { // ZoneId is a sealed class (cannot be mocked) starting with Java 19 return randomZone(); } + if (argClass == Configuration.class) { + return EsqlTestUtils.randomConfiguration(); + } try { return mock(argClass); } catch (MockitoException e) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/MultiTypeEsFieldTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/MultiTypeEsFieldTests.java index bebfcd7f7bdbc..0fa8719f17744 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/MultiTypeEsFieldTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/MultiTypeEsFieldTests.java @@ -42,7 +42,7 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isString; +import static org.elasticsearch.xpack.esql.core.type.DataType.isString; /** * This test was originally based on the tests for sub-classes of EsField, like InvalidMappedFieldTests. diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsAction.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsAction.java index a87297702bd30..7cc501ff888f4 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsAction.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsAction.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.CountDown; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; @@ -170,7 +171,7 @@ public LocalAction( final IndexNameExpressionResolver resolver, final ThreadPool threadPool ) { - super(NAME, actionFilters, transportService.getTaskManager()); + super(NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.clusterService = clusterService; this.client = client; this.resolver = resolver; diff --git a/x-pack/plugin/geoip-enterprise-downloader/build.gradle b/x-pack/plugin/geoip-enterprise-downloader/build.gradle new file mode 100644 index 0000000000000..ab16609ac7aad --- /dev/null +++ b/x-pack/plugin/geoip-enterprise-downloader/build.gradle @@ -0,0 +1,19 @@ +apply plugin: 'elasticsearch.internal-es-plugin' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' +esplugin { + name 'x-pack-geoip-enterprise-downloader' + description 'Elasticsearch Expanded Pack Plugin - Geoip Enterprise Downloader' + classname 'org.elasticsearch.xpack.geoip.EnterpriseDownloaderPlugin' + extendedPlugins = ['x-pack-core'] +} +base { + archivesName = 'x-pack-geoip-enterprise-downloader' +} + +dependencies { + compileOnly project(path: xpackModule('core')) + testImplementation(testArtifact(project(xpackModule('core')))) +} + +addQaCheckDependencies(project) diff --git a/x-pack/plugin/geoip-enterprise-downloader/src/main/java/org/elasticsearch/xpack/geoip/EnterpriseDownloaderPlugin.java b/x-pack/plugin/geoip-enterprise-downloader/src/main/java/org/elasticsearch/xpack/geoip/EnterpriseDownloaderPlugin.java new file mode 100644 index 0000000000000..e34ecdda81d72 --- /dev/null +++ b/x-pack/plugin/geoip-enterprise-downloader/src/main/java/org/elasticsearch/xpack/geoip/EnterpriseDownloaderPlugin.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.geoip; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.core.XPackPlugin; + +import java.util.Collection; +import java.util.List; + +/** + * This plugin is used to start the enterprise geoip downloader task (See {@link org.elasticsearch.ingest.EnterpriseGeoIpTask}). That task + * requires having a platinum license. But the geoip code is in a non-xpack module that doesn't know about licensing. This plugin has a + * license listener that will start the task if the license is valid, and will stop the task if it becomes invalid. This lets us enforce + * the license without having to either put license logic into a non-xpack module, or put a lot of shared geoip code (much of which does + * not require a platinum license) into xpack. + */ +public class EnterpriseDownloaderPlugin extends Plugin { + + private final Settings settings; + private EnterpriseGeoIpDownloaderLicenseListener enterpriseGeoIpDownloaderLicenseListener; + + public EnterpriseDownloaderPlugin(final Settings settings) { + this.settings = settings; + } + + protected XPackLicenseState getLicenseState() { + return XPackPlugin.getSharedLicenseState(); + } + + @Override + public Collection createComponents(PluginServices services) { + enterpriseGeoIpDownloaderLicenseListener = new EnterpriseGeoIpDownloaderLicenseListener( + services.client(), + services.clusterService(), + services.threadPool(), + getLicenseState() + ); + enterpriseGeoIpDownloaderLicenseListener.init(); + return List.of(enterpriseGeoIpDownloaderLicenseListener); + } +} diff --git a/x-pack/plugin/geoip-enterprise-downloader/src/main/java/org/elasticsearch/xpack/geoip/EnterpriseGeoIpDownloaderLicenseListener.java b/x-pack/plugin/geoip-enterprise-downloader/src/main/java/org/elasticsearch/xpack/geoip/EnterpriseGeoIpDownloaderLicenseListener.java new file mode 100644 index 0000000000000..d6e6f57f10976 --- /dev/null +++ b/x-pack/plugin/geoip-enterprise-downloader/src/main/java/org/elasticsearch/xpack/geoip/EnterpriseGeoIpDownloaderLicenseListener.java @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.geoip; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.UpdateForV9; +import org.elasticsearch.ingest.EnterpriseGeoIpTask.EnterpriseGeoIpTaskParams; +import org.elasticsearch.license.License; +import org.elasticsearch.license.LicenseStateListener; +import org.elasticsearch.license.LicensedFeature; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; +import org.elasticsearch.persistent.PersistentTasksService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.RemoteTransportException; +import org.elasticsearch.xpack.core.XPackField; + +import java.util.Objects; + +import static org.elasticsearch.ingest.EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER; + +public class EnterpriseGeoIpDownloaderLicenseListener implements LicenseStateListener, ClusterStateListener { + private static final Logger logger = LogManager.getLogger(EnterpriseGeoIpDownloaderLicenseListener.class); + // Note: This custom type is GeoIpMetadata.TYPE, but that class is not exposed to this plugin + static final String INGEST_GEOIP_CUSTOM_METADATA_TYPE = "ingest_geoip"; + + private final PersistentTasksService persistentTasksService; + private final ClusterService clusterService; + private final XPackLicenseState licenseState; + private static final LicensedFeature.Momentary ENTERPRISE_GEOIP_FEATURE = LicensedFeature.momentary( + null, + XPackField.ENTERPRISE_GEOIP_DOWNLOADER, + License.OperationMode.PLATINUM + ); + private volatile boolean licenseIsValid = false; + private volatile boolean hasIngestGeoIpMetadata = false; + + protected EnterpriseGeoIpDownloaderLicenseListener( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + XPackLicenseState licenseState + ) { + this.persistentTasksService = new PersistentTasksService(clusterService, threadPool, client); + this.clusterService = clusterService; + this.licenseState = licenseState; + } + + @UpdateForV9 // use MINUS_ONE once that means no timeout + private static final TimeValue MASTER_TIMEOUT = TimeValue.MAX_VALUE; + private volatile boolean licenseStateListenerRegistered; + + public void init() { + listenForLicenseStateChanges(); + clusterService.addListener(this); + } + + void listenForLicenseStateChanges() { + assert licenseStateListenerRegistered == false : "listenForLicenseStateChanges() should only be called once"; + licenseStateListenerRegistered = true; + licenseState.addListener(this); + } + + @Override + public void licenseStateChanged() { + licenseIsValid = ENTERPRISE_GEOIP_FEATURE.checkWithoutTracking(licenseState); + maybeUpdateTaskState(clusterService.state()); + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + hasIngestGeoIpMetadata = event.state().metadata().custom(INGEST_GEOIP_CUSTOM_METADATA_TYPE) != null; + final boolean ingestGeoIpCustomMetaChangedInEvent = event.metadataChanged() + && event.changedCustomMetadataSet().contains(INGEST_GEOIP_CUSTOM_METADATA_TYPE); + final boolean masterNodeChanged = Objects.equals( + event.state().nodes().getMasterNode(), + event.previousState().nodes().getMasterNode() + ) == false; + /* + * We don't want to potentially start the task on every cluster state change, so only maybeUpdateTaskState if this cluster change + * event involved the modification of custom geoip metadata OR a master node change + */ + if (ingestGeoIpCustomMetaChangedInEvent || (masterNodeChanged && hasIngestGeoIpMetadata)) { + maybeUpdateTaskState(event.state()); + } + } + + private void maybeUpdateTaskState(ClusterState state) { + // We should only start/stop task from single node, master is the best as it will go through it anyway + if (state.nodes().isLocalNodeElectedMaster()) { + if (licenseIsValid) { + if (hasIngestGeoIpMetadata) { + ensureTaskStarted(); + } + } else { + ensureTaskStopped(); + } + } + } + + private void ensureTaskStarted() { + assert licenseIsValid : "Task should never be started without valid license"; + persistentTasksService.sendStartRequest( + ENTERPRISE_GEOIP_DOWNLOADER, + ENTERPRISE_GEOIP_DOWNLOADER, + new EnterpriseGeoIpTaskParams(), + MASTER_TIMEOUT, + ActionListener.wrap(r -> logger.debug("Started enterprise geoip downloader task"), e -> { + Throwable t = e instanceof RemoteTransportException ? ExceptionsHelper.unwrapCause(e) : e; + if (t instanceof ResourceAlreadyExistsException == false) { + logger.error("failed to create enterprise geoip downloader task", e); + } + }) + ); + } + + private void ensureTaskStopped() { + ActionListener> listener = ActionListener.wrap( + r -> logger.debug("Stopped enterprise geoip downloader task"), + e -> { + Throwable t = e instanceof RemoteTransportException ? ExceptionsHelper.unwrapCause(e) : e; + if (t instanceof ResourceNotFoundException == false) { + logger.error("failed to remove enterprise geoip downloader task", e); + } + } + ); + persistentTasksService.sendRemoveRequest(ENTERPRISE_GEOIP_DOWNLOADER, MASTER_TIMEOUT, listener); + } +} diff --git a/x-pack/plugin/geoip-enterprise-downloader/src/test/java/org/elasticsearch/xpack/geoip/EnterpriseGeoIpDownloaderLicenseListenerTests.java b/x-pack/plugin/geoip-enterprise-downloader/src/test/java/org/elasticsearch/xpack/geoip/EnterpriseGeoIpDownloaderLicenseListenerTests.java new file mode 100644 index 0000000000000..5a5aacd392f3c --- /dev/null +++ b/x-pack/plugin/geoip-enterprise-downloader/src/test/java/org/elasticsearch/xpack/geoip/EnterpriseGeoIpDownloaderLicenseListenerTests.java @@ -0,0 +1,219 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.geoip; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.license.License; +import org.elasticsearch.license.TestUtils; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.license.internal.XPackLicenseStatus; +import org.elasticsearch.node.Node; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; +import org.elasticsearch.persistent.RemovePersistentTaskAction; +import org.elasticsearch.persistent.StartPersistentTaskAction; +import org.elasticsearch.telemetry.metric.MeterRegistry; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.util.Map; +import java.util.UUID; + +import static org.elasticsearch.xpack.geoip.EnterpriseGeoIpDownloaderLicenseListener.INGEST_GEOIP_CUSTOM_METADATA_TYPE; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class EnterpriseGeoIpDownloaderLicenseListenerTests extends ESTestCase { + + private ThreadPool threadPool; + + @Before + public void setup() { + threadPool = new ThreadPool(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(), MeterRegistry.NOOP); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdownNow(); + } + + public void testAllConditionsMetOnStart() { + // Should never start if not master node, even if all other conditions have been met + final XPackLicenseState licenseState = getAlwaysValidLicense(); + ClusterService clusterService = createClusterService(true, false); + TaskStartAndRemoveMockClient client = new TaskStartAndRemoveMockClient(threadPool, true, false); + EnterpriseGeoIpDownloaderLicenseListener listener = new EnterpriseGeoIpDownloaderLicenseListener( + client, + clusterService, + threadPool, + licenseState + ); + listener.init(); + listener.licenseStateChanged(); + listener.clusterChanged(new ClusterChangedEvent("test", createClusterState(true, true), clusterService.state())); + client.assertTaskStartHasBeenCalled(); + } + + public void testLicenseChanges() { + final TestUtils.UpdatableLicenseState licenseState = new TestUtils.UpdatableLicenseState(); + licenseState.update(new XPackLicenseStatus(License.OperationMode.TRIAL, false, "")); + ClusterService clusterService = createClusterService(true, true); + TaskStartAndRemoveMockClient client = new TaskStartAndRemoveMockClient(threadPool, false, true); + EnterpriseGeoIpDownloaderLicenseListener listener = new EnterpriseGeoIpDownloaderLicenseListener( + client, + clusterService, + threadPool, + licenseState + ); + listener.init(); + listener.licenseStateChanged(); + listener.clusterChanged(new ClusterChangedEvent("test", clusterService.state(), clusterService.state())); + client.expectStartTask = true; + client.expectRemoveTask = false; + licenseState.update(new XPackLicenseStatus(License.OperationMode.TRIAL, true, "")); + listener.licenseStateChanged(); + client.assertTaskStartHasBeenCalled(); + client.expectStartTask = false; + client.expectRemoveTask = true; + licenseState.update(new XPackLicenseStatus(License.OperationMode.TRIAL, false, "")); + listener.licenseStateChanged(); + client.assertTaskRemoveHasBeenCalled(); + } + + public void testDatabaseChanges() { + final XPackLicenseState licenseState = getAlwaysValidLicense(); + ClusterService clusterService = createClusterService(true, false); + TaskStartAndRemoveMockClient client = new TaskStartAndRemoveMockClient(threadPool, false, false); + EnterpriseGeoIpDownloaderLicenseListener listener = new EnterpriseGeoIpDownloaderLicenseListener( + client, + clusterService, + threadPool, + licenseState + ); + listener.init(); + listener.licenseStateChanged(); + listener.clusterChanged(new ClusterChangedEvent("test", clusterService.state(), clusterService.state())); + // add a geoip database, so the task ought to be started: + client.expectStartTask = true; + listener.clusterChanged(new ClusterChangedEvent("test", createClusterState(true, true), clusterService.state())); + client.assertTaskStartHasBeenCalled(); + // Now we remove the geoip databases. The task ought to just be left alone. + client.expectStartTask = false; + client.expectRemoveTask = false; + listener.clusterChanged(new ClusterChangedEvent("test", createClusterState(true, false), clusterService.state())); + } + + public void testMasterChanges() { + // Should never start if not master node, even if all other conditions have been met + final XPackLicenseState licenseState = getAlwaysValidLicense(); + ClusterService clusterService = createClusterService(false, false); + TaskStartAndRemoveMockClient client = new TaskStartAndRemoveMockClient(threadPool, false, false); + EnterpriseGeoIpDownloaderLicenseListener listener = new EnterpriseGeoIpDownloaderLicenseListener( + client, + clusterService, + threadPool, + licenseState + ); + listener.init(); + listener.licenseStateChanged(); + listener.clusterChanged(new ClusterChangedEvent("test", createClusterState(false, true), clusterService.state())); + client.expectStartTask = true; + listener.clusterChanged(new ClusterChangedEvent("test", createClusterState(true, true), clusterService.state())); + } + + private XPackLicenseState getAlwaysValidLicense() { + return new XPackLicenseState(() -> 0); + } + + private ClusterService createClusterService(boolean isMasterNode, boolean hasGeoIpDatabases) { + ClusterService clusterService = mock(ClusterService.class); + ClusterState state = createClusterState(isMasterNode, hasGeoIpDatabases); + when(clusterService.state()).thenReturn(state); + return clusterService; + } + + private ClusterState createClusterState(boolean isMasterNode, boolean hasGeoIpDatabases) { + String indexName = randomAlphaOfLength(5); + Index index = new Index(indexName, UUID.randomUUID().toString()); + IndexMetadata.Builder idxMeta = IndexMetadata.builder(index.getName()) + .settings(indexSettings(IndexVersion.current(), 1, 0).put("index.uuid", index.getUUID())); + String nodeId = ESTestCase.randomAlphaOfLength(8); + DiscoveryNodes.Builder discoveryNodesBuilder = DiscoveryNodes.builder().add(DiscoveryNodeUtils.create(nodeId)).localNodeId(nodeId); + if (isMasterNode) { + discoveryNodesBuilder.masterNodeId(nodeId); + } + ClusterState.Builder clusterStateBuilder = ClusterState.builder(new ClusterName("name")); + if (hasGeoIpDatabases) { + PersistentTasksCustomMetadata tasksCustomMetadata = new PersistentTasksCustomMetadata(1L, Map.of()); + clusterStateBuilder.metadata(Metadata.builder().putCustom(INGEST_GEOIP_CUSTOM_METADATA_TYPE, tasksCustomMetadata).put(idxMeta)); + } + return clusterStateBuilder.nodes(discoveryNodesBuilder).build(); + } + + private static class TaskStartAndRemoveMockClient extends NoOpClient { + + boolean expectStartTask; + boolean expectRemoveTask; + private boolean taskStartCalled = false; + private boolean taskRemoveCalled = false; + + private TaskStartAndRemoveMockClient(ThreadPool threadPool, boolean expectStartTask, boolean expectRemoveTask) { + super(threadPool); + this.expectStartTask = expectStartTask; + this.expectRemoveTask = expectRemoveTask; + } + + @Override + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + if (action.equals(StartPersistentTaskAction.INSTANCE)) { + if (expectStartTask) { + taskStartCalled = true; + } else { + fail("Should not start task"); + } + } else if (action.equals(RemovePersistentTaskAction.INSTANCE)) { + if (expectRemoveTask) { + taskRemoveCalled = true; + } else { + fail("Should not remove task"); + } + } else { + throw new IllegalStateException("unexpected action called [" + action.name() + "]"); + } + } + + void assertTaskStartHasBeenCalled() { + assertTrue(taskStartCalled); + } + + void assertTaskRemoveHasBeenCalled() { + assertTrue(taskRemoveCalled); + } + } +} diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java index c225f94694c01..10d8f90efef5b 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java @@ -309,6 +309,11 @@ public SimilarityMeasure similarity() { public DenseVectorFieldMapper.ElementType elementType() { return elementType != null ? elementType : DenseVectorFieldMapper.ElementType.FLOAT; } + + @Override + public String modelId() { + return model; + } } } diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java index b2f3b6f774a6f..fae11d5b53ca3 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java @@ -135,7 +135,7 @@ protected ServiceSettings getServiceSettingsFromMap(Map serviceS } } - public record TestServiceSettings(String model_id) implements ServiceSettings { + public record TestServiceSettings(String modelId) implements ServiceSettings { static final String NAME = "test_reranking_service_settings"; @@ -162,7 +162,7 @@ public TestServiceSettings(StreamInput in) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field("model_id", model_id); + builder.field("model_id", modelId); builder.endObject(); return builder; } @@ -179,14 +179,19 @@ public TransportVersion getMinimalSupportedVersion() { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeString(model_id); + out.writeString(modelId); + } + + @Override + public String modelId() { + return modelId; } @Override public ToXContentObject getFilteredXContentObject() { return (builder, params) -> { builder.startObject(); - builder.field("model_id", model_id); + builder.field("model_id", modelId); builder.endObject(); return builder; }; diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java index 7d5c21b78ee8a..fee9855b188c2 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java @@ -224,6 +224,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(shouldReturnHiddenField); } + @Override + public String modelId() { + return model; + } + @Override public ToXContentObject getFilteredXContentObject() { return (builder, params) -> { diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java index 776232f1e29e6..17037e56b2db3 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java @@ -341,6 +341,11 @@ public void writeTo(StreamOutput out) throws IOException { } + @Override + public String modelId() { + return null; + } + @Override public ToXContentObject getFilteredXContentObject() { return this; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java index f8ce9df1fb194..476ab3355a0b8 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java @@ -98,23 +98,7 @@ public static List getNamedWriteables() { // Default secret settings namedWriteables.add(new NamedWriteableRegistry.Entry(SecretSettings.class, DefaultSecretSettings.NAME, DefaultSecretSettings::new)); - addInternalElserNamedWriteables(namedWriteables); - - // Internal TextEmbedding service config - namedWriteables.add( - new NamedWriteableRegistry.Entry( - ServiceSettings.class, - ElasticsearchInternalServiceSettings.NAME, - ElasticsearchInternalServiceSettings::new - ) - ); - namedWriteables.add( - new NamedWriteableRegistry.Entry( - ServiceSettings.class, - MultilingualE5SmallInternalServiceSettings.NAME, - MultilingualE5SmallInternalServiceSettings::new - ) - ); + addInternalNamedWriteables(namedWriteables); addHuggingFaceNamedWriteables(namedWriteables); addOpenAiNamedWriteables(namedWriteables); @@ -374,13 +358,28 @@ private static void addGoogleVertexAiNamedWriteables(List namedWriteables) { + private static void addInternalNamedWriteables(List namedWriteables) { namedWriteables.add( new NamedWriteableRegistry.Entry(ServiceSettings.class, ElserInternalServiceSettings.NAME, ElserInternalServiceSettings::new) ); namedWriteables.add( new NamedWriteableRegistry.Entry(TaskSettings.class, ElserMlNodeTaskSettings.NAME, ElserMlNodeTaskSettings::new) ); + namedWriteables.add( + new NamedWriteableRegistry.Entry( + ServiceSettings.class, + ElasticsearchInternalServiceSettings.NAME, + ElasticsearchInternalServiceSettings::new + ) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry( + ServiceSettings.class, + MultilingualE5SmallInternalServiceSettings.NAME, + MultilingualE5SmallInternalServiceSettings::new + ) + ); + } private static void addChunkedInferenceResultsNamedWriteables(List namedWriteables) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index 1c388f7399260..fce2c54c535c9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -84,6 +84,8 @@ import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserService; import org.elasticsearch.xpack.inference.services.mistral.MistralService; import org.elasticsearch.xpack.inference.services.openai.OpenAiService; +import org.elasticsearch.xpack.inference.telemetry.InferenceAPMStats; +import org.elasticsearch.xpack.inference.telemetry.StatsMap; import java.util.ArrayList; import java.util.Collection; @@ -194,7 +196,10 @@ public Collection createComponents(PluginServices services) { var actionFilter = new ShardBulkInferenceActionFilter(registry, modelRegistry); shardBulkInferenceActionFilter.set(actionFilter); - return List.of(modelRegistry, registry, httpClientManager); + var statsFactory = new InferenceAPMStats.Factory(services.telemetryProvider().getMeterRegistry()); + var statsMap = new StatsMap<>(InferenceAPMStats::key, statsFactory::newInferenceRequestAPMCounter); + + return List.of(modelRegistry, registry, httpClientManager, statsMap); } @Override @@ -271,11 +276,11 @@ public List> getExecutorBuilders(Settings settingsToUse) { @Override public List> getSettings() { return Stream.of( - HttpSettings.getSettings(), - HttpClientManager.getSettings(), - ThrottlerManager.getSettings(), + HttpSettings.getSettingsDefinitions(), + HttpClientManager.getSettingsDefinitions(), + ThrottlerManager.getSettingsDefinitions(), RetrySettings.getSettingsDefinitions(), - Truncator.getSettings(), + Truncator.getSettingsDefinitions(), RequestExecutorServiceSettings.getSettingsDefinitions(), List.of(SKIP_VALIDATE_AND_START) ).flatMap(Collection::stream).collect(Collectors.toList()); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/Truncator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/Truncator.java index eabed7f6a7bd3..45ab9b160a8e6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/Truncator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/Truncator.java @@ -34,7 +34,7 @@ public class Truncator { Setting.Property.Dynamic ); - public static List> getSettings() { + public static List> getSettingsDefinitions() { return List.of(REDUCTION_PERCENTAGE_SETTING); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/ActionUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/ActionUtils.java index e4d6e39fdf1f2..27d1f1bd14e2c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/ActionUtils.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/ActionUtils.java @@ -24,13 +24,13 @@ public static ActionListener wrapFailuresInElasticsearc String errorMessage, ActionListener listener ) { - return ActionListener.wrap(listener::onResponse, e -> { + return listener.delegateResponse((l, e) -> { var unwrappedException = ExceptionsHelper.unwrapCause(e); if (unwrappedException instanceof ElasticsearchException esException) { - listener.onFailure(esException); + l.onFailure(esException); } else { - listener.onFailure(createInternalServerError(unwrappedException, errorMessage)); + l.onFailure(createInternalServerError(unwrappedException, errorMessage)); } }); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockEmbeddingsAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/SenderExecutableAction.java similarity index 59% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockEmbeddingsAction.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/SenderExecutableAction.java index 3f8be0c3cccbe..5451fcb467bd7 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockEmbeddingsAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/SenderExecutableAction.java @@ -5,44 +5,38 @@ * 2.0. */ -package org.elasticsearch.xpack.inference.external.action.amazonbedrock; +package org.elasticsearch.xpack.inference.external.action; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; import org.elasticsearch.xpack.inference.external.http.sender.RequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import java.util.Objects; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError; import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException; -public class AmazonBedrockEmbeddingsAction implements ExecutableAction { +public class SenderExecutableAction implements ExecutableAction { private final Sender sender; private final RequestManager requestManager; - private final String errorMessage; + private final String failedToSendRequestErrorMessage; - public AmazonBedrockEmbeddingsAction(Sender sender, RequestManager requestManager, String errorMessage) { + public SenderExecutableAction(Sender sender, RequestManager requestManager, String failedToSendRequestErrorMessage) { this.sender = Objects.requireNonNull(sender); this.requestManager = Objects.requireNonNull(requestManager); - this.errorMessage = Objects.requireNonNull(errorMessage); + this.failedToSendRequestErrorMessage = Objects.requireNonNull(failedToSendRequestErrorMessage); } @Override public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { + var wrappedListener = wrapFailuresInElasticsearchException(failedToSendRequestErrorMessage, listener); try { - ActionListener wrappedListener = wrapFailuresInElasticsearchException(errorMessage, listener); - sender.send(requestManager, inferenceInputs, timeout, wrappedListener); - } catch (ElasticsearchException e) { - listener.onFailure(e); } catch (Exception e) { - listener.onFailure(createInternalServerError(e, errorMessage)); + wrappedListener.onFailure(e); } } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/SingleInputSenderExecutableAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/SingleInputSenderExecutableAction.java new file mode 100644 index 0000000000000..4e97554b56445 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/SingleInputSenderExecutableAction.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.action; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; +import org.elasticsearch.xpack.inference.external.http.sender.RequestManager; +import org.elasticsearch.xpack.inference.external.http.sender.Sender; + +import java.util.Objects; + +public class SingleInputSenderExecutableAction extends SenderExecutableAction { + private final String requestTypeForInputValidationError; + + public SingleInputSenderExecutableAction( + Sender sender, + RequestManager requestManager, + String failedToSendRequestErrorMessage, + String requestTypeForInputValidationError + ) { + super(sender, requestManager, failedToSendRequestErrorMessage); + this.requestTypeForInputValidationError = Objects.requireNonNull(requestTypeForInputValidationError); + } + + @Override + public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { + if (inferenceInputs instanceof DocumentsOnlyInput == false) { + listener.onFailure(new ElasticsearchStatusException("Invalid inference input type", RestStatus.INTERNAL_SERVER_ERROR)); + return; + } + + var docsOnlyInput = (DocumentsOnlyInput) inferenceInputs; + if (docsOnlyInput.getInputs().size() > 1) { + listener.onFailure( + new ElasticsearchStatusException(requestTypeForInputValidationError + " only accepts 1 input", RestStatus.BAD_REQUEST) + ); + return; + } + + super.execute(inferenceInputs, timeout, listener); + } + +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockActionCreator.java index 5f9fc532e33b2..2715298c22d63 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockActionCreator.java @@ -10,6 +10,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.sender.AmazonBedrockChatCompletionRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.AmazonBedrockEmbeddingsRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; @@ -43,7 +44,7 @@ public ExecutableAction create(AmazonBedrockEmbeddingsModel embeddingsModel, Map timeout ); var errorMessage = constructFailedToSendRequestMessage(null, "Amazon Bedrock embeddings"); - return new AmazonBedrockEmbeddingsAction(sender, requestManager, errorMessage); + return new SenderExecutableAction(sender, requestManager, errorMessage); } @Override @@ -51,6 +52,6 @@ public ExecutableAction create(AmazonBedrockChatCompletionModel completionModel, var overriddenModel = AmazonBedrockChatCompletionModel.of(completionModel, taskSettings); var requestManager = new AmazonBedrockChatCompletionRequestManager(overriddenModel, serviceComponents.threadPool(), timeout); var errorMessage = constructFailedToSendRequestMessage(null, "Amazon Bedrock completion"); - return new AmazonBedrockChatCompletionAction(sender, requestManager, errorMessage); + return new SenderExecutableAction(sender, requestManager, errorMessage); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockChatCompletionAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockChatCompletionAction.java deleted file mode 100644 index 9d3c39d3ac4d9..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockChatCompletionAction.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.action.amazonbedrock; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; -import org.elasticsearch.xpack.inference.external.http.sender.RequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.Sender; - -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException; - -public class AmazonBedrockChatCompletionAction implements ExecutableAction { - private final Sender sender; - private final RequestManager requestManager; - private final String errorMessage; - - public AmazonBedrockChatCompletionAction(Sender sender, RequestManager requestManager, String errorMessage) { - this.sender = Objects.requireNonNull(sender); - this.requestManager = Objects.requireNonNull(requestManager); - this.errorMessage = Objects.requireNonNull(errorMessage); - } - - @Override - public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { - try { - ActionListener wrappedListener = wrapFailuresInElasticsearchException(errorMessage, listener); - - sender.send(requestManager, inferenceInputs, timeout, wrappedListener); - } catch (ElasticsearchException e) { - listener.onFailure(e); - } catch (Exception e) { - listener.onFailure(createInternalServerError(e, errorMessage)); - } - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicActionCreator.java index fa386c80643b0..aea6d065e09d1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicActionCreator.java @@ -8,6 +8,8 @@ package org.elasticsearch.xpack.inference.external.action.anthropic; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SingleInputSenderExecutableAction; +import org.elasticsearch.xpack.inference.external.http.sender.AnthropicCompletionRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.services.ServiceComponents; import org.elasticsearch.xpack.inference.services.anthropic.completion.AnthropicChatCompletionModel; @@ -15,10 +17,13 @@ import java.util.Map; import java.util.Objects; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; + /** * Provides a way to construct an {@link ExecutableAction} using the visitor pattern based on the anthropic model type. */ public class AnthropicActionCreator implements AnthropicActionVisitor { + private static final String ERROR_PREFIX = "Anthropic chat completions"; private final Sender sender; private final ServiceComponents serviceComponents; @@ -30,7 +35,8 @@ public AnthropicActionCreator(Sender sender, ServiceComponents serviceComponents @Override public ExecutableAction create(AnthropicChatCompletionModel model, Map taskSettings) { var overriddenModel = AnthropicChatCompletionModel.of(model, taskSettings); - - return new AnthropicChatCompletionAction(sender, overriddenModel, serviceComponents); + var requestCreator = AnthropicCompletionRequestManager.of(overriddenModel, serviceComponents.threadPool()); + var errorMessage = constructFailedToSendRequestMessage(overriddenModel.getUri(), ERROR_PREFIX); + return new SingleInputSenderExecutableAction(sender, requestCreator, errorMessage, ERROR_PREFIX); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicChatCompletionAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicChatCompletionAction.java deleted file mode 100644 index 9891d671764a4..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicChatCompletionAction.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.action.anthropic; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.AnthropicCompletionRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; -import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; -import org.elasticsearch.xpack.inference.external.http.sender.Sender; -import org.elasticsearch.xpack.inference.services.ServiceComponents; -import org.elasticsearch.xpack.inference.services.anthropic.completion.AnthropicChatCompletionModel; - -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException; - -public class AnthropicChatCompletionAction implements ExecutableAction { - - private final String errorMessage; - private final AnthropicCompletionRequestManager requestCreator; - - private final Sender sender; - - public AnthropicChatCompletionAction(Sender sender, AnthropicChatCompletionModel model, ServiceComponents serviceComponents) { - Objects.requireNonNull(serviceComponents); - Objects.requireNonNull(model); - this.sender = Objects.requireNonNull(sender); - this.requestCreator = AnthropicCompletionRequestManager.of(model, serviceComponents.threadPool()); - this.errorMessage = constructFailedToSendRequestMessage(model.getUri(), "Anthropic chat completions"); - } - - @Override - public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { - if (inferenceInputs instanceof DocumentsOnlyInput == false) { - listener.onFailure(new ElasticsearchStatusException("Invalid inference input type", RestStatus.INTERNAL_SERVER_ERROR)); - return; - } - - var docsOnlyInput = (DocumentsOnlyInput) inferenceInputs; - if (docsOnlyInput.getInputs().size() > 1) { - listener.onFailure(new ElasticsearchStatusException("Anthropic completions only accepts 1 input", RestStatus.BAD_REQUEST)); - return; - } - - try { - ActionListener wrappedListener = wrapFailuresInElasticsearchException(errorMessage, listener); - - sender.send(requestCreator, inferenceInputs, timeout, wrappedListener); - } catch (ElasticsearchException e) { - listener.onFailure(e); - } catch (Exception e) { - listener.onFailure(createInternalServerError(e, errorMessage)); - } - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioAction.java deleted file mode 100644 index 843084312b621..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioAction.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.action.azureaistudio; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.AzureAiStudioRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; -import org.elasticsearch.xpack.inference.external.http.sender.Sender; - -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException; - -public class AzureAiStudioAction implements ExecutableAction { - protected final Sender sender; - protected final AzureAiStudioRequestManager requestCreator; - protected final String errorMessage; - - protected AzureAiStudioAction(Sender sender, AzureAiStudioRequestManager requestCreator, String errorMessage) { - this.sender = sender; - this.requestCreator = requestCreator; - this.errorMessage = errorMessage; - } - - @Override - public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { - try { - ActionListener wrappedListener = wrapFailuresInElasticsearchException(errorMessage, listener); - - sender.send(requestCreator, inferenceInputs, timeout, wrappedListener); - } catch (ElasticsearchException e) { - listener.onFailure(e); - } catch (Exception e) { - listener.onFailure(createInternalServerError(e, errorMessage)); - } - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioActionCreator.java index 213ac22518922..6a80cee3afd57 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioActionCreator.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.inference.external.action.azureaistudio; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.sender.AzureAiStudioChatCompletionRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.AzureAiStudioEmbeddingsRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; @@ -34,7 +35,7 @@ public ExecutableAction create(AzureAiStudioChatCompletionModel completionModel, var overriddenModel = AzureAiStudioChatCompletionModel.of(completionModel, taskSettings); var requestManager = new AzureAiStudioChatCompletionRequestManager(overriddenModel, serviceComponents.threadPool()); var errorMessage = constructFailedToSendRequestMessage(completionModel.uri(), "Azure AI Studio completion"); - return new AzureAiStudioAction(sender, requestManager, errorMessage); + return new SenderExecutableAction(sender, requestManager, errorMessage); } @Override @@ -46,6 +47,6 @@ public ExecutableAction create(AzureAiStudioEmbeddingsModel embeddingsModel, Map serviceComponents.threadPool() ); var errorMessage = constructFailedToSendRequestMessage(embeddingsModel.uri(), "Azure AI Studio embeddings"); - return new AzureAiStudioAction(sender, requestManager, errorMessage); + return new SenderExecutableAction(sender, requestManager, errorMessage); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreator.java index 73ba286c9031a..1454b7c92ad91 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreator.java @@ -8,6 +8,10 @@ package org.elasticsearch.xpack.inference.external.action.azureopenai; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SingleInputSenderExecutableAction; +import org.elasticsearch.xpack.inference.external.http.sender.AzureOpenAiCompletionRequestManager; +import org.elasticsearch.xpack.inference.external.http.sender.AzureOpenAiEmbeddingsRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.services.ServiceComponents; import org.elasticsearch.xpack.inference.services.azureopenai.completion.AzureOpenAiCompletionModel; @@ -16,10 +20,13 @@ import java.util.Map; import java.util.Objects; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; + /** * Provides a way to construct an {@link ExecutableAction} using the visitor pattern based on the openai model type. */ public class AzureOpenAiActionCreator implements AzureOpenAiActionVisitor { + private static final String COMPLETION_ERROR_PREFIX = "Azure OpenAI completion"; private final Sender sender; private final ServiceComponents serviceComponents; @@ -31,12 +38,20 @@ public AzureOpenAiActionCreator(Sender sender, ServiceComponents serviceComponen @Override public ExecutableAction create(AzureOpenAiEmbeddingsModel model, Map taskSettings) { var overriddenModel = AzureOpenAiEmbeddingsModel.of(model, taskSettings); - return new AzureOpenAiEmbeddingsAction(sender, overriddenModel, serviceComponents); + var requestCreator = new AzureOpenAiEmbeddingsRequestManager( + overriddenModel, + serviceComponents.truncator(), + serviceComponents.threadPool() + ); + var errorMessage = constructFailedToSendRequestMessage(overriddenModel.getUri(), "Azure OpenAI embeddings"); + return new SenderExecutableAction(sender, requestCreator, errorMessage); } @Override public ExecutableAction create(AzureOpenAiCompletionModel model, Map taskSettings) { var overriddenModel = AzureOpenAiCompletionModel.of(model, taskSettings); - return new AzureOpenAiCompletionAction(sender, overriddenModel, serviceComponents); + var requestCreator = new AzureOpenAiCompletionRequestManager(overriddenModel, serviceComponents.threadPool()); + var errorMessage = constructFailedToSendRequestMessage(overriddenModel.getUri(), COMPLETION_ERROR_PREFIX); + return new SingleInputSenderExecutableAction(sender, requestCreator, errorMessage, COMPLETION_ERROR_PREFIX); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionAction.java deleted file mode 100644 index d38d02ef9620f..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionAction.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.action.azureopenai; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.AzureOpenAiCompletionRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; -import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; -import org.elasticsearch.xpack.inference.external.http.sender.Sender; -import org.elasticsearch.xpack.inference.services.ServiceComponents; -import org.elasticsearch.xpack.inference.services.azureopenai.completion.AzureOpenAiCompletionModel; - -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException; - -public class AzureOpenAiCompletionAction implements ExecutableAction { - - private final String errorMessage; - private final AzureOpenAiCompletionRequestManager requestCreator; - private final Sender sender; - - public AzureOpenAiCompletionAction(Sender sender, AzureOpenAiCompletionModel model, ServiceComponents serviceComponents) { - Objects.requireNonNull(serviceComponents); - Objects.requireNonNull(model); - this.sender = Objects.requireNonNull(sender); - this.requestCreator = new AzureOpenAiCompletionRequestManager(model, serviceComponents.threadPool()); - this.errorMessage = constructFailedToSendRequestMessage(model.getUri(), "Azure OpenAI completion"); - } - - @Override - public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { - if (inferenceInputs instanceof DocumentsOnlyInput == false) { - listener.onFailure(new ElasticsearchStatusException("Invalid inference input type", RestStatus.INTERNAL_SERVER_ERROR)); - return; - } - - var docsOnlyInput = (DocumentsOnlyInput) inferenceInputs; - if (docsOnlyInput.getInputs().size() > 1) { - listener.onFailure(new ElasticsearchStatusException("Azure OpenAI completion only accepts 1 input", RestStatus.BAD_REQUEST)); - return; - } - - try { - ActionListener wrappedListener = wrapFailuresInElasticsearchException(errorMessage, listener); - - sender.send(requestCreator, inferenceInputs, timeout, wrappedListener); - } catch (ElasticsearchException e) { - listener.onFailure(e); - } catch (Exception e) { - listener.onFailure(createInternalServerError(e, errorMessage)); - } - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsAction.java deleted file mode 100644 index 1b2226dd3f9f7..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsAction.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.action.azureopenai; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.AzureOpenAiEmbeddingsRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; -import org.elasticsearch.xpack.inference.external.http.sender.Sender; -import org.elasticsearch.xpack.inference.services.ServiceComponents; -import org.elasticsearch.xpack.inference.services.azureopenai.embeddings.AzureOpenAiEmbeddingsModel; - -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException; - -public class AzureOpenAiEmbeddingsAction implements ExecutableAction { - - private final String errorMessage; - private final AzureOpenAiEmbeddingsRequestManager requestCreator; - private final Sender sender; - - public AzureOpenAiEmbeddingsAction(Sender sender, AzureOpenAiEmbeddingsModel model, ServiceComponents serviceComponents) { - Objects.requireNonNull(serviceComponents); - Objects.requireNonNull(model); - this.sender = Objects.requireNonNull(sender); - requestCreator = new AzureOpenAiEmbeddingsRequestManager(model, serviceComponents.truncator(), serviceComponents.threadPool()); - errorMessage = constructFailedToSendRequestMessage(model.getUri(), "Azure OpenAI embeddings"); - } - - @Override - public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { - try { - ActionListener wrappedListener = wrapFailuresInElasticsearchException(errorMessage, listener); - - sender.send(requestCreator, inferenceInputs, timeout, wrappedListener); - } catch (ElasticsearchException e) { - listener.onFailure(e); - } catch (Exception e) { - listener.onFailure(createInternalServerError(e, errorMessage)); - } - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreator.java index 81bc90433d34a..9462ab1a361b4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreator.java @@ -9,6 +9,11 @@ import org.elasticsearch.inference.InputType; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SingleInputSenderExecutableAction; +import org.elasticsearch.xpack.inference.external.http.sender.CohereCompletionRequestManager; +import org.elasticsearch.xpack.inference.external.http.sender.CohereEmbeddingsRequestManager; +import org.elasticsearch.xpack.inference.external.http.sender.CohereRerankRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.services.ServiceComponents; import org.elasticsearch.xpack.inference.services.cohere.completion.CohereCompletionModel; @@ -18,10 +23,13 @@ import java.util.Map; import java.util.Objects; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; + /** * Provides a way to construct an {@link ExecutableAction} using the visitor pattern based on the cohere model type. */ public class CohereActionCreator implements CohereActionVisitor { + private static final String COMPLETION_ERROR_PREFIX = "Cohere completion"; private final Sender sender; private final ServiceComponents serviceComponents; @@ -34,20 +42,34 @@ public CohereActionCreator(Sender sender, ServiceComponents serviceComponents) { @Override public ExecutableAction create(CohereEmbeddingsModel model, Map taskSettings, InputType inputType) { var overriddenModel = CohereEmbeddingsModel.of(model, taskSettings, inputType); - - return new CohereEmbeddingsAction(sender, overriddenModel, serviceComponents.threadPool()); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( + overriddenModel.getServiceSettings().getCommonSettings().uri(), + "Cohere embeddings" + ); + // TODO - Batching pass the batching class on to the CohereEmbeddingsRequestManager + var requestCreator = CohereEmbeddingsRequestManager.of(overriddenModel, serviceComponents.threadPool()); + return new SenderExecutableAction(sender, requestCreator, failedToSendRequestErrorMessage); } @Override public ExecutableAction create(CohereRerankModel model, Map taskSettings) { var overriddenModel = CohereRerankModel.of(model, taskSettings); - - return new CohereRerankAction(sender, overriddenModel, serviceComponents.threadPool()); + var requestCreator = CohereRerankRequestManager.of(overriddenModel, serviceComponents.threadPool()); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( + overriddenModel.getServiceSettings().uri(), + "Cohere rerank" + ); + return new SenderExecutableAction(sender, requestCreator, failedToSendRequestErrorMessage); } @Override public ExecutableAction create(CohereCompletionModel model, Map taskSettings) { // no overridden model as task settings are always empty for cohere completion model - return new CohereCompletionAction(sender, model, serviceComponents.threadPool()); + var requestManager = CohereCompletionRequestManager.of(model, serviceComponents.threadPool()); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( + model.getServiceSettings().uri(), + COMPLETION_ERROR_PREFIX + ); + return new SingleInputSenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage, COMPLETION_ERROR_PREFIX); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionAction.java deleted file mode 100644 index 1df1019306699..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionAction.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.action.cohere; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.CohereCompletionRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; -import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; -import org.elasticsearch.xpack.inference.external.http.sender.Sender; -import org.elasticsearch.xpack.inference.services.cohere.completion.CohereCompletionModel; - -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException; - -public class CohereCompletionAction implements ExecutableAction { - - private final String failedToSendRequestErrorMessage; - - private final Sender sender; - - private final CohereCompletionRequestManager requestManager; - - public CohereCompletionAction(Sender sender, CohereCompletionModel model, ThreadPool threadPool) { - Objects.requireNonNull(model); - this.sender = Objects.requireNonNull(sender); - this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.getServiceSettings().uri(), "Cohere completion"); - this.requestManager = CohereCompletionRequestManager.of(model, threadPool); - } - - @Override - public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { - if (inferenceInputs instanceof DocumentsOnlyInput == false) { - listener.onFailure(new ElasticsearchStatusException("Invalid inference input type", RestStatus.INTERNAL_SERVER_ERROR)); - return; - } - - var docsOnlyInput = (DocumentsOnlyInput) inferenceInputs; - if (docsOnlyInput.getInputs().size() > 1) { - listener.onFailure(new ElasticsearchStatusException("Cohere completion only accepts 1 input", RestStatus.BAD_REQUEST)); - return; - } - - try { - ActionListener wrappedListener = wrapFailuresInElasticsearchException( - failedToSendRequestErrorMessage, - listener - ); - sender.send(requestManager, inferenceInputs, timeout, wrappedListener); - } catch (ElasticsearchException e) { - listener.onFailure(e); - } catch (Exception e) { - listener.onFailure(createInternalServerError(e, failedToSendRequestErrorMessage)); - } - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsAction.java deleted file mode 100644 index b4815f8f0d1bf..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsAction.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.action.cohere; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.CohereEmbeddingsRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; -import org.elasticsearch.xpack.inference.external.http.sender.Sender; -import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsModel; - -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException; - -public class CohereEmbeddingsAction implements ExecutableAction { - private final String failedToSendRequestErrorMessage; - private final Sender sender; - private final CohereEmbeddingsRequestManager requestCreator; - - public CohereEmbeddingsAction(Sender sender, CohereEmbeddingsModel model, ThreadPool threadPool) { - Objects.requireNonNull(model); - this.sender = Objects.requireNonNull(sender); - this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( - model.getServiceSettings().getCommonSettings().uri(), - "Cohere embeddings" - ); - // TODO - Batching pass the batching class on to the CohereEmbeddingsRequestManager - requestCreator = CohereEmbeddingsRequestManager.of(model, threadPool); - } - - @Override - public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { - try { - ActionListener wrappedListener = wrapFailuresInElasticsearchException( - failedToSendRequestErrorMessage, - listener - ); - sender.send(requestCreator, inferenceInputs, timeout, wrappedListener); - } catch (ElasticsearchException e) { - listener.onFailure(e); - } catch (Exception e) { - listener.onFailure(createInternalServerError(e, failedToSendRequestErrorMessage)); - } - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereRerankAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereRerankAction.java deleted file mode 100644 index 0613b8ef76453..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereRerankAction.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.action.cohere; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.CohereRerankRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; -import org.elasticsearch.xpack.inference.external.http.sender.Sender; -import org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankModel; - -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException; - -public class CohereRerankAction implements ExecutableAction { - private final String failedToSendRequestErrorMessage; - private final Sender sender; - private final CohereRerankRequestManager requestCreator; - - public CohereRerankAction(Sender sender, CohereRerankModel model, ThreadPool threadPool) { - Objects.requireNonNull(model); - this.sender = Objects.requireNonNull(sender); - this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( - model.getServiceSettings().getCommonSettings().uri(), - "Cohere rerank" - ); - requestCreator = CohereRerankRequestManager.of(model, threadPool); - } - - @Override - public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { - try { - ActionListener wrappedListener = wrapFailuresInElasticsearchException( - failedToSendRequestErrorMessage, - listener - ); - sender.send(requestCreator, inferenceInputs, timeout, wrappedListener); - } catch (ElasticsearchException e) { - listener.onFailure(e); - } catch (Exception e) { - listener.onFailure(createInternalServerError(e, failedToSendRequestErrorMessage)); - } - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionCreator.java index 86154faefabc5..3871b5fb98882 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionCreator.java @@ -8,6 +8,10 @@ package org.elasticsearch.xpack.inference.external.action.googleaistudio; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SingleInputSenderExecutableAction; +import org.elasticsearch.xpack.inference.external.http.sender.GoogleAiStudioCompletionRequestManager; +import org.elasticsearch.xpack.inference.external.http.sender.GoogleAiStudioEmbeddingsRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.services.ServiceComponents; import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModel; @@ -16,8 +20,11 @@ import java.util.Map; import java.util.Objects; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; + public class GoogleAiStudioActionCreator implements GoogleAiStudioActionVisitor { + private static final String COMPLETION_ERROR_MESSAGE = "Google AI Studio completion"; private final Sender sender; private final ServiceComponents serviceComponents; @@ -30,11 +37,19 @@ public GoogleAiStudioActionCreator(Sender sender, ServiceComponents serviceCompo @Override public ExecutableAction create(GoogleAiStudioCompletionModel model, Map taskSettings) { // no overridden model as task settings are always empty for Google AI Studio completion model - return new GoogleAiStudioCompletionAction(sender, model, serviceComponents.threadPool()); + var requestManager = new GoogleAiStudioCompletionRequestManager(model, serviceComponents.threadPool()); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), COMPLETION_ERROR_MESSAGE); + return new SingleInputSenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage, COMPLETION_ERROR_MESSAGE); } @Override public ExecutableAction create(GoogleAiStudioEmbeddingsModel model, Map taskSettings) { - return new GoogleAiStudioEmbeddingsAction(sender, model, serviceComponents); + var requestManager = new GoogleAiStudioEmbeddingsRequestManager( + model, + serviceComponents.truncator(), + serviceComponents.threadPool() + ); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google AI Studio embeddings"); + return new SenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionAction.java deleted file mode 100644 index 7f918ae9a7db7..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionAction.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.action.googleaistudio; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; -import org.elasticsearch.xpack.inference.external.http.sender.GoogleAiStudioCompletionRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; -import org.elasticsearch.xpack.inference.external.http.sender.Sender; -import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModel; - -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException; - -public class GoogleAiStudioCompletionAction implements ExecutableAction { - - private final String failedToSendRequestErrorMessage; - - private final GoogleAiStudioCompletionRequestManager requestManager; - - private final Sender sender; - - public GoogleAiStudioCompletionAction(Sender sender, GoogleAiStudioCompletionModel model, ThreadPool threadPool) { - Objects.requireNonNull(threadPool); - Objects.requireNonNull(model); - this.sender = Objects.requireNonNull(sender); - this.requestManager = new GoogleAiStudioCompletionRequestManager(model, threadPool); - this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google AI Studio completion"); - } - - @Override - public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { - if (inferenceInputs instanceof DocumentsOnlyInput == false) { - listener.onFailure(new ElasticsearchStatusException("Invalid inference input type", RestStatus.INTERNAL_SERVER_ERROR)); - return; - } - - var docsOnlyInput = (DocumentsOnlyInput) inferenceInputs; - if (docsOnlyInput.getInputs().size() > 1) { - listener.onFailure( - new ElasticsearchStatusException("Google AI Studio completion only accepts 1 input", RestStatus.BAD_REQUEST) - ); - return; - } - - try { - ActionListener wrappedListener = wrapFailuresInElasticsearchException( - failedToSendRequestErrorMessage, - listener - ); - sender.send(requestManager, inferenceInputs, timeout, wrappedListener); - } catch (ElasticsearchException e) { - listener.onFailure(e); - } catch (Exception e) { - listener.onFailure(createInternalServerError(e, failedToSendRequestErrorMessage)); - } - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsAction.java deleted file mode 100644 index 5ce780193c789..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsAction.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.action.googleaistudio; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.GoogleAiStudioEmbeddingsRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; -import org.elasticsearch.xpack.inference.external.http.sender.Sender; -import org.elasticsearch.xpack.inference.services.ServiceComponents; -import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsModel; - -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException; - -public class GoogleAiStudioEmbeddingsAction implements ExecutableAction { - - private final String failedToSendRequestErrorMessage; - - private final GoogleAiStudioEmbeddingsRequestManager requestManager; - - private final Sender sender; - - public GoogleAiStudioEmbeddingsAction(Sender sender, GoogleAiStudioEmbeddingsModel model, ServiceComponents serviceComponents) { - Objects.requireNonNull(serviceComponents); - Objects.requireNonNull(model); - this.sender = Objects.requireNonNull(sender); - this.requestManager = new GoogleAiStudioEmbeddingsRequestManager( - model, - serviceComponents.truncator(), - serviceComponents.threadPool() - ); - this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google AI Studio embeddings"); - } - - @Override - public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { - try { - ActionListener wrappedListener = wrapFailuresInElasticsearchException( - failedToSendRequestErrorMessage, - listener - ); - - sender.send(requestManager, inferenceInputs, timeout, wrappedListener); - } catch (ElasticsearchException e) { - listener.onFailure(e); - } catch (Exception e) { - listener.onFailure(createInternalServerError(e, failedToSendRequestErrorMessage)); - } - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiActionCreator.java index ed2a205151a4c..27b3ae95f1aa4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiActionCreator.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.inference.external.action.googlevertexai; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; +import org.elasticsearch.xpack.inference.external.http.sender.GoogleVertexAiEmbeddingsRequestManager; +import org.elasticsearch.xpack.inference.external.http.sender.GoogleVertexAiRerankRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.services.ServiceComponents; import org.elasticsearch.xpack.inference.services.googlevertexai.embeddings.GoogleVertexAiEmbeddingsModel; @@ -16,6 +19,8 @@ import java.util.Map; import java.util.Objects; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; + public class GoogleVertexAiActionCreator implements GoogleVertexAiActionVisitor { private final Sender sender; @@ -29,11 +34,19 @@ public GoogleVertexAiActionCreator(Sender sender, ServiceComponents serviceCompo @Override public ExecutableAction create(GoogleVertexAiEmbeddingsModel model, Map taskSettings) { - return new GoogleVertexAiEmbeddingsAction(sender, model, serviceComponents); + var requestManager = new GoogleVertexAiEmbeddingsRequestManager( + model, + serviceComponents.truncator(), + serviceComponents.threadPool() + ); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google Vertex AI embeddings"); + return new SenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage); } @Override public ExecutableAction create(GoogleVertexAiRerankModel model, Map taskSettings) { - return new GoogleVertexAiRerankAction(sender, model, serviceComponents.threadPool()); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google Vertex AI rerank"); + var requestManager = GoogleVertexAiRerankRequestManager.of(model, serviceComponents.threadPool()); + return new SenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiEmbeddingsAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiEmbeddingsAction.java deleted file mode 100644 index f9814224c101a..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiEmbeddingsAction.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.action.googlevertexai; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.GoogleVertexAiEmbeddingsRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; -import org.elasticsearch.xpack.inference.external.http.sender.Sender; -import org.elasticsearch.xpack.inference.services.ServiceComponents; -import org.elasticsearch.xpack.inference.services.googlevertexai.embeddings.GoogleVertexAiEmbeddingsModel; - -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException; - -public class GoogleVertexAiEmbeddingsAction implements ExecutableAction { - - private final String failedToSendRequestErrorMessage; - - private final GoogleVertexAiEmbeddingsRequestManager requestManager; - - private final Sender sender; - - public GoogleVertexAiEmbeddingsAction(Sender sender, GoogleVertexAiEmbeddingsModel model, ServiceComponents serviceComponents) { - Objects.requireNonNull(serviceComponents); - Objects.requireNonNull(model); - this.sender = Objects.requireNonNull(sender); - this.requestManager = new GoogleVertexAiEmbeddingsRequestManager( - model, - serviceComponents.truncator(), - serviceComponents.threadPool() - ); - this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google Vertex AI embeddings"); - } - - @Override - public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { - try { - ActionListener wrappedListener = wrapFailuresInElasticsearchException( - failedToSendRequestErrorMessage, - listener - ); - - sender.send(requestManager, inferenceInputs, timeout, wrappedListener); - } catch (ElasticsearchException e) { - listener.onFailure(e); - } catch (Exception e) { - listener.onFailure(createInternalServerError(e, failedToSendRequestErrorMessage)); - } - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiRerankAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiRerankAction.java deleted file mode 100644 index 2827de3b1962d..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiRerankAction.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.action.googlevertexai; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.GoogleVertexAiRerankRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; -import org.elasticsearch.xpack.inference.external.http.sender.Sender; -import org.elasticsearch.xpack.inference.services.googlevertexai.rerank.GoogleVertexAiRerankModel; - -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException; - -public class GoogleVertexAiRerankAction implements ExecutableAction { - - private final String failedToSendRequestErrorMessage; - - private final Sender sender; - - private final GoogleVertexAiRerankRequestManager requestManager; - - public GoogleVertexAiRerankAction(Sender sender, GoogleVertexAiRerankModel model, ThreadPool threadPool) { - Objects.requireNonNull(model); - this.sender = Objects.requireNonNull(sender); - this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google Vertex AI rerank"); - this.requestManager = GoogleVertexAiRerankRequestManager.of(model, threadPool); - } - - @Override - public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { - try { - ActionListener wrappedListener = wrapFailuresInElasticsearchException( - failedToSendRequestErrorMessage, - listener - ); - sender.send(requestManager, inferenceInputs, timeout, wrappedListener); - } catch (ElasticsearchException e) { - listener.onFailure(e); - } catch (Exception e) { - listener.onFailure(createInternalServerError(e, failedToSendRequestErrorMessage)); - } - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceAction.java deleted file mode 100644 index 1e5f01f801e17..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceAction.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.action.huggingface; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler; -import org.elasticsearch.xpack.inference.external.http.sender.HuggingFaceRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; -import org.elasticsearch.xpack.inference.external.http.sender.Sender; -import org.elasticsearch.xpack.inference.services.ServiceComponents; -import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceModel; - -import java.util.Objects; - -import static org.elasticsearch.core.Strings.format; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException; - -public class HuggingFaceAction implements ExecutableAction { - private final String errorMessage; - private final Sender sender; - private final HuggingFaceRequestManager requestCreator; - - public HuggingFaceAction( - Sender sender, - HuggingFaceModel model, - ServiceComponents serviceComponents, - ResponseHandler responseHandler, - String requestType - ) { - Objects.requireNonNull(serviceComponents); - Objects.requireNonNull(requestType); - this.sender = Objects.requireNonNull(sender); - requestCreator = HuggingFaceRequestManager.of( - model, - responseHandler, - serviceComponents.truncator(), - serviceComponents.threadPool() - ); - errorMessage = format( - "Failed to send Hugging Face %s request from inference entity id [%s]", - requestType, - model.getInferenceEntityId() - ); - } - - @Override - public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { - try { - ActionListener wrappedListener = wrapFailuresInElasticsearchException(errorMessage, listener); - sender.send(requestCreator, inferenceInputs, timeout, wrappedListener); - } catch (ElasticsearchException e) { - listener.onFailure(e); - } catch (Exception e) { - listener.onFailure(createInternalServerError(e, errorMessage)); - } - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceActionCreator.java index ba46519814b04..f023753d9a73b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceActionCreator.java @@ -8,6 +8,8 @@ package org.elasticsearch.xpack.inference.external.action.huggingface; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; +import org.elasticsearch.xpack.inference.external.http.sender.HuggingFaceRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.external.huggingface.HuggingFaceResponseHandler; import org.elasticsearch.xpack.inference.external.response.huggingface.HuggingFaceElserResponseEntity; @@ -18,6 +20,8 @@ import java.util.Objects; +import static org.elasticsearch.core.Strings.format; + /** * Provides a way to construct an {@link ExecutableAction} using the visitor pattern based on the hugging face model type. */ @@ -36,14 +40,34 @@ public ExecutableAction create(HuggingFaceEmbeddingsModel model) { "hugging face text embeddings", HuggingFaceEmbeddingsResponseEntity::fromResponse ); - - return new HuggingFaceAction(sender, model, serviceComponents, responseHandler, "text embeddings"); + var requestCreator = HuggingFaceRequestManager.of( + model, + responseHandler, + serviceComponents.truncator(), + serviceComponents.threadPool() + ); + var errorMessage = format( + "Failed to send Hugging Face %s request from inference entity id [%s]", + "text embeddings", + model.getInferenceEntityId() + ); + return new SenderExecutableAction(sender, requestCreator, errorMessage); } @Override public ExecutableAction create(HuggingFaceElserModel model) { var responseHandler = new HuggingFaceResponseHandler("hugging face elser", HuggingFaceElserResponseEntity::fromResponse); - - return new HuggingFaceAction(sender, model, serviceComponents, responseHandler, "ELSER"); + var requestCreator = HuggingFaceRequestManager.of( + model, + responseHandler, + serviceComponents.truncator(), + serviceComponents.threadPool() + ); + var errorMessage = format( + "Failed to send Hugging Face %s request from inference entity id [%s]", + "ELSER", + model.getInferenceEntityId() + ); + return new SenderExecutableAction(sender, requestCreator, errorMessage); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralAction.java deleted file mode 100644 index f7b51e80a04b3..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralAction.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.action.mistral; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; -import org.elasticsearch.xpack.inference.external.http.sender.MistralEmbeddingsRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.Sender; - -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException; - -public class MistralAction implements ExecutableAction { - protected final Sender sender; - protected final MistralEmbeddingsRequestManager requestCreator; - protected final String errorMessage; - - protected MistralAction(Sender sender, MistralEmbeddingsRequestManager requestCreator, String errorMessage) { - this.sender = sender; - this.requestCreator = requestCreator; - this.errorMessage = errorMessage; - } - - @Override - public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { - try { - ActionListener wrappedListener = wrapFailuresInElasticsearchException(errorMessage, listener); - - sender.send(requestCreator, inferenceInputs, timeout, wrappedListener); - } catch (ElasticsearchException e) { - listener.onFailure(e); - } catch (Exception e) { - listener.onFailure(createInternalServerError(e, errorMessage)); - } - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralActionCreator.java index a023973ea6aa5..21a80ee9d21fa 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralActionCreator.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.inference.external.action.mistral; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.sender.MistralEmbeddingsRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.services.ServiceComponents; @@ -35,6 +36,6 @@ public ExecutableAction create(MistralEmbeddingsModel embeddingsModel, Map taskSettings) { var overriddenModel = OpenAiEmbeddingsModel.of(model, taskSettings); - - return new OpenAiEmbeddingsAction(sender, overriddenModel, serviceComponents); + var requestCreator = OpenAiEmbeddingsRequestManager.of( + overriddenModel, + serviceComponents.truncator(), + serviceComponents.threadPool() + ); + var errorMessage = constructFailedToSendRequestMessage(overriddenModel.getServiceSettings().uri(), "OpenAI embeddings"); + return new SenderExecutableAction(sender, requestCreator, errorMessage); } @Override public ExecutableAction create(OpenAiChatCompletionModel model, Map taskSettings) { var overriddenModel = OpenAiChatCompletionModel.of(model, taskSettings); - - return new OpenAiChatCompletionAction(sender, overriddenModel, serviceComponents); + var requestCreator = OpenAiCompletionRequestManager.of(overriddenModel, serviceComponents.threadPool()); + var errorMessage = constructFailedToSendRequestMessage(overriddenModel.getServiceSettings().uri(), COMPLETION_ERROR_PREFIX); + return new SingleInputSenderExecutableAction(sender, requestCreator, errorMessage, COMPLETION_ERROR_PREFIX); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionAction.java deleted file mode 100644 index e11e9d5ad8cc9..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionAction.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.action.openai; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; -import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; -import org.elasticsearch.xpack.inference.external.http.sender.OpenAiCompletionRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.Sender; -import org.elasticsearch.xpack.inference.services.ServiceComponents; -import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModel; - -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException; - -public class OpenAiChatCompletionAction implements ExecutableAction { - - private final String errorMessage; - private final OpenAiCompletionRequestManager requestCreator; - - private final Sender sender; - - public OpenAiChatCompletionAction(Sender sender, OpenAiChatCompletionModel model, ServiceComponents serviceComponents) { - Objects.requireNonNull(serviceComponents); - Objects.requireNonNull(model); - this.sender = Objects.requireNonNull(sender); - this.requestCreator = OpenAiCompletionRequestManager.of(model, serviceComponents.threadPool()); - this.errorMessage = constructFailedToSendRequestMessage(model.getServiceSettings().uri(), "OpenAI chat completions"); - } - - @Override - public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { - if (inferenceInputs instanceof DocumentsOnlyInput == false) { - listener.onFailure(new ElasticsearchStatusException("Invalid inference input type", RestStatus.INTERNAL_SERVER_ERROR)); - return; - } - - var docsOnlyInput = (DocumentsOnlyInput) inferenceInputs; - if (docsOnlyInput.getInputs().size() > 1) { - listener.onFailure(new ElasticsearchStatusException("OpenAI completions only accepts 1 input", RestStatus.BAD_REQUEST)); - return; - } - - try { - ActionListener wrappedListener = wrapFailuresInElasticsearchException(errorMessage, listener); - - sender.send(requestCreator, inferenceInputs, timeout, wrappedListener); - } catch (ElasticsearchException e) { - listener.onFailure(e); - } catch (Exception e) { - listener.onFailure(createInternalServerError(e, errorMessage)); - } - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsAction.java deleted file mode 100644 index 3e92d206b4257..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsAction.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.action.openai; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; -import org.elasticsearch.xpack.inference.external.http.sender.OpenAiEmbeddingsRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.Sender; -import org.elasticsearch.xpack.inference.services.ServiceComponents; -import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsModel; - -import java.util.Objects; - -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError; -import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException; - -public class OpenAiEmbeddingsAction implements ExecutableAction { - - private final String errorMessage; - private final OpenAiEmbeddingsRequestManager requestCreator; - private final Sender sender; - - public OpenAiEmbeddingsAction(Sender sender, OpenAiEmbeddingsModel model, ServiceComponents serviceComponents) { - Objects.requireNonNull(serviceComponents); - Objects.requireNonNull(model); - this.sender = Objects.requireNonNull(sender); - requestCreator = OpenAiEmbeddingsRequestManager.of(model, serviceComponents.truncator(), serviceComponents.threadPool()); - errorMessage = constructFailedToSendRequestMessage(model.getServiceSettings().uri(), "OpenAI embeddings"); - } - - @Override - public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { - try { - ActionListener wrappedListener = wrapFailuresInElasticsearchException(errorMessage, listener); - - sender.send(requestCreator, inferenceInputs, timeout, wrappedListener); - } catch (ElasticsearchException e) { - listener.onFailure(e); - } catch (Exception e) { - listener.onFailure(createInternalServerError(e, errorMessage)); - } - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClientManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClientManager.java index 8be3b76f68c54..e5d76b9bb5570 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClientManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClientManager.java @@ -153,7 +153,7 @@ private IdleConnectionEvictor createConnectionEvictor() { return new IdleConnectionEvictor(threadPool, connectionManager, evictionInterval, connectionMaxIdle); } - public static List> getSettings() { + public static List> getSettingsDefinitions() { return List.of( MAX_TOTAL_CONNECTIONS, MAX_ROUTE_CONNECTIONS, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java index 642b76d775173..b2825d1b79cbf 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java @@ -45,7 +45,7 @@ private void setMaxResponseSize(ByteSizeValue maxResponseSize) { this.maxResponseSize = maxResponseSize; } - public static List> getSettings() { + public static List> getSettingsDefinitions() { return List.of(MAX_HTTP_RESPONSE_SIZE); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereRerankRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereRerankRequest.java index 492807f74b32a..4ec04c0187329 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereRerankRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereRerankRequest.java @@ -39,7 +39,7 @@ public CohereRerankRequest(String query, List input, CohereRerankModel m this.input = Objects.requireNonNull(input); this.query = Objects.requireNonNull(query); taskSettings = model.getTaskSettings(); - this.model = model.getServiceSettings().getCommonSettings().modelId(); + this.model = model.getServiceSettings().modelId(); inferenceEntityId = model.getInferenceEntityId(); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/logging/ThrottlerManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/logging/ThrottlerManager.java index 2a84494d6af21..d333cc92d61de 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/logging/ThrottlerManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/logging/ThrottlerManager.java @@ -102,7 +102,7 @@ public void close() { throttler.close(); } - public static List> getSettings() { + public static List> getSettingsDefinitions() { return List.of(STATS_RESET_INTERVAL_SETTING, LOGGER_WAIT_DURATION_SETTING); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java index 9f810b829bea9..7e46dcfea7592 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java @@ -9,6 +9,7 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.core.Nullable; @@ -131,18 +132,31 @@ public static Object removeAsOneOfTypes( return null; } - public static AdaptiveAllocationsSettings removeAsAdaptiveAllocationsSettings(Map sourceMap, String key) { + public static AdaptiveAllocationsSettings removeAsAdaptiveAllocationsSettings( + Map sourceMap, + String key, + ValidationException validationException + ) { if (AdaptiveAllocationsFeatureFlag.isEnabled() == false) { return null; } Map settingsMap = ServiceUtils.removeFromMap(sourceMap, key); - return settingsMap == null - ? null - : new AdaptiveAllocationsSettings( - ServiceUtils.removeAsType(settingsMap, ENABLED.getPreferredName(), Boolean.class), - ServiceUtils.removeAsType(settingsMap, MIN_NUMBER_OF_ALLOCATIONS.getPreferredName(), Integer.class), - ServiceUtils.removeAsType(settingsMap, MAX_NUMBER_OF_ALLOCATIONS.getPreferredName(), Integer.class) - ); + if (settingsMap == null) { + return null; + } + AdaptiveAllocationsSettings settings = new AdaptiveAllocationsSettings( + ServiceUtils.removeAsType(settingsMap, ENABLED.getPreferredName(), Boolean.class, validationException), + ServiceUtils.removeAsType(settingsMap, MIN_NUMBER_OF_ALLOCATIONS.getPreferredName(), Integer.class, validationException), + ServiceUtils.removeAsType(settingsMap, MAX_NUMBER_OF_ALLOCATIONS.getPreferredName(), Integer.class, validationException) + ); + for (String settingName : settingsMap.keySet()) { + validationException.addValidationError(invalidSettingError(settingName, key)); + } + ActionRequestValidationException exception = settings.validate(); + if (exception != null) { + validationException.addValidationErrors(exception.validationErrors()); + } + return settings; } @SuppressWarnings("unchecked") @@ -196,6 +210,10 @@ public static String missingSettingErrorMsg(String settingName, String scope) { return Strings.format("[%s] does not contain the required setting [%s]", scope, settingName); } + public static String missingOneOfSettingsErrorMsg(List settingNames, String scope) { + return Strings.format("[%s] does not contain one of the required settings [%s]", scope, String.join(", ", settingNames)); + } + public static String invalidTypeErrorMsg(String settingName, Object foundObject, String expectedType) { return Strings.format( "field [%s] is not of the expected type. The value [%s] cannot be converted to a [%s]", @@ -411,9 +429,6 @@ public static Integer extractOptionalPositiveInteger( if (optionalField != null && optionalField <= 0) { validationException.addValidationError(ServiceUtils.mustBeAPositiveIntegerErrorMessage(settingName, scope, optionalField)); - } - - if (validationException.validationErrors().size() > initialValidationErrorCount) { return null; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockModel.java index 13ca8bd7bd749..90dfce3b7a76f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockModel.java @@ -68,7 +68,7 @@ public RateLimitSettings rateLimitSettings() { private void setPropertiesFromServiceSettings(AmazonBedrockServiceSettings serviceSettings) { this.region = serviceSettings.region(); - this.model = serviceSettings.model(); + this.model = serviceSettings.modelId(); this.provider = serviceSettings.provider(); this.rateLimitSettings = serviceSettings.rateLimitSettings(); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java index 459ca367058f8..d12929eecb88e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java @@ -287,7 +287,7 @@ private AmazonBedrockEmbeddingsModel updateModelWithEmbeddingDetails(AmazonBedro AmazonBedrockEmbeddingsServiceSettings settingsToUse = new AmazonBedrockEmbeddingsServiceSettings( serviceSettings.region(), - serviceSettings.model(), + serviceSettings.modelId(), serviceSettings.provider(), embeddingSize, serviceSettings.dimensionsSetByUser(), diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceSettings.java index 13c7c0a8c5938..b572df4f1ee05 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceSettings.java @@ -108,7 +108,8 @@ public String region() { return region; } - public String model() { + @Override + public String modelId() { return model; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionServiceSettings.java index 3a70a26a82387..7976407b6fb67 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionServiceSettings.java @@ -113,7 +113,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_COMPLETION_INFERENCE_SERVICE_ADDED; + return TransportVersions.V_8_14_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceSettings.java index 03034ae70c2b6..b7776c8b63071 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceSettings.java @@ -114,6 +114,11 @@ public RateLimitSettings rateLimitSettings() { return this.rateLimitSettings; } + @Override + public String modelId() { + return null; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(target); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettings.java index fc11d96269b68..49684d6749756 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettings.java @@ -141,7 +141,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_AZURE_OPENAI_EMBEDDINGS; + return TransportVersions.V_8_14_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettings.java index dc001993b366f..f5d70b89cee50 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettings.java @@ -76,7 +76,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_AZURE_OPENAI_EMBEDDINGS; + return TransportVersions.V_8_14_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java index 48e45f368bfe2..06217e8079b06 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java @@ -104,7 +104,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_AZURE_OPENAI_EMBEDDINGS; + return TransportVersions.V_8_14_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionServiceSettings.java index 92dc461d9008c..16f58574a8068 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionServiceSettings.java @@ -127,6 +127,11 @@ public String deploymentId() { return deploymentId; } + @Override + public String modelId() { + return null; + } + @Override public RateLimitSettings rateLimitSettings() { return rateLimitSettings; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java index a9e40569d4e7a..941a4bdeeb41a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java @@ -247,6 +247,11 @@ public DenseVectorFieldMapper.ElementType elementType() { return DenseVectorFieldMapper.ElementType.FLOAT; } + @Override + public String modelId() { + return null; + } + @Override public String getWriteableName() { return NAME; @@ -285,7 +290,7 @@ protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder buil @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_AZURE_OPENAI_EMBEDDINGS; + return TransportVersions.V_8_14_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettings.java index 49329a55a18ef..36cf89ea2d846 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettings.java @@ -91,7 +91,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_AZURE_OPENAI_EMBEDDINGS; + return TransportVersions.V_8_14_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettings.java index d477a8c5a5f55..7f28459e9b8be 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettings.java @@ -46,7 +46,7 @@ public class CohereServiceSettings extends FilteredXContentObject implements Ser private static final Logger logger = LogManager.getLogger(CohereServiceSettings.class); // Production key rate limits for all endpoints: https://docs.cohere.com/docs/going-live#production-key-specifications // 10K requests a minute - private static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(10_000); + public static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(10_000); public static CohereServiceSettings fromMap(Map map, ConfigurationParseContext context) { ValidationException validationException = new ValidationException(); @@ -159,6 +159,7 @@ public Integer maxInputTokens() { return maxInputTokens; } + @Override public String modelId() { return modelId; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingType.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingType.java index 8dbbbf7011e86..11e405df3cde9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingType.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingType.java @@ -112,7 +112,7 @@ public DenseVectorFieldMapper.ElementType toElementType() { * @return the embedding type that is known to the version passed in */ public static CohereEmbeddingType translateToVersion(CohereEmbeddingType embeddingType, TransportVersion version) { - if (version.before(TransportVersions.ML_INFERENCE_EMBEDDING_BYTE_ADDED) && embeddingType == BYTE) { + if (version.before(TransportVersions.V_8_14_0) && embeddingType == BYTE) { return INT8; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsServiceSettings.java index 685dac0f3877c..b25b9fc8fd351 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsServiceSettings.java @@ -134,6 +134,11 @@ public Integer dimensions() { return commonSettings.dimensions(); } + @Override + public String modelId() { + return commonSettings.modelId(); + } + public CohereEmbeddingType getEmbeddingType() { return embeddingType; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankModel.java index cc8116a70bcc8..b84b98973bbe5 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankModel.java @@ -59,7 +59,7 @@ public CohereRerankModel( new ModelConfigurations(modelId, taskType, service, serviceSettings, taskSettings), new ModelSecrets(secretSettings), secretSettings, - serviceSettings.getCommonSettings() + serviceSettings ); } @@ -100,6 +100,6 @@ public ExecutableAction accept(CohereActionVisitor visitor, Map @Override public URI uri() { - return getServiceSettings().getCommonSettings().uri(); + return getServiceSettings().uri(); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java index 6a74fe533e3db..1132dff34ed6e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java @@ -7,43 +7,119 @@ package org.elasticsearch.xpack.inference.services.cohere.rerank; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ServiceSettings; +import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; -import org.elasticsearch.xpack.inference.services.cohere.CohereServiceSettings; +import org.elasticsearch.xpack.inference.services.cohere.CohereRateLimitServiceSettings; +import org.elasticsearch.xpack.inference.services.cohere.CohereService; import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; import java.io.IOException; +import java.net.URI; import java.util.Map; import java.util.Objects; -public class CohereRerankServiceSettings extends FilteredXContentObject implements ServiceSettings { +import static org.elasticsearch.xpack.inference.services.ServiceFields.DIMENSIONS; +import static org.elasticsearch.xpack.inference.services.ServiceFields.MAX_INPUT_TOKENS; +import static org.elasticsearch.xpack.inference.services.ServiceFields.MODEL_ID; +import static org.elasticsearch.xpack.inference.services.ServiceFields.URL; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.convertToUri; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.createOptionalUri; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalString; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractSimilarity; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeAsType; +import static org.elasticsearch.xpack.inference.services.cohere.CohereServiceSettings.DEFAULT_RATE_LIMIT_SETTINGS; + +public class CohereRerankServiceSettings extends FilteredXContentObject implements ServiceSettings, CohereRateLimitServiceSettings { public static final String NAME = "cohere_rerank_service_settings"; - public static CohereRerankServiceSettings fromMap(Map map, ConfigurationParseContext parseContext) { + private static final Logger logger = LogManager.getLogger(CohereRerankServiceSettings.class); + + public static CohereRerankServiceSettings fromMap(Map map, ConfigurationParseContext context) { ValidationException validationException = new ValidationException(); - var commonServiceSettings = CohereServiceSettings.fromMap(map, parseContext); + + String url = extractOptionalString(map, URL, ModelConfigurations.SERVICE_SETTINGS, validationException); + + // We need to extract/remove those fields to avoid unknown service settings errors + extractSimilarity(map, ModelConfigurations.SERVICE_SETTINGS, validationException); + removeAsType(map, DIMENSIONS, Integer.class); + removeAsType(map, MAX_INPUT_TOKENS, Integer.class); + + URI uri = convertToUri(url, URL, ModelConfigurations.SERVICE_SETTINGS, validationException); + String modelId = extractOptionalString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException); + RateLimitSettings rateLimitSettings = RateLimitSettings.of( + map, + DEFAULT_RATE_LIMIT_SETTINGS, + validationException, + CohereService.NAME, + context + ); if (validationException.validationErrors().isEmpty() == false) { throw validationException; } - return new CohereRerankServiceSettings(commonServiceSettings); + return new CohereRerankServiceSettings(uri, modelId, rateLimitSettings); } - private final CohereServiceSettings commonSettings; + private final URI uri; + + private final String modelId; + + private final RateLimitSettings rateLimitSettings; + + public CohereRerankServiceSettings(@Nullable URI uri, @Nullable String modelId, @Nullable RateLimitSettings rateLimitSettings) { + this.uri = uri; + this.modelId = modelId; + this.rateLimitSettings = Objects.requireNonNullElse(rateLimitSettings, DEFAULT_RATE_LIMIT_SETTINGS); + } - public CohereRerankServiceSettings(CohereServiceSettings commonSettings) { - this.commonSettings = commonSettings; + public CohereRerankServiceSettings(@Nullable String url, @Nullable String modelId, @Nullable RateLimitSettings rateLimitSettings) { + this(createOptionalUri(url), modelId, rateLimitSettings); } public CohereRerankServiceSettings(StreamInput in) throws IOException { - commonSettings = new CohereServiceSettings(in); + this.uri = createOptionalUri(in.readOptionalString()); + + if (in.getTransportVersion().before(TransportVersions.ML_INFERENCE_COHERE_UNUSED_RERANK_SETTINGS_REMOVED)) { + // An older node sends these fields, so we need to skip them to progress through the serialized data + in.readOptionalEnum(SimilarityMeasure.class); + in.readOptionalVInt(); + in.readOptionalVInt(); + } + + this.modelId = in.readOptionalString(); + + if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED)) { + this.rateLimitSettings = new RateLimitSettings(in); + } else { + this.rateLimitSettings = DEFAULT_RATE_LIMIT_SETTINGS; + } + } + + public URI uri() { + return uri; + } + + @Override + public String modelId() { + return modelId; + } + + @Override + public RateLimitSettings rateLimitSettings() { + return rateLimitSettings; } @Override @@ -55,7 +131,7 @@ public String getWriteableName() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - commonSettings.toXContentFragment(builder, params); + toXContentFragmentOfExposedFields(builder, params); builder.endObject(); return builder; @@ -63,35 +139,56 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException { - commonSettings.toXContentFragmentOfExposedFields(builder, params); + if (uri != null) { + builder.field(URL, uri.toString()); + } + + if (modelId != null) { + builder.field(MODEL_ID, modelId); + } + + rateLimitSettings.toXContent(builder, params); return builder; } @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_COHERE_RERANK; + return TransportVersions.V_8_14_0; } @Override public void writeTo(StreamOutput out) throws IOException { - commonSettings.writeTo(out); + var uriToWrite = uri != null ? uri.toString() : null; + out.writeOptionalString(uriToWrite); + + if (out.getTransportVersion().before(TransportVersions.ML_INFERENCE_COHERE_UNUSED_RERANK_SETTINGS_REMOVED)) { + // An old node expects this data to be present, so we need to send at least the booleans + // indicating that the fields are not set + out.writeOptionalEnum(null); + out.writeOptionalVInt(null); + out.writeOptionalVInt(null); + } + + out.writeOptionalString(modelId); + + if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED)) { + rateLimitSettings.writeTo(out); + } } @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - CohereRerankServiceSettings that = (CohereRerankServiceSettings) o; - return Objects.equals(commonSettings, that.commonSettings); + public boolean equals(Object object) { + if (this == object) return true; + if (object == null || getClass() != object.getClass()) return false; + CohereRerankServiceSettings that = (CohereRerankServiceSettings) object; + return Objects.equals(uri, that.uri) + && Objects.equals(modelId, that.modelId) + && Objects.equals(rateLimitSettings, that.rateLimitSettings); } @Override public int hashCode() { - return Objects.hash(commonSettings); - } - - public CohereServiceSettings getCommonSettings() { - return commonSettings; + return Objects.hash(uri, modelId, rateLimitSettings); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettings.java index 82f2d0e6f7ada..a01f6a4e65b8f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettings.java @@ -136,7 +136,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_COHERE_RERANK; + return TransportVersions.V_8_14_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java new file mode 100644 index 0000000000000..574ca77d4587e --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java @@ -0,0 +1,206 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elasticsearch; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.InferenceService; +import org.elasticsearch.inference.InferenceServiceExtension; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction; +import org.elasticsearch.xpack.core.ml.action.InferModelAction; +import org.elasticsearch.xpack.core.ml.action.PutTrainedModelAction; +import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; +import org.elasticsearch.xpack.core.ml.action.StopTrainedModelDeploymentAction; +import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig; +import org.elasticsearch.xpack.core.ml.inference.TrainedModelInput; +import org.elasticsearch.xpack.core.ml.inference.TrainedModelPrefixStrings; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfigUpdate; +import org.elasticsearch.xpack.inference.services.elser.ElserInternalModel; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; + +import static org.elasticsearch.xpack.core.ClientHelper.INFERENCE_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; + +public abstract class BaseElasticsearchInternalService implements InferenceService { + + protected final OriginSettingClient client; + + private static final Logger logger = LogManager.getLogger(BaseElasticsearchInternalService.class); + + public BaseElasticsearchInternalService(InferenceServiceExtension.InferenceServiceFactoryContext context) { + this.client = new OriginSettingClient(context.client(), ClientHelper.INFERENCE_ORIGIN); + } + + /** + * The task types supported by the service + * @return Set of supported. + */ + protected abstract EnumSet supportedTaskTypes(); + + @Override + public void start(Model model, ActionListener listener) { + if (model instanceof ElasticsearchInternalModel == false) { + listener.onFailure(notElasticsearchModelException(model)); + return; + } + + if (supportedTaskTypes().contains(model.getTaskType()) == false) { + listener.onFailure( + new IllegalStateException(TaskType.unsupportedTaskTypeErrorMsg(model.getConfigurations().getTaskType(), name())) + ); + return; + } + + var esModel = (ElasticsearchInternalModel) model; + var startRequest = esModel.getStartTrainedModelDeploymentActionRequest(); + var responseListener = esModel.getCreateTrainedModelAssignmentActionListener(model, listener); + + client.execute(StartTrainedModelDeploymentAction.INSTANCE, startRequest, responseListener); + } + + @Override + public void stop(String inferenceEntityId, ActionListener listener) { + var request = new StopTrainedModelDeploymentAction.Request(inferenceEntityId); + request.setForce(true); + client.execute( + StopTrainedModelDeploymentAction.INSTANCE, + request, + listener.delegateFailureAndWrap((delegatedResponseListener, response) -> delegatedResponseListener.onResponse(Boolean.TRUE)) + ); + } + + protected static IllegalStateException notElasticsearchModelException(Model model) { + return new IllegalStateException( + "Error starting model, [" + model.getConfigurations().getInferenceEntityId() + "] is not an Elasticsearch service model" + ); + } + + @Override + public void putModel(Model model, ActionListener listener) { + if (model instanceof ElasticsearchInternalModel == false) { + listener.onFailure(notElasticsearchModelException(model)); + return; + } else if (model instanceof MultilingualE5SmallModel e5Model) { + putBuiltInModel(e5Model.getServiceSettings().modelId(), listener); + } else if (model instanceof ElserInternalModel elserModel) { + putBuiltInModel(elserModel.getServiceSettings().modelId(), listener); + } else if (model instanceof CustomElandModel) { + logger.info("Custom eland model detected, model must have been already loaded into the cluster with eland."); + listener.onResponse(Boolean.TRUE); + } else { + listener.onFailure( + new IllegalArgumentException( + "Can not download model automatically for [" + + model.getConfigurations().getInferenceEntityId() + + "] you may need to download it through the trained models API or with eland." + ) + ); + return; + } + } + + private void putBuiltInModel(String modelId, ActionListener listener) { + var input = new TrainedModelInput(List.of("text_field")); // by convention text_field is used + var config = TrainedModelConfig.builder().setInput(input).setModelId(modelId).validate(true).build(); + PutTrainedModelAction.Request putRequest = new PutTrainedModelAction.Request(config, false, true); + executeAsyncWithOrigin( + client, + INFERENCE_ORIGIN, + PutTrainedModelAction.INSTANCE, + putRequest, + ActionListener.wrap(response -> listener.onResponse(Boolean.TRUE), e -> { + if (e instanceof ElasticsearchStatusException esException + && esException.getMessage().contains(PutTrainedModelAction.MODEL_ALREADY_EXISTS_ERROR_MESSAGE_FRAGMENT)) { + listener.onResponse(Boolean.TRUE); + } else { + listener.onFailure(e); + } + }) + ); + } + + @Override + public void isModelDownloaded(Model model, ActionListener listener) { + ActionListener getModelsResponseListener = listener.delegateFailure((delegate, response) -> { + if (response.getResources().count() < 1) { + delegate.onResponse(Boolean.FALSE); + } else { + delegate.onResponse(Boolean.TRUE); + } + }); + + if (model instanceof ElasticsearchInternalModel == false) { + listener.onFailure(notElasticsearchModelException(model)); + } else if (model.getServiceSettings() instanceof ElasticsearchInternalServiceSettings internalServiceSettings) { + String modelId = internalServiceSettings.modelId(); + GetTrainedModelsAction.Request getRequest = new GetTrainedModelsAction.Request(modelId); + executeAsyncWithOrigin(client, INFERENCE_ORIGIN, GetTrainedModelsAction.INSTANCE, getRequest, getModelsResponseListener); + } else { + listener.onFailure( + new IllegalArgumentException( + "Unable to determine supported model for [" + + model.getConfigurations().getInferenceEntityId() + + "] please verify the request and submit a bug report if necessary." + ) + ); + } + } + + @Override + public boolean isInClusterService() { + return true; + } + + @Override + public void close() throws IOException {} + + public static String selectDefaultModelVariantBasedOnClusterArchitecture( + Set modelArchitectures, + String linuxX86OptimisedModel, + String platformAgnosticModel + ) { + // choose a default model version based on the cluster architecture + boolean homogenous = modelArchitectures.size() == 1; + if (homogenous && modelArchitectures.iterator().next().equals("linux-x86_64")) { + // Use the hardware optimized model + return linuxX86OptimisedModel; + } else { + // default to the platform-agnostic model + return platformAgnosticModel; + } + } + + public static InferModelAction.Request buildInferenceRequest( + String id, + InferenceConfigUpdate update, + List inputs, + InputType inputType, + TimeValue timeout, + boolean chunk + ) { + var request = InferModelAction.Request.forTextInput(id, update, inputs, true, timeout); + request.setPrefixType( + InputType.SEARCH == inputType ? TrainedModelPrefixStrings.PrefixType.SEARCH : TrainedModelPrefixStrings.PrefixType.INGEST + ); + request.setHighPriority(InputType.SEARCH == inputType); + request.setChunked(chunk); + return request; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandEmbeddingModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandEmbeddingModel.java index bb4e0c2c513ac..59203d00e589a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandEmbeddingModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandEmbeddingModel.java @@ -7,34 +7,17 @@ package org.elasticsearch.xpack.inference.services.elasticsearch; -import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; - -import java.util.Map; public class CustomElandEmbeddingModel extends CustomElandModel { - public CustomElandEmbeddingModel( - String inferenceEntityId, - TaskType taskType, - String service, - Map serviceSettings, - ConfigurationParseContext context - ) { - this(inferenceEntityId, taskType, service, CustomElandInternalTextEmbeddingServiceSettings.fromMap(serviceSettings, context)); - } - public CustomElandEmbeddingModel( String inferenceEntityId, TaskType taskType, String service, CustomElandInternalTextEmbeddingServiceSettings serviceSettings ) { - super( - new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings), - serviceSettings.getElasticsearchInternalServiceSettings() - ); + super(inferenceEntityId, taskType, service, serviceSettings); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalServiceSettings.java index b74dbe482acc6..3cc7e0c6c2b53 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalServiceSettings.java @@ -7,29 +7,21 @@ package org.elasticsearch.xpack.inference.services.elasticsearch; -import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.inference.ModelConfigurations; -import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; -import org.elasticsearch.xpack.inference.services.ServiceUtils; import java.io.IOException; -import java.util.Map; - -import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredPositiveInteger; -import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredString; public class CustomElandInternalServiceSettings extends ElasticsearchInternalServiceSettings { public static final String NAME = "custom_eland_model_internal_service_settings"; + public CustomElandInternalServiceSettings(ElasticsearchInternalServiceSettings other) { + super(other); + } + public CustomElandInternalServiceSettings( - int numAllocations, + Integer numAllocations, int numThreads, String modelId, AdaptiveAllocationsSettings adaptiveAllocationsSettings @@ -37,94 +29,12 @@ public CustomElandInternalServiceSettings( super(numAllocations, numThreads, modelId, adaptiveAllocationsSettings); } - /** - * Parse the CustomElandServiceSettings from map and validate the setting values. - * - * This method does not verify the model variant - * - * If required setting are missing or the values are invalid an - * {@link ValidationException} is thrown. - * - * @param map Source map containing the config - * @return The {@code CustomElandServiceSettings} builder - */ - public static CustomElandInternalServiceSettings fromMap(Map map) { - ValidationException validationException = new ValidationException(); - - Integer numAllocations = extractRequiredPositiveInteger( - map, - NUM_ALLOCATIONS, - ModelConfigurations.SERVICE_SETTINGS, - validationException - ); - Integer numThreads = extractRequiredPositiveInteger(map, NUM_THREADS, ModelConfigurations.SERVICE_SETTINGS, validationException); - AdaptiveAllocationsSettings adaptiveAllocationsSettings = ServiceUtils.removeAsAdaptiveAllocationsSettings( - map, - ADAPTIVE_ALLOCATIONS - ); - if (adaptiveAllocationsSettings != null) { - ActionRequestValidationException exception = adaptiveAllocationsSettings.validate(); - if (exception != null) { - validationException.addValidationErrors(exception.validationErrors()); - } - } - String modelId = extractRequiredString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException); - - if (validationException.validationErrors().isEmpty() == false) { - throw validationException; - } - - var builder = new Builder() { - @Override - public CustomElandInternalServiceSettings build() { - return new CustomElandInternalServiceSettings( - getNumAllocations(), - getNumThreads(), - getModelId(), - getAdaptiveAllocationsSettings() - ); - } - }; - builder.setNumAllocations(numAllocations); - builder.setNumThreads(numThreads); - builder.setModelId(modelId); - builder.setAdaptiveAllocationsSettings(adaptiveAllocationsSettings); - return builder.build(); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return super.toXContent(builder, params); - } - public CustomElandInternalServiceSettings(StreamInput in) throws IOException { - super( - in.readVInt(), - in.readVInt(), - in.readString(), - in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS) - ? in.readOptionalWriteable(AdaptiveAllocationsSettings::new) - : null - ); - } - - @Override - public boolean isFragment() { - return super.isFragment(); + super(in); } @Override public String getWriteableName() { return CustomElandInternalServiceSettings.NAME; } - - @Override - public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_8_13_0; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettings.java index 8413d06045601..381c97969e79f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettings.java @@ -7,14 +7,12 @@ package org.elasticsearch.xpack.inference.services.elasticsearch; -import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.inference.ModelConfigurations; -import org.elasticsearch.inference.ServiceSettings; import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -33,7 +31,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalPositiveInteger; import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractSimilarity; -public class CustomElandInternalTextEmbeddingServiceSettings implements ServiceSettings { +public class CustomElandInternalTextEmbeddingServiceSettings extends ElasticsearchInternalServiceSettings { public static final String NAME = "custom_eland_model_internal_text_embedding_service_settings"; @@ -51,12 +49,12 @@ public class CustomElandInternalTextEmbeddingServiceSettings implements ServiceS */ public static CustomElandInternalTextEmbeddingServiceSettings fromMap(Map map, ConfigurationParseContext context) { return switch (context) { - case REQUEST -> fromRequestMap(map); - case PERSISTENT -> fromPersistedMap(map); + case REQUEST -> forRequest(map); + case PERSISTENT -> forPersisted(map); }; } - private static CustomElandInternalTextEmbeddingServiceSettings fromRequestMap(Map map) { + private static CustomElandInternalTextEmbeddingServiceSettings forRequest(Map map) { ValidationException validationException = new ValidationException(); var commonFields = commonFieldsFromMap(map, validationException); @@ -67,7 +65,7 @@ private static CustomElandInternalTextEmbeddingServiceSettings fromRequestMap(Ma return new CustomElandInternalTextEmbeddingServiceSettings(commonFields); } - private static CustomElandInternalTextEmbeddingServiceSettings fromPersistedMap(Map map) { + private static CustomElandInternalTextEmbeddingServiceSettings forPersisted(Map map) { var commonFields = commonFieldsFromMap(map); Integer dims = extractOptionalPositiveInteger(map, DIMENSIONS, ModelConfigurations.SERVICE_SETTINGS, new ValidationException()); @@ -97,13 +95,12 @@ private static CommonFields commonFieldsFromMap(Map map, Validat ); return new CommonFields( - internalSettings, + internalSettings.build(), Objects.requireNonNullElse(similarity, SimilarityMeasure.COSINE), Objects.requireNonNullElse(elementType, DenseVectorFieldMapper.ElementType.FLOAT) ); } - private final ElasticsearchInternalServiceSettings internalServiceSettings; private final Integer dimensions; private final SimilarityMeasure similarityMeasure; private final DenseVectorFieldMapper.ElementType elementType; @@ -134,19 +131,14 @@ public CustomElandInternalTextEmbeddingServiceSettings( SimilarityMeasure similarityMeasure, DenseVectorFieldMapper.ElementType elementType ) { - internalServiceSettings = new ElasticsearchInternalServiceSettings( - numAllocations, - numThreads, - modelId, - adaptiveAllocationsSettings - ); + super(numAllocations, numThreads, modelId, adaptiveAllocationsSettings); this.dimensions = dimensions; this.similarityMeasure = Objects.requireNonNull(similarityMeasure); this.elementType = Objects.requireNonNull(elementType); } public CustomElandInternalTextEmbeddingServiceSettings(StreamInput in) throws IOException { - internalServiceSettings = new ElasticsearchInternalServiceSettings(in); + super(in); if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_ELAND_SETTINGS_ADDED)) { dimensions = in.readOptionalVInt(); similarityMeasure = in.readEnum(SimilarityMeasure.class); @@ -163,7 +155,12 @@ private CustomElandInternalTextEmbeddingServiceSettings(CommonFields commonField } private CustomElandInternalTextEmbeddingServiceSettings(CommonFields commonFields, Integer dimensions) { - internalServiceSettings = commonFields.internalServiceSettings; + super( + commonFields.internalServiceSettings.getNumAllocations(), + commonFields.internalServiceSettings.getNumThreads(), + commonFields.internalServiceSettings.modelId(), + commonFields.internalServiceSettings.getAdaptiveAllocationsSettings() + ); this.dimensions = dimensions; similarityMeasure = commonFields.similarityMeasure; elementType = commonFields.elementType; @@ -173,7 +170,7 @@ private CustomElandInternalTextEmbeddingServiceSettings(CommonFields commonField public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - internalServiceSettings.addXContentFragment(builder, params); + addInternalSettingsToXContent(builder, params); if (dimensions != null) { builder.field(DIMENSIONS, dimensions); @@ -196,14 +193,9 @@ public String getWriteableName() { return CustomElandInternalTextEmbeddingServiceSettings.NAME; } - @Override - public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_8_13_0; - } - @Override public void writeTo(StreamOutput out) throws IOException { - internalServiceSettings.writeTo(out); + super.writeTo(out); if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_ELAND_SETTINGS_ADDED)) { out.writeOptionalVInt(dimensions); @@ -212,10 +204,6 @@ public void writeTo(StreamOutput out) throws IOException { } } - public ElasticsearchInternalServiceSettings getElasticsearchInternalServiceSettings() { - return internalServiceSettings; - } - @Override public DenseVectorFieldMapper.ElementType elementType() { return elementType; @@ -241,7 +229,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; CustomElandInternalTextEmbeddingServiceSettings that = (CustomElandInternalTextEmbeddingServiceSettings) o; - return Objects.equals(internalServiceSettings, that.internalServiceSettings) + return super.equals(that) && Objects.equals(dimensions, that.dimensions) && Objects.equals(similarityMeasure, that.similarityMeasure) && Objects.equals(elementType, that.elementType); @@ -249,7 +237,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(internalServiceSettings, dimensions, similarityMeasure, elementType); + return Objects.hash(super.hashCode(), dimensions, similarityMeasure, elementType); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandModel.java index 703fca8c74c31..83f22f08b620d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandModel.java @@ -10,37 +10,30 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.inference.Model; -import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.TaskSettings; +import org.elasticsearch.inference.TaskType; import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; -import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings; -import java.util.Objects; +public class CustomElandModel extends ElasticsearchInternalModel { -import static org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus.State.STARTED; - -public class CustomElandModel extends Model implements ElasticsearchModel { - private final InternalServiceSettings internalServiceSettings; - - public CustomElandModel(ModelConfigurations configurations, InternalServiceSettings internalServiceSettings) { - super(configurations); - this.internalServiceSettings = Objects.requireNonNull(internalServiceSettings); - } - - public String getModelId() { - return internalServiceSettings.getModelId(); + public CustomElandModel( + String inferenceEntityId, + TaskType taskType, + String service, + ElasticsearchInternalServiceSettings internalServiceSettings + ) { + super(inferenceEntityId, taskType, service, internalServiceSettings); } - @Override - public StartTrainedModelDeploymentAction.Request getStartTrainedModelDeploymentActionRequest() { - var startRequest = new StartTrainedModelDeploymentAction.Request(internalServiceSettings.getModelId(), this.getInferenceEntityId()); - startRequest.setNumberOfAllocations(internalServiceSettings.getNumAllocations()); - startRequest.setThreadsPerAllocation(internalServiceSettings.getNumThreads()); - startRequest.setAdaptiveAllocationsSettings(internalServiceSettings.getAdaptiveAllocationsSettings()); - startRequest.setWaitForState(STARTED); - - return startRequest; + public CustomElandModel( + String inferenceEntityId, + TaskType taskType, + String service, + ElasticsearchInternalServiceSettings internalServiceSettings, + TaskSettings taskSettings + ) { + super(inferenceEntityId, taskType, service, internalServiceSettings, taskSettings); } @Override @@ -60,10 +53,9 @@ public void onFailure(Exception e) { if (ExceptionsHelper.unwrapCause(e) instanceof ResourceNotFoundException) { listener.onFailure( new ResourceNotFoundException( - "Could not start the TextEmbeddingService service as the " - + "custom eland model [{0}] for this platform cannot be found." + "Could not start the inference as the custom eland model [{0}] for this platform cannot be found." + " Custom models need to be loaded into the cluster with eland before they can be started.", - getModelId() + internalServiceSettings.modelId() ) ); return; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java index d880450739319..63f4a3dbf8472 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java @@ -7,40 +7,18 @@ package org.elasticsearch.xpack.inference.services.elasticsearch; -import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; - -import java.util.Map; public class CustomElandRerankModel extends CustomElandModel { public CustomElandRerankModel( - String inferenceEntityId, - TaskType taskType, - String service, - Map serviceSettings, - Map taskSettings, - ConfigurationParseContext context - ) { - this( - inferenceEntityId, - taskType, - service, - CustomElandInternalServiceSettings.fromMap(serviceSettings), - CustomElandRerankTaskSettings.defaultsFromMap(taskSettings) - ); - } - - // default for testing - CustomElandRerankModel( String inferenceEntityId, TaskType taskType, String service, CustomElandInternalServiceSettings serviceSettings, CustomElandRerankTaskSettings taskSettings ) { - super(new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings), serviceSettings); + super(inferenceEntityId, taskType, service, serviceSettings, taskSettings); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java index 0b586af5005fb..523aff20b8e05 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java @@ -107,7 +107,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_COHERE_RERANK; + return TransportVersions.V_8_14_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java new file mode 100644 index 0000000000000..405c687839629 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elasticsearch; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.TaskSettings; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; +import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; + +import static org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus.State.STARTED; + +public abstract class ElasticsearchInternalModel extends Model { + + protected final ElasticsearchInternalServiceSettings internalServiceSettings; + + public ElasticsearchInternalModel( + String inferenceEntityId, + TaskType taskType, + String service, + ElasticsearchInternalServiceSettings internalServiceSettings + ) { + super(new ModelConfigurations(inferenceEntityId, taskType, service, internalServiceSettings)); + this.internalServiceSettings = internalServiceSettings; + } + + public ElasticsearchInternalModel( + String inferenceEntityId, + TaskType taskType, + String service, + ElasticsearchInternalServiceSettings internalServiceSettings, + TaskSettings taskSettings + ) { + super(new ModelConfigurations(inferenceEntityId, taskType, service, internalServiceSettings, taskSettings)); + this.internalServiceSettings = internalServiceSettings; + } + + public StartTrainedModelDeploymentAction.Request getStartTrainedModelDeploymentActionRequest() { + var startRequest = new StartTrainedModelDeploymentAction.Request(internalServiceSettings.modelId(), this.getInferenceEntityId()); + startRequest.setNumberOfAllocations(internalServiceSettings.getNumAllocations()); + startRequest.setThreadsPerAllocation(internalServiceSettings.getNumThreads()); + startRequest.setAdaptiveAllocationsSettings(internalServiceSettings.getAdaptiveAllocationsSettings()); + startRequest.setWaitForState(STARTED); + + return startRequest; + } + + public abstract ActionListener getCreateTrainedModelAssignmentActionListener( + Model model, + ActionListener listener + ); +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java index 9dc88be16ddbb..c3a0111562319 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java @@ -13,13 +13,12 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.common.ValidationException; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.inference.ChunkingOptions; import org.elasticsearch.inference.InferenceResults; -import org.elasticsearch.inference.InferenceService; import org.elasticsearch.inference.InferenceServiceExtension; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; @@ -27,7 +26,6 @@ import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; @@ -39,20 +37,17 @@ import org.elasticsearch.xpack.core.ml.action.StopTrainedModelDeploymentAction; import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig; import org.elasticsearch.xpack.core.ml.inference.TrainedModelInput; -import org.elasticsearch.xpack.core.ml.inference.TrainedModelPrefixStrings; import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextEmbeddingFloatResults; -import org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextSimilarityConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.ServiceUtils; -import org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -64,9 +59,8 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMap; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; -import static org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings.MODEL_ID; -public class ElasticsearchInternalService implements InferenceService { +public class ElasticsearchInternalService extends BaseElasticsearchInternalService { public static final String NAME = "elasticsearch"; @@ -77,12 +71,15 @@ public class ElasticsearchInternalService implements InferenceService { MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86 ); - private final OriginSettingClient client; - private static final Logger logger = LogManager.getLogger(ElasticsearchInternalService.class); public ElasticsearchInternalService(InferenceServiceExtension.InferenceServiceFactoryContext context) { - this.client = new OriginSettingClient(context.client(), ClientHelper.INFERENCE_ORIGIN); + super(context); + } + + @Override + protected EnumSet supportedTaskTypes() { + return EnumSet.of(TaskType.RERANK, TaskType.TEXT_EMBEDDING); } @Override @@ -96,14 +93,16 @@ public void parseRequestConfig( try { Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); Map taskSettingsMap = removeFromMap(config, ModelConfigurations.TASK_SETTINGS); - String modelId = (String) serviceSettingsMap.get(MODEL_ID); + + throwIfNotEmptyMap(config, name()); + + String modelId = (String) serviceSettingsMap.get(ElasticsearchInternalServiceSettings.MODEL_ID); if (modelId == null) { - throw new IllegalArgumentException("Error parsing request config, model id is missing"); + throw new ValidationException().addValidationError("Error parsing request config, model id is missing"); } if (MULTILINGUAL_E5_SMALL_VALID_IDS.contains(modelId)) { e5Case(inferenceEntityId, taskType, config, platformArchitectures, serviceSettingsMap, modelListener); } else { - throwIfNotEmptyMap(config, name()); customElandCase(inferenceEntityId, taskType, serviceSettingsMap, taskSettingsMap, modelListener); } } catch (Exception e) { @@ -118,7 +117,7 @@ private void customElandCase( Map taskSettingsMap, ActionListener modelListener ) { - String modelId = (String) serviceSettingsMap.get(MODEL_ID); + String modelId = (String) serviceSettingsMap.get(ElasticsearchInternalServiceSettings.MODEL_ID); var request = new GetTrainedModelsAction.Request(modelId); var getModelsListener = modelListener.delegateFailureAndWrap((delegate, response) -> { @@ -154,13 +153,37 @@ private static CustomElandModel createCustomElandModel( Map taskSettings, ConfigurationParseContext context ) { + return switch (taskType) { - case TEXT_EMBEDDING -> new CustomElandEmbeddingModel(inferenceEntityId, taskType, NAME, serviceSettings, context); - case RERANK -> new CustomElandRerankModel(inferenceEntityId, taskType, NAME, serviceSettings, taskSettings, context); + case TEXT_EMBEDDING -> new CustomElandEmbeddingModel( + inferenceEntityId, + taskType, + NAME, + CustomElandInternalTextEmbeddingServiceSettings.fromMap(serviceSettings, context) + ); + case RERANK -> new CustomElandRerankModel( + inferenceEntityId, + taskType, + NAME, + elandServiceSettings(serviceSettings, context), + CustomElandRerankTaskSettings.fromMap(taskSettings) + ); default -> throw new ElasticsearchStatusException(TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), RestStatus.BAD_REQUEST); }; } + private static CustomElandInternalServiceSettings elandServiceSettings( + Map settingsMap, + ConfigurationParseContext context + ) { + return switch (context) { + case REQUEST -> new CustomElandInternalServiceSettings( + ElasticsearchInternalServiceSettings.fromRequestMap(settingsMap).build() + ); + case PERSISTENT -> new CustomElandInternalServiceSettings(ElasticsearchInternalServiceSettings.fromPersistedMap(settingsMap)); + }; + } + private void e5Case( String inferenceEntityId, TaskType taskType, @@ -169,16 +192,22 @@ private void e5Case( Map serviceSettingsMap, ActionListener modelListener ) { - var e5ServiceSettings = MultilingualE5SmallInternalServiceSettings.fromMap(serviceSettingsMap); - - if (e5ServiceSettings.getModelId() == null) { - e5ServiceSettings.setModelId(selectDefaultModelVariantBasedOnClusterArchitecture(platformArchitectures)); + var esServiceSettingsBuilder = ElasticsearchInternalServiceSettings.fromRequestMap(serviceSettingsMap); + + if (esServiceSettingsBuilder.getModelId() == null) { + esServiceSettingsBuilder.setModelId( + selectDefaultModelVariantBasedOnClusterArchitecture( + platformArchitectures, + MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86, + MULTILINGUAL_E5_SMALL_MODEL_ID + ) + ); } - if (modelVariantDoesNotMatchArchitecturesAndIsNotPlatformAgnostic(platformArchitectures, e5ServiceSettings)) { + if (modelVariantDoesNotMatchArchitecturesAndIsNotPlatformAgnostic(platformArchitectures, esServiceSettingsBuilder.getModelId())) { throw new IllegalArgumentException( "Error parsing request config, model id does not match any models available on this platform. Was [" - + e5ServiceSettings.getModelId() + + esServiceSettingsBuilder.getModelId() + "]" ); } @@ -191,17 +220,22 @@ private void e5Case( inferenceEntityId, taskType, NAME, - (MultilingualE5SmallInternalServiceSettings) e5ServiceSettings.build() + new MultilingualE5SmallInternalServiceSettings(esServiceSettingsBuilder.build()) ) ); } private static boolean modelVariantDoesNotMatchArchitecturesAndIsNotPlatformAgnostic( Set platformArchitectures, - InternalServiceSettings.Builder e5ServiceSettings + String modelId ) { - return e5ServiceSettings.getModelId().equals(selectDefaultModelVariantBasedOnClusterArchitecture(platformArchitectures)) == false - && e5ServiceSettings.getModelId().equals(MULTILINGUAL_E5_SMALL_MODEL_ID) == false; + return modelId.equals( + selectDefaultModelVariantBasedOnClusterArchitecture( + platformArchitectures, + MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86, + MULTILINGUAL_E5_SMALL_MODEL_ID + ) + ) && modelId.equals(MULTILINGUAL_E5_SMALL_MODEL_ID) == false; } @Override @@ -219,7 +253,7 @@ public Model parsePersistedConfig(String inferenceEntityId, TaskType taskType, M Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); Map taskSettingsMap = removeFromMap(config, ModelConfigurations.TASK_SETTINGS); - String modelId = (String) serviceSettingsMap.get(MODEL_ID); + String modelId = (String) serviceSettingsMap.get(ElasticsearchInternalServiceSettings.MODEL_ID); if (modelId == null) { throw new IllegalArgumentException("Error parsing request config, model id is missing"); } @@ -229,7 +263,7 @@ public Model parsePersistedConfig(String inferenceEntityId, TaskType taskType, M inferenceEntityId, taskType, NAME, - (MultilingualE5SmallInternalServiceSettings) MultilingualE5SmallInternalServiceSettings.fromMap(serviceSettingsMap).build() + new MultilingualE5SmallInternalServiceSettings(ElasticsearchInternalServiceSettings.fromPersistedMap(serviceSettingsMap)) ); } else { return createCustomElandModel( @@ -250,7 +284,7 @@ public void checkModelConfig(Model model, ActionListener listener) { // model id. To get around this we'll have the getEmbeddingSize() method use the model id instead of inference id. So we need // to create a temporary model that overrides the inference id with the model id. var temporaryModelWithModelId = new CustomElandEmbeddingModel( - elandModel.getModelId(), + elandModel.getServiceSettings().modelId(), elandModel.getTaskType(), elandModel.getConfigurations().getService(), elandModel.getServiceSettings() @@ -268,10 +302,10 @@ public void checkModelConfig(Model model, ActionListener listener) { private static CustomElandEmbeddingModel updateModelWithEmbeddingDetails(CustomElandEmbeddingModel model, int embeddingSize) { CustomElandInternalTextEmbeddingServiceSettings serviceSettings = new CustomElandInternalTextEmbeddingServiceSettings( - model.getServiceSettings().getElasticsearchInternalServiceSettings().getNumAllocations(), - model.getServiceSettings().getElasticsearchInternalServiceSettings().getNumThreads(), - model.getServiceSettings().getElasticsearchInternalServiceSettings().getModelId(), - model.getServiceSettings().getElasticsearchInternalServiceSettings().getAdaptiveAllocationsSettings(), + model.getServiceSettings().getNumAllocations(), + model.getServiceSettings().getNumThreads(), + model.getServiceSettings().modelId(), + model.getServiceSettings().getAdaptiveAllocationsSettings(), embeddingSize, model.getServiceSettings().similarity(), model.getServiceSettings().elementType() @@ -439,8 +473,8 @@ private static ChunkedInferenceServiceResults translateToChunkedResult(Inference @Override public void start(Model model, ActionListener listener) { - if (model instanceof ElasticsearchModel == false) { - listener.onFailure(notTextEmbeddingModelException(model)); + if (model instanceof ElasticsearchInternalModel == false) { + listener.onFailure(notElasticsearchModelException(model)); return; } @@ -451,8 +485,8 @@ public void start(Model model, ActionListener listener) { return; } - var startRequest = ((ElasticsearchModel) model).getStartTrainedModelDeploymentActionRequest(); - var responseListener = ((ElasticsearchModel) model).getCreateTrainedModelAssignmentActionListener(model, listener); + var startRequest = ((ElasticsearchInternalModel) model).getStartTrainedModelDeploymentActionRequest(); + var responseListener = ((ElasticsearchInternalModel) model).getCreateTrainedModelAssignmentActionListener(model, listener); client.execute(StartTrainedModelDeploymentAction.INSTANCE, startRequest, responseListener); } @@ -470,11 +504,11 @@ public void stop(String inferenceEntityId, ActionListener listener) { @Override public void putModel(Model model, ActionListener listener) { - if (model instanceof ElasticsearchModel == false) { - listener.onFailure(notTextEmbeddingModelException(model)); + if (model instanceof ElasticsearchInternalModel == false) { + listener.onFailure(notElasticsearchModelException(model)); return; } else if (model instanceof MultilingualE5SmallModel e5Model) { - String modelId = e5Model.getServiceSettings().getModelId(); + String modelId = e5Model.getServiceSettings().modelId(); var input = new TrainedModelInput(List.of("text_field")); // by convention text_field is used var config = TrainedModelConfig.builder().setInput(input).setModelId(modelId).validate(true).build(); PutTrainedModelAction.Request putRequest = new PutTrainedModelAction.Request(config, false, true); @@ -517,12 +551,12 @@ public void isModelDownloaded(Model model, ActionListener listener) { } }); - if (model instanceof ElasticsearchModel == false) { - listener.onFailure(notTextEmbeddingModelException(model)); - } else if (model.getServiceSettings() instanceof InternalServiceSettings internalServiceSettings) { - String modelId = internalServiceSettings.getModelId(); + if (model.getServiceSettings() instanceof ElasticsearchInternalServiceSettings internalServiceSettings) { + String modelId = internalServiceSettings.modelId(); GetTrainedModelsAction.Request getRequest = new GetTrainedModelsAction.Request(modelId); executeAsyncWithOrigin(client, INFERENCE_ORIGIN, GetTrainedModelsAction.INSTANCE, getRequest, getModelsResponseListener); + } else if (model instanceof ElasticsearchInternalModel == false) { + listener.onFailure(notElasticsearchModelException(model)); } else { listener.onFailure( new IllegalArgumentException( @@ -534,42 +568,16 @@ public void isModelDownloaded(Model model, ActionListener listener) { } } - private static IllegalStateException notTextEmbeddingModelException(Model model) { - return new IllegalStateException( - "Error starting model, [" + model.getConfigurations().getInferenceEntityId() + "] is not a text embedding model" - ); - } - - @Override - public boolean isInClusterService() { - return true; - } - @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_L2_NORM_SIMILARITY_ADDED; + return TransportVersions.V_8_14_0; } - @Override - public void close() throws IOException {} - @Override public String name() { return NAME; } - private static String selectDefaultModelVariantBasedOnClusterArchitecture(Set modelArchitectures) { - // choose a default model version based on the cluster architecture - boolean homogenous = modelArchitectures.size() == 1; - if (homogenous && modelArchitectures.iterator().next().equals("linux-x86_64")) { - // Use the hardware optimized model - return MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86; - } else { - // default to the platform-agnostic model - return MULTILINGUAL_E5_SMALL_MODEL_ID; - } - } - private RankedDocsResults textSimilarityResultsToRankedDocs( List results, Function inputSupplier @@ -601,21 +609,4 @@ private RankedDocsResults textSimilarityResultsToRankedDocs( Collections.sort(rankings); return new RankedDocsResults(rankings); } - - public static InferModelAction.Request buildInferenceRequest( - String id, - InferenceConfigUpdate update, - List inputs, - InputType inputType, - TimeValue timeout, - boolean chunk - ) { - var request = InferModelAction.Request.forTextInput(id, update, inputs, true, timeout); - request.setPrefixType( - InputType.SEARCH == inputType ? TrainedModelPrefixStrings.PrefixType.SEARCH : TrainedModelPrefixStrings.PrefixType.INGEST - ); - request.setHighPriority(InputType.SEARCH == inputType); - request.setChunked(chunk); - return request; - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java index ff4ef4ff0358f..1acf19c5373b7 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java @@ -9,28 +9,68 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; -import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ServiceSettings; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; import org.elasticsearch.xpack.inference.services.ServiceUtils; -import org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings; import java.io.IOException; +import java.util.List; import java.util.Map; import java.util.Objects; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalPositiveInteger; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalString; import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredPositiveInteger; -import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredString; -public class ElasticsearchInternalServiceSettings extends InternalServiceSettings { +public class ElasticsearchInternalServiceSettings implements ServiceSettings { public static final String NAME = "text_embedding_internal_service_settings"; private static final int FAILED_INT_PARSE_VALUE = -1; - public static ElasticsearchInternalServiceSettings fromMap(Map map, ValidationException validationException) { - Integer numAllocations = extractRequiredPositiveInteger( + public static final String NUM_ALLOCATIONS = "num_allocations"; + public static final String NUM_THREADS = "num_threads"; + public static final String MODEL_ID = "model_id"; + public static final String ADAPTIVE_ALLOCATIONS = "adaptive_allocations"; + + private final Integer numAllocations; + private final int numThreads; + private final String modelId; + private final AdaptiveAllocationsSettings adaptiveAllocationsSettings; + + public static ElasticsearchInternalServiceSettings fromPersistedMap(Map map) { + return fromRequestMap(map).build(); + } + + /** + * Parse the ElasticsearchInternalServiceSettings from the map. + * Validates that present threading settings are of the right type and value, + * The model id is optional, it is for the inference service to check and + * potentially set a default value for the model id. + * Throws an {@code ValidationException} on validation failures + * + * @param map The request map. + * @return A builder to allow the settings to be modified. + */ + public static ElasticsearchInternalServiceSettings.Builder fromRequestMap(Map map) { + var validationException = new ValidationException(); + var builder = fromMap(map, validationException); + if (validationException.validationErrors().isEmpty() == false) { + throw validationException; + } + return builder; + } + + protected static ElasticsearchInternalServiceSettings.Builder fromMap( + Map map, + ValidationException validationException + ) { + Integer numAllocations = extractOptionalPositiveInteger( map, NUM_ALLOCATIONS, ModelConfigurations.SERVICE_SETTINGS, @@ -39,44 +79,115 @@ public static ElasticsearchInternalServiceSettings fromMap(Map m Integer numThreads = extractRequiredPositiveInteger(map, NUM_THREADS, ModelConfigurations.SERVICE_SETTINGS, validationException); AdaptiveAllocationsSettings adaptiveAllocationsSettings = ServiceUtils.removeAsAdaptiveAllocationsSettings( map, - ADAPTIVE_ALLOCATIONS + ADAPTIVE_ALLOCATIONS, + validationException ); - if (adaptiveAllocationsSettings != null) { - ActionRequestValidationException exception = adaptiveAllocationsSettings.validate(); - if (exception != null) { - validationException.addValidationErrors(exception.validationErrors()); - } + + // model id is optional as the ELSER and E5 service will default it + String modelId = extractOptionalString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException); + + if (numAllocations == null && adaptiveAllocationsSettings == null) { + validationException.addValidationError( + ServiceUtils.missingOneOfSettingsErrorMsg( + List.of(NUM_ALLOCATIONS, ADAPTIVE_ALLOCATIONS), + ModelConfigurations.SERVICE_SETTINGS + ) + ); } - String modelId = extractRequiredString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException); // if an error occurred while parsing, we'll set these to an invalid value, so we don't accidentally get a // null pointer when doing unboxing - return new ElasticsearchInternalServiceSettings( - Objects.requireNonNullElse(numAllocations, FAILED_INT_PARSE_VALUE), - Objects.requireNonNullElse(numThreads, FAILED_INT_PARSE_VALUE), - modelId, - adaptiveAllocationsSettings - ); + return new ElasticsearchInternalServiceSettings.Builder().setNumAllocations(numAllocations) + .setNumThreads(Objects.requireNonNullElse(numThreads, FAILED_INT_PARSE_VALUE)) + .setModelId(modelId) + .setAdaptiveAllocationsSettings(adaptiveAllocationsSettings); } public ElasticsearchInternalServiceSettings( - int numAllocations, + Integer numAllocations, int numThreads, - String modelVariant, + String modelId, AdaptiveAllocationsSettings adaptiveAllocationsSettings ) { - super(numAllocations, numThreads, modelVariant, adaptiveAllocationsSettings); + this.numAllocations = numAllocations; + this.numThreads = numThreads; + this.modelId = Objects.requireNonNull(modelId); + this.adaptiveAllocationsSettings = adaptiveAllocationsSettings; + } + + protected ElasticsearchInternalServiceSettings(ElasticsearchInternalServiceSettings other) { + this.numAllocations = other.numAllocations; + this.numThreads = other.numThreads; + this.modelId = other.modelId; + this.adaptiveAllocationsSettings = other.adaptiveAllocationsSettings; } public ElasticsearchInternalServiceSettings(StreamInput in) throws IOException { - super( - in.readVInt(), - in.readVInt(), - in.readString(), - in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS) - ? in.readOptionalWriteable(AdaptiveAllocationsSettings::new) - : null - ); + if (in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { + this.numAllocations = in.readOptionalVInt(); + } else { + this.numAllocations = in.readVInt(); + } + this.numThreads = in.readVInt(); + this.modelId = in.readString(); + this.adaptiveAllocationsSettings = in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS) + ? in.readOptionalWriteable(AdaptiveAllocationsSettings::new) + : null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { + out.writeOptionalVInt(getNumAllocations()); + } else { + out.writeVInt(getNumAllocations()); + } + out.writeVInt(getNumThreads()); + out.writeString(modelId()); + if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { + out.writeOptionalWriteable(getAdaptiveAllocationsSettings()); + } + } + + @Override + public String modelId() { + return modelId; + } + + public Integer getNumAllocations() { + return numAllocations; + } + + public int getNumThreads() { + return numThreads; + } + + public AdaptiveAllocationsSettings getAdaptiveAllocationsSettings() { + return adaptiveAllocationsSettings; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + addInternalSettingsToXContent(builder, params); + builder.endObject(); + return builder; + } + + protected void addInternalSettingsToXContent(XContentBuilder builder, Params params) throws IOException { + if (numAllocations != null) { + builder.field(NUM_ALLOCATIONS, numAllocations); + } + builder.field(NUM_THREADS, getNumThreads()); + builder.field(MODEL_ID, modelId()); + if (adaptiveAllocationsSettings != null) { + builder.field(ADAPTIVE_ALLOCATIONS, adaptiveAllocationsSettings); + } + } + + @Override + public ToXContentObject getFilteredXContentObject() { + return this; } @Override @@ -89,4 +200,65 @@ public TransportVersion getMinimalSupportedVersion() { return TransportVersions.V_8_13_0; } + public static class Builder { + private Integer numAllocations; + private int numThreads; + private String modelId; + private AdaptiveAllocationsSettings adaptiveAllocationsSettings; + + public ElasticsearchInternalServiceSettings build() { + return new ElasticsearchInternalServiceSettings(numAllocations, numThreads, modelId, adaptiveAllocationsSettings); + } + + public Builder setNumAllocations(Integer numAllocations) { + this.numAllocations = numAllocations; + return this; + } + + public Builder setNumThreads(int numThreads) { + this.numThreads = numThreads; + return this; + } + + public Builder setModelId(String modelId) { + this.modelId = modelId; + return this; + } + + public Builder setAdaptiveAllocationsSettings(AdaptiveAllocationsSettings adaptiveAllocationsSettings) { + this.adaptiveAllocationsSettings = adaptiveAllocationsSettings; + return this; + } + + public String getModelId() { + return modelId; + } + + public Integer getNumAllocations() { + return numAllocations; + } + + public int getNumThreads() { + return numThreads; + } + + public AdaptiveAllocationsSettings getAdaptiveAllocationsSettings() { + return adaptiveAllocationsSettings; + } + } + + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ElasticsearchInternalServiceSettings that = (ElasticsearchInternalServiceSettings) o; + return Objects.equals(numAllocations, that.numAllocations) + && numThreads == that.numThreads + && Objects.equals(modelId, that.modelId) + && Objects.equals(adaptiveAllocationsSettings, that.adaptiveAllocationsSettings); + } + + @Override + public int hashCode() { + return Objects.hash(numAllocations, numThreads, modelId, adaptiveAllocationsSettings); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchModel.java deleted file mode 100644 index 627e570b24163..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchModel.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.services.elasticsearch; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.inference.Model; -import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; -import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; - -public interface ElasticsearchModel { - String getModelId(); - - StartTrainedModelDeploymentAction.Request getStartTrainedModelDeploymentActionRequest(); - - ActionListener getCreateTrainedModelAssignmentActionListener( - Model model, - ActionListener listener - ); -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallInternalServiceSettings.java index e4aa9616fb332..2f27fa073b4f0 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallInternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallInternalServiceSettings.java @@ -7,25 +7,16 @@ package org.elasticsearch.xpack.inference.services.elasticsearch; -import org.elasticsearch.TransportVersions; -import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; -import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; -import org.elasticsearch.xpack.inference.services.ServiceUtils; -import org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings; import java.io.IOException; import java.util.Arrays; import java.util.Map; -import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredPositiveInteger; - public class MultilingualE5SmallInternalServiceSettings extends ElasticsearchInternalServiceSettings { public static final String NAME = "multilingual_e5_small_service_settings"; @@ -33,8 +24,12 @@ public class MultilingualE5SmallInternalServiceSettings extends ElasticsearchInt static final int DIMENSIONS = 384; static final SimilarityMeasure SIMILARITY = SimilarityMeasure.COSINE; + public MultilingualE5SmallInternalServiceSettings(ElasticsearchInternalServiceSettings other) { + super(other); + } + public MultilingualE5SmallInternalServiceSettings( - int numAllocations, + Integer numAllocations, int numThreads, String modelId, AdaptiveAllocationsSettings adaptiveAllocationsSettings @@ -43,14 +38,7 @@ public MultilingualE5SmallInternalServiceSettings( } public MultilingualE5SmallInternalServiceSettings(StreamInput in) throws IOException { - super( - in.readVInt(), - in.readVInt(), - in.readString(), - in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS) - ? in.readOptionalWriteable(AdaptiveAllocationsSettings::new) - : null - ); + super(in); } /** @@ -60,38 +48,13 @@ public MultilingualE5SmallInternalServiceSettings(StreamInput in) throws IOExcep * {@link ValidationException} is thrown. * * @param map Source map containing the config - * @return The {@code MultilingualE5SmallServiceSettings} builder + * @return The builder */ - public static MultilingualE5SmallInternalServiceSettings.Builder fromMap(Map map) { + public static ElasticsearchInternalServiceSettings.Builder fromRequestMap(Map map) { ValidationException validationException = new ValidationException(); - var requestFields = extractRequestFields(map, validationException); - - if (validationException.validationErrors().isEmpty() == false) { - throw validationException; - } - - return createBuilder(requestFields); - } + var baseSettings = ElasticsearchInternalServiceSettings.fromMap(map, validationException); - private static RequestFields extractRequestFields(Map map, ValidationException validationException) { - Integer numAllocations = extractRequiredPositiveInteger( - map, - NUM_ALLOCATIONS, - ModelConfigurations.SERVICE_SETTINGS, - validationException - ); - Integer numThreads = extractRequiredPositiveInteger(map, NUM_THREADS, ModelConfigurations.SERVICE_SETTINGS, validationException); - AdaptiveAllocationsSettings adaptiveAllocationsSettings = ServiceUtils.removeAsAdaptiveAllocationsSettings( - map, - ADAPTIVE_ALLOCATIONS - ); - if (adaptiveAllocationsSettings != null) { - ActionRequestValidationException exception = adaptiveAllocationsSettings.validate(); - if (exception != null) { - validationException.addValidationErrors(exception.validationErrors()); - } - } - String modelId = ServiceUtils.removeAsType(map, MODEL_ID, String.class); + String modelId = baseSettings.getModelId(); if (modelId != null) { if (ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_VALID_IDS.contains(modelId) == false) { validationException.addValidationError( @@ -103,38 +66,11 @@ private static RequestFields extractRequestFields(Map map, Valid } } - return new RequestFields(numAllocations, numThreads, modelId, adaptiveAllocationsSettings); - } - - private static MultilingualE5SmallInternalServiceSettings.Builder createBuilder(RequestFields requestFields) { - var builder = new InternalServiceSettings.Builder() { - @Override - public MultilingualE5SmallInternalServiceSettings build() { - return new MultilingualE5SmallInternalServiceSettings( - getNumAllocations(), - getNumThreads(), - getModelId(), - getAdaptiveAllocationsSettings() - ); - } - }; - builder.setNumAllocations(requestFields.numAllocations); - builder.setNumThreads(requestFields.numThreads); - builder.setModelId(requestFields.modelId); - builder.setAdaptiveAllocationsSettings(requestFields.adaptiveAllocationsSettings); - return builder; - } - - private record RequestFields( - @Nullable Integer numAllocations, - @Nullable Integer numThreads, - @Nullable String modelId, - @Nullable AdaptiveAllocationsSettings adaptiveAllocationsSettings - ) {} + if (validationException.validationErrors().isEmpty() == false) { + throw validationException; + } - @Override - public boolean isFragment() { - return super.isFragment(); + return baseSettings; } @Override @@ -142,11 +78,6 @@ public String getWriteableName() { return MultilingualE5SmallInternalServiceSettings.NAME; } - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - } - @Override public SimilarityMeasure similarity() { return SIMILARITY; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallModel.java index f22118d00cc29..59e5e9c1550c5 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallModel.java @@ -10,15 +10,11 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.inference.Model; -import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.TaskType; import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; -import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; -import static org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus.State.STARTED; - -public class MultilingualE5SmallModel extends Model implements ElasticsearchModel { +public class MultilingualE5SmallModel extends ElasticsearchInternalModel { public MultilingualE5SmallModel( String inferenceEntityId, @@ -26,7 +22,7 @@ public MultilingualE5SmallModel( String service, MultilingualE5SmallInternalServiceSettings serviceSettings ) { - super(new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings)); + super(inferenceEntityId, taskType, service, serviceSettings); } @Override @@ -34,25 +30,6 @@ public MultilingualE5SmallInternalServiceSettings getServiceSettings() { return (MultilingualE5SmallInternalServiceSettings) super.getServiceSettings(); } - @Override - public String getModelId() { - return getServiceSettings().getModelId(); - } - - @Override - public StartTrainedModelDeploymentAction.Request getStartTrainedModelDeploymentActionRequest() { - var startRequest = new StartTrainedModelDeploymentAction.Request( - this.getServiceSettings().getModelId(), - this.getInferenceEntityId() - ); - startRequest.setNumberOfAllocations(this.getServiceSettings().getNumAllocations()); - startRequest.setThreadsPerAllocation(this.getServiceSettings().getNumThreads()); - startRequest.setAdaptiveAllocationsSettings(this.getServiceSettings().getAdaptiveAllocationsSettings()); - startRequest.setWaitForState(STARTED); - - return startRequest; - } - @Override public ActionListener getCreateTrainedModelAssignmentActionListener( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalModel.java index 82c0052e16970..bb668c314649d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalModel.java @@ -7,11 +7,15 @@ package org.elasticsearch.xpack.inference.services.elser; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.inference.Model; -import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalModel; -public class ElserInternalModel extends Model { +public class ElserInternalModel extends ElasticsearchInternalModel { public ElserInternalModel( String inferenceEntityId, @@ -20,7 +24,7 @@ public ElserInternalModel( ElserInternalServiceSettings serviceSettings, ElserMlNodeTaskSettings taskSettings ) { - super(new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings)); + super(inferenceEntityId, taskType, service, serviceSettings, taskSettings); } @Override @@ -32,4 +36,31 @@ public ElserInternalServiceSettings getServiceSettings() { public ElserMlNodeTaskSettings getTaskSettings() { return (ElserMlNodeTaskSettings) super.getTaskSettings(); } + + @Override + public ActionListener getCreateTrainedModelAssignmentActionListener( + Model model, + ActionListener listener + ) { + return new ActionListener<>() { + @Override + public void onResponse(CreateTrainedModelAssignmentAction.Response response) { + listener.onResponse(Boolean.TRUE); + } + + @Override + public void onFailure(Exception e) { + if (ExceptionsHelper.unwrapCause(e) instanceof ResourceNotFoundException) { + listener.onFailure( + new ResourceNotFoundException( + "Could not start the ELSER service as the ELSER model for this platform cannot be found." + + " ELSER needs to be downloaded before it can be started." + ) + ); + return; + } + listener.onFailure(e); + } + }; + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java index 54434a7563dab..03d7682600e7c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java @@ -10,17 +10,14 @@ package org.elasticsearch.xpack.inference.services.elser; import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.inference.ChunkingOptions; import org.elasticsearch.inference.InferenceResults; -import org.elasticsearch.inference.InferenceService; import org.elasticsearch.inference.InferenceServiceExtension; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; @@ -28,39 +25,30 @@ import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; -import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction; import org.elasticsearch.xpack.core.ml.action.InferModelAction; -import org.elasticsearch.xpack.core.ml.action.PutTrainedModelAction; -import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; -import org.elasticsearch.xpack.core.ml.action.StopTrainedModelDeploymentAction; -import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig; -import org.elasticsearch.xpack.core.ml.inference.TrainedModelInput; import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextExpansionConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate; -import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.inference.services.ServiceUtils; +import org.elasticsearch.xpack.inference.services.elasticsearch.BaseElasticsearchInternalService; -import java.io.IOException; import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Set; import static org.elasticsearch.xpack.core.ClientHelper.INFERENCE_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; -import static org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus.State.STARTED; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; -import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.buildInferenceRequest; -public class ElserInternalService implements InferenceService { +public class ElserInternalService extends BaseElasticsearchInternalService { public static final String NAME = "elser"; @@ -77,14 +65,13 @@ public class ElserInternalService implements InferenceService { private static final String OLD_MODEL_ID_FIELD_NAME = "model_version"; - private final OriginSettingClient client; - public ElserInternalService(InferenceServiceExtension.InferenceServiceFactoryContext context) { - this.client = new OriginSettingClient(context.client(), ClientHelper.INFERENCE_ORIGIN); + super(context); } - public boolean isInClusterService() { - return true; + @Override + protected EnumSet supportedTaskTypes() { + return EnumSet.of(TaskType.SPARSE_EMBEDDING); } @Override @@ -97,10 +84,12 @@ public void parseRequestConfig( ) { try { Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); - var serviceSettingsBuilder = ElserInternalServiceSettings.fromMap(serviceSettingsMap); + var serviceSettingsBuilder = ElserInternalServiceSettings.fromRequestMap(serviceSettingsMap); if (serviceSettingsBuilder.getModelId() == null) { - serviceSettingsBuilder.setModelId(selectDefaultModelVersionBasedOnClusterArchitecture(modelArchitectures)); + serviceSettingsBuilder.setModelId( + selectDefaultModelVariantBasedOnClusterArchitecture(modelArchitectures, ELSER_V2_MODEL_LINUX_X86, ELSER_V2_MODEL) + ); } Map taskSettingsMap; @@ -122,7 +111,7 @@ public void parseRequestConfig( inferenceEntityId, taskType, NAME, - (ElserInternalServiceSettings) serviceSettingsBuilder.build(), + new ElserInternalServiceSettings(serviceSettingsBuilder.build()), taskSettings ) ); @@ -131,18 +120,6 @@ public void parseRequestConfig( } } - private static String selectDefaultModelVersionBasedOnClusterArchitecture(Set modelArchitectures) { - // choose a default model ID based on the cluster architecture - boolean homogenous = modelArchitectures.size() == 1; - if (homogenous && modelArchitectures.iterator().next().equals("linux-x86_64")) { - // Use the hardware optimized model - return ELSER_V2_MODEL_LINUX_X86; - } else { - // default to the platform-agnostic model - return ELSER_V2_MODEL; - } - } - @Override public ElserInternalModel parsePersistedConfigWithSecrets( String inferenceEntityId, @@ -164,7 +141,7 @@ public ElserInternalModel parsePersistedConfig(String inferenceEntityId, TaskTyp serviceSettingsMap.put(ElserInternalServiceSettings.MODEL_ID, modelId); } - var serviceSettingsBuilder = ElserInternalServiceSettings.fromMap(serviceSettingsMap); + var serviceSettings = ElserInternalServiceSettings.fromPersistedMap(serviceSettingsMap); Map taskSettingsMap; // task settings are optional @@ -176,85 +153,7 @@ public ElserInternalModel parsePersistedConfig(String inferenceEntityId, TaskTyp var taskSettings = taskSettingsFromMap(taskType, taskSettingsMap); - return new ElserInternalModel( - inferenceEntityId, - taskType, - NAME, - (ElserInternalServiceSettings) serviceSettingsBuilder.build(), - taskSettings - ); - } - - @Override - public void start(Model model, ActionListener listener) { - if (model instanceof ElserInternalModel == false) { - listener.onFailure( - new IllegalStateException( - "Error starting model, [" + model.getConfigurations().getInferenceEntityId() + "] is not an ELSER model" - ) - ); - return; - } - - if (model.getConfigurations().getTaskType() != TaskType.SPARSE_EMBEDDING) { - listener.onFailure( - new IllegalStateException(TaskType.unsupportedTaskTypeErrorMsg(model.getConfigurations().getTaskType(), NAME)) - ); - return; - } - - client.execute(StartTrainedModelDeploymentAction.INSTANCE, startDeploymentRequest(model), elserNotDownloadedListener(listener)); - } - - private static StartTrainedModelDeploymentAction.Request startDeploymentRequest(Model model) { - var elserModel = (ElserInternalModel) model; - var serviceSettings = elserModel.getServiceSettings(); - - var startRequest = new StartTrainedModelDeploymentAction.Request( - serviceSettings.getModelId(), - model.getConfigurations().getInferenceEntityId() - ); - startRequest.setNumberOfAllocations(serviceSettings.getNumAllocations()); - startRequest.setThreadsPerAllocation(serviceSettings.getNumThreads()); - startRequest.setAdaptiveAllocationsSettings(serviceSettings.getAdaptiveAllocationsSettings()); - startRequest.setWaitForState(STARTED); - return startRequest; - } - - private static ActionListener elserNotDownloadedListener( - ActionListener listener - ) { - return new ActionListener<>() { - @Override - public void onResponse(CreateTrainedModelAssignmentAction.Response response) { - listener.onResponse(Boolean.TRUE); - } - - @Override - public void onFailure(Exception e) { - if (ExceptionsHelper.unwrapCause(e) instanceof ResourceNotFoundException) { - listener.onFailure( - new ResourceNotFoundException( - "Could not start the ELSER service as the ELSER model for this platform cannot be found." - + " ELSER needs to be downloaded before it can be started." - ) - ); - return; - } - listener.onFailure(e); - } - }; - } - - @Override - public void stop(String inferenceEntityId, ActionListener listener) { - var request = new StopTrainedModelDeploymentAction.Request(inferenceEntityId); - request.setForce(true); - client.execute( - StopTrainedModelDeploymentAction.INSTANCE, - request, - listener.delegateFailureAndWrap((delegatedResponseListener, response) -> delegatedResponseListener.onResponse(Boolean.TRUE)) - ); + return new ElserInternalModel(inferenceEntityId, taskType, NAME, new ElserInternalServiceSettings(serviceSettings), taskSettings); } @Override @@ -352,32 +251,6 @@ private void checkCompatibleTaskType(TaskType taskType) { } } - @Override - public void putModel(Model model, ActionListener listener) { - if (model instanceof ElserInternalModel == false) { - listener.onFailure( - new IllegalStateException( - "Error starting model, [" + model.getConfigurations().getInferenceEntityId() + "] is not an ELSER model" - ) - ); - return; - } else { - String modelId = ((ElserInternalModel) model).getServiceSettings().getModelId(); - var input = new TrainedModelInput(List.of("text_field")); // by convention text_field is used - var config = TrainedModelConfig.builder().setInput(input).setModelId(modelId).validate(true).build(); - PutTrainedModelAction.Request putRequest = new PutTrainedModelAction.Request(config, false, true); - executeAsyncWithOrigin( - client, - INFERENCE_ORIGIN, - PutTrainedModelAction.INSTANCE, - putRequest, - listener.delegateFailure((l, r) -> { - l.onResponse(Boolean.TRUE); - }) - ); - } - } - @Override public void isModelDownloaded(Model model, ActionListener listener) { ActionListener getModelsResponseListener = listener.delegateFailure((delegate, response) -> { @@ -389,7 +262,7 @@ public void isModelDownloaded(Model model, ActionListener listener) { }); if (model instanceof ElserInternalModel elserModel) { - String modelId = elserModel.getServiceSettings().getModelId(); + String modelId = elserModel.getServiceSettings().modelId(); GetTrainedModelsAction.Request getRequest = new GetTrainedModelsAction.Request(modelId); executeAsyncWithOrigin(client, INFERENCE_ORIGIN, GetTrainedModelsAction.INSTANCE, getRequest, getModelsResponseListener); } else { @@ -437,9 +310,6 @@ public String name() { return NAME; } - @Override - public void close() throws IOException {} - @Override public TransportVersion getMinimalSupportedVersion() { return TransportVersions.V_8_12_0; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettings.java index fcbf7394ccb33..75797919b3616 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettings.java @@ -9,102 +9,56 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; -import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; -import org.elasticsearch.xpack.inference.services.ServiceUtils; -import org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings; import java.io.IOException; +import java.util.Arrays; import java.util.Map; -import java.util.Objects; -import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalString; -import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredPositiveInteger; +import static org.elasticsearch.xpack.inference.services.elser.ElserInternalService.VALID_ELSER_MODEL_IDS; -public class ElserInternalServiceSettings extends InternalServiceSettings { +public class ElserInternalServiceSettings extends ElasticsearchInternalServiceSettings { public static final String NAME = "elser_mlnode_service_settings"; - /** - * Parse the Elser service setting from map and validate the setting values. - * - * If required setting are missing or the values are invalid an - * {@link ValidationException} is thrown. - * - * @param map Source map containing the config - * @return The {@code ElserInternalServiceSettings} - */ - public static ElserInternalServiceSettings.Builder fromMap(Map map) { + public static ElasticsearchInternalServiceSettings.Builder fromRequestMap(Map map) { ValidationException validationException = new ValidationException(); - - Integer numAllocations = extractRequiredPositiveInteger( - map, - NUM_ALLOCATIONS, - ModelConfigurations.SERVICE_SETTINGS, - validationException - ); - Integer numThreads = extractRequiredPositiveInteger(map, NUM_THREADS, ModelConfigurations.SERVICE_SETTINGS, validationException); - AdaptiveAllocationsSettings adaptiveAllocationsSettings = ServiceUtils.removeAsAdaptiveAllocationsSettings( - map, - ADAPTIVE_ALLOCATIONS - ); - if (adaptiveAllocationsSettings != null) { - ActionRequestValidationException exception = adaptiveAllocationsSettings.validate(); - if (exception != null) { - validationException.addValidationErrors(exception.validationErrors()); - } - } - String modelId = extractOptionalString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException); - - if (modelId != null && ElserInternalService.VALID_ELSER_MODEL_IDS.contains(modelId) == false) { - validationException.addValidationError("unknown ELSER model id [" + modelId + "]"); + var baseSettings = ElasticsearchInternalServiceSettings.fromMap(map, validationException); + + String modelId = baseSettings.getModelId(); + if (modelId != null && VALID_ELSER_MODEL_IDS.contains(modelId) == false) { + var ve = new ValidationException(); + ve.addValidationError( + "Unknown ELSER model ID [" + modelId + "]. Valid models are " + Arrays.toString(VALID_ELSER_MODEL_IDS.toArray()) + ); + throw ve; } if (validationException.validationErrors().isEmpty() == false) { throw validationException; } - var builder = new InternalServiceSettings.Builder() { - @Override - public ElserInternalServiceSettings build() { - return new ElserInternalServiceSettings( - getNumAllocations(), - getNumThreads(), - getModelId(), - getAdaptiveAllocationsSettings() - ); - } - }; - builder.setNumAllocations(numAllocations); - builder.setNumThreads(numThreads); - builder.setAdaptiveAllocationsSettings(adaptiveAllocationsSettings); - builder.setModelId(modelId); - return builder; + return baseSettings; + } + + public ElserInternalServiceSettings(ElasticsearchInternalServiceSettings other) { + super(other); } public ElserInternalServiceSettings( - int numAllocations, + Integer numAllocations, int numThreads, String modelId, AdaptiveAllocationsSettings adaptiveAllocationsSettings ) { - super(numAllocations, numThreads, modelId, adaptiveAllocationsSettings); - Objects.requireNonNull(modelId); + this(new ElasticsearchInternalServiceSettings(numAllocations, numThreads, modelId, adaptiveAllocationsSettings)); } public ElserInternalServiceSettings(StreamInput in) throws IOException { - super( - in.readVInt(), - in.readVInt(), - in.getTransportVersion().onOrAfter(TransportVersions.V_8_11_X) ? in.readString() : ElserInternalService.ELSER_V2_MODEL, - in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS) - ? in.readOptionalWriteable(AdaptiveAllocationsSettings::new) - : null - ); + super(in); } @Override @@ -116,32 +70,4 @@ public String getWriteableName() { public TransportVersion getMinimalSupportedVersion() { return TransportVersions.V_8_11_X; } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(getNumAllocations()); - out.writeVInt(getNumThreads()); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_11_X)) { - out.writeString(getModelId()); - } - if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { - out.writeOptionalWriteable(getAdaptiveAllocationsSettings()); - } - } - - @Override - public int hashCode() { - return Objects.hash(NAME, getNumAllocations(), getNumThreads(), getModelId(), getAdaptiveAllocationsSettings()); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ElserInternalServiceSettings that = (ElserInternalServiceSettings) o; - return getNumAllocations() == that.getNumAllocations() - && getNumThreads() == that.getNumThreads() - && Objects.equals(getModelId(), that.getModelId()) - && Objects.equals(getAdaptiveAllocationsSettings(), that.getAdaptiveAllocationsSettings()); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsServiceSettings.java index f4bf40d290399..097ce6240439b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsServiceSettings.java @@ -164,6 +164,7 @@ public String location() { return location; } + @Override public String modelId() { return modelId; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankServiceSettings.java index 0a0271d611a71..431f704c10091 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankServiceSettings.java @@ -82,6 +82,7 @@ public String projectId() { return projectId; } + @Override public String modelId() { return modelId; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettings.java index eb9c99f5bfd91..c136e73a0c452 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettings.java @@ -196,6 +196,11 @@ public Integer dimensions() { return dimensions; } + @Override + public String modelId() { + return null; + } + public Integer maxInputTokens() { return maxInputTokens; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettings.java index 8b4bd61649de0..b48e71867ea6d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettings.java @@ -97,6 +97,11 @@ public int maxInputTokens() { return ELSER_TOKEN_LIMIT; } + @Override + public String modelId() { + return null; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java index d85b2b095ba2c..18f2570946662 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java @@ -245,7 +245,7 @@ private MistralEmbeddingsModel updateEmbeddingModelConfig(MistralEmbeddingsModel var similarityToUse = similarityFromModel == null ? SimilarityMeasure.DOT_PRODUCT : similarityFromModel; MistralEmbeddingsServiceSettings serviceSettings = new MistralEmbeddingsServiceSettings( - embeddingServiceSettings.model(), + embeddingServiceSettings.modelId(), embeddingsSize, embeddingServiceSettings.maxInputTokens(), similarityToUse, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsModel.java index 2631dfecccab3..d883e7c687c20 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsModel.java @@ -71,7 +71,7 @@ public MistralEmbeddingsModel( } private void setPropertiesFromServiceSettings(MistralEmbeddingsServiceSettings serviceSettings) { - this.model = serviceSettings.model(); + this.model = serviceSettings.modelId(); this.rateLimitSettings = serviceSettings.rateLimitSettings(); setEndpointUrl(); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettings.java index 2e4d546e1dc4c..3310b8624e723 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettings.java @@ -107,7 +107,8 @@ public TransportVersion getMinimalSupportedVersion() { return ADD_MISTRAL_EMBEDDINGS_INFERENCE; } - public String model() { + @Override + public String modelId() { return this.model; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java index c4ab8bd99b8b0..96436996098f8 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java @@ -187,7 +187,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_COMPLETION_INFERENCE_SERVICE_ADDED; + return TransportVersions.V_8_14_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettings.java index 2d5a407f3c1a6..922bbcda2c746 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettings.java @@ -82,7 +82,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_COMPLETION_INFERENCE_SERVICE_ADDED; + return TransportVersions.V_8_14_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/InternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/InternalServiceSettings.java deleted file mode 100644 index 2cbe2f930c84d..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/InternalServiceSettings.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.services.settings; - -import org.elasticsearch.TransportVersions; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.inference.ServiceSettings; -import org.elasticsearch.xcontent.ToXContentObject; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; - -import java.io.IOException; -import java.util.Objects; - -public abstract class InternalServiceSettings implements ServiceSettings { - - public static final String NUM_ALLOCATIONS = "num_allocations"; - public static final String NUM_THREADS = "num_threads"; - public static final String MODEL_ID = "model_id"; - public static final String ADAPTIVE_ALLOCATIONS = "adaptive_allocations"; - - private final int numAllocations; - private final int numThreads; - private final String modelId; - private final AdaptiveAllocationsSettings adaptiveAllocationsSettings; - - public InternalServiceSettings( - int numAllocations, - int numThreads, - String modelId, - AdaptiveAllocationsSettings adaptiveAllocationsSettings - ) { - this.numAllocations = numAllocations; - this.numThreads = numThreads; - this.modelId = modelId; - this.adaptiveAllocationsSettings = adaptiveAllocationsSettings; - } - - public int getNumAllocations() { - return numAllocations; - } - - public int getNumThreads() { - return numThreads; - } - - public String getModelId() { - return modelId; - } - - public AdaptiveAllocationsSettings getAdaptiveAllocationsSettings() { - return adaptiveAllocationsSettings; - } - - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - InternalServiceSettings that = (InternalServiceSettings) o; - return numAllocations == that.numAllocations - && numThreads == that.numThreads - && Objects.equals(modelId, that.modelId) - && Objects.equals(adaptiveAllocationsSettings, that.adaptiveAllocationsSettings); - } - - @Override - public int hashCode() { - return Objects.hash(numAllocations, numThreads, modelId, adaptiveAllocationsSettings); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - addXContentFragment(builder, params); - builder.endObject(); - return builder; - } - - public void addXContentFragment(XContentBuilder builder, Params params) throws IOException { - builder.field(NUM_ALLOCATIONS, getNumAllocations()); - builder.field(NUM_THREADS, getNumThreads()); - builder.field(MODEL_ID, getModelId()); - builder.field(ADAPTIVE_ALLOCATIONS, getAdaptiveAllocationsSettings()); - } - - @Override - public ToXContentObject getFilteredXContentObject() { - return this; - } - - @Override - public boolean isFragment() { - return ServiceSettings.super.isFragment(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(getNumAllocations()); - out.writeVInt(getNumThreads()); - out.writeString(getModelId()); - if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { - out.writeOptionalWriteable(getAdaptiveAllocationsSettings()); - } - } - - public abstract static class Builder { - private int numAllocations; - private int numThreads; - private String modelId; - private AdaptiveAllocationsSettings adaptiveAllocationsSettings; - - public abstract InternalServiceSettings build(); - - public void setNumAllocations(int numAllocations) { - this.numAllocations = numAllocations; - } - - public void setNumThreads(int numThreads) { - this.numThreads = numThreads; - } - - public void setModelId(String modelId) { - this.modelId = modelId; - } - - public void setAdaptiveAllocationsSettings(AdaptiveAllocationsSettings adaptiveAllocationsSettings) { - this.adaptiveAllocationsSettings = adaptiveAllocationsSettings; - } - - public String getModelId() { - return modelId; - } - - public int getNumAllocations() { - return numAllocations; - } - - public int getNumThreads() { - return numThreads; - } - - public AdaptiveAllocationsSettings getAdaptiveAllocationsSettings() { - return adaptiveAllocationsSettings; - } - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/InferenceAPMStats.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/InferenceAPMStats.java new file mode 100644 index 0000000000000..76977fef76045 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/InferenceAPMStats.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.telemetry; + +import org.elasticsearch.inference.Model; +import org.elasticsearch.telemetry.metric.LongCounter; +import org.elasticsearch.telemetry.metric.MeterRegistry; + +import java.util.Map; +import java.util.Objects; + +public class InferenceAPMStats extends InferenceStats { + + private final LongCounter inferenceAPMRequestCounter; + + public InferenceAPMStats(Model model, MeterRegistry meterRegistry) { + super(model); + this.inferenceAPMRequestCounter = meterRegistry.registerLongCounter( + "es.inference.requests.count", + "Inference API request counts for a particular service, task type, model ID", + "operations" + ); + } + + @Override + public void increment() { + super.increment(); + inferenceAPMRequestCounter.incrementBy(1, Map.of("service", service, "task_type", taskType.toString(), "model_id", modelId)); + } + + public static final class Factory { + private final MeterRegistry meterRegistry; + + public Factory(MeterRegistry meterRegistry) { + this.meterRegistry = Objects.requireNonNull(meterRegistry); + } + + public InferenceAPMStats newInferenceRequestAPMCounter(Model model) { + return new InferenceAPMStats(model, meterRegistry); + } + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/InferenceStats.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/InferenceStats.java new file mode 100644 index 0000000000000..d639f9da71f56 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/InferenceStats.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.telemetry; + +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.xpack.core.inference.InferenceRequestStats; + +import java.util.Objects; +import java.util.concurrent.atomic.LongAdder; + +public class InferenceStats implements Stats { + protected final String service; + protected final TaskType taskType; + protected final String modelId; + protected final LongAdder counter = new LongAdder(); + + public static String key(Model model) { + StringBuilder builder = new StringBuilder(); + builder.append(model.getConfigurations().getService()); + builder.append(":"); + builder.append(model.getTaskType()); + + if (model.getServiceSettings().modelId() != null) { + builder.append(":"); + builder.append(model.getServiceSettings().modelId()); + } + + return builder.toString(); + } + + public InferenceStats(Model model) { + Objects.requireNonNull(model); + + service = model.getConfigurations().getService(); + taskType = model.getTaskType(); + modelId = model.getServiceSettings().modelId(); + } + + @Override + public void increment() { + counter.increment(); + } + + @Override + public long getCount() { + return counter.sum(); + } + + @Override + public InferenceRequestStats toSerializableForm() { + return new InferenceRequestStats(service, taskType, modelId, getCount()); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/Stats.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/Stats.java new file mode 100644 index 0000000000000..bb1e9c98fc2cb --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/Stats.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.telemetry; + +import org.elasticsearch.xpack.core.inference.SerializableStats; + +public interface Stats { + + /** + * Increase the counter by one. + */ + void increment(); + + /** + * Return the current value of the counter. + * @return the current value of the counter + */ + long getCount(); + + /** + * Convert the object into a serializable form that can be written across nodes and returned in xcontent format. + * @return the serializable format of the object + */ + SerializableStats toSerializableForm(); +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/StatsMap.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/StatsMap.java new file mode 100644 index 0000000000000..1cfecfb4507d6 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/StatsMap.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.telemetry; + +import org.elasticsearch.xpack.core.inference.SerializableStats; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * A map to provide tracking incrementing statistics. + * + * @param The input to derive the keys and values for the map + * @param The type of the values stored in the map + */ +public class StatsMap { + + private final ConcurrentMap stats = new ConcurrentHashMap<>(); + private final Function keyCreator; + private final Function valueCreator; + + /** + * @param keyCreator a function for creating a key in the map based on the input provided + * @param valueCreator a function for creating a value in the map based on the input provided + */ + public StatsMap(Function keyCreator, Function valueCreator) { + this.keyCreator = Objects.requireNonNull(keyCreator); + this.valueCreator = Objects.requireNonNull(valueCreator); + } + + /** + * Increment the counter for a particular value in a thread safe manner. + * @param input the input to derive the appropriate key in the map + */ + public void increment(Input input) { + var value = stats.computeIfAbsent(keyCreator.apply(input), key -> valueCreator.apply(input)); + value.increment(); + } + + /** + * Build a map that can be serialized. This takes a snapshot of the current state. Any concurrent calls to increment may or may not + * be represented in the resulting serializable map. + * @return a map that is more easily serializable + */ + public Map toSerializableMap() { + return stats.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().toSerializableForm())); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java index fe33a3d092667..fb841bd6953cb 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java @@ -65,11 +65,11 @@ public static ClusterService mockClusterService(Settings settings) { var clusterService = mock(ClusterService.class); var registeredSettings = Stream.of( - HttpSettings.getSettings(), - HttpClientManager.getSettings(), - ThrottlerManager.getSettings(), + HttpSettings.getSettingsDefinitions(), + HttpClientManager.getSettingsDefinitions(), + ThrottlerManager.getSettingsDefinitions(), RetrySettings.getSettingsDefinitions(), - Truncator.getSettings(), + Truncator.getSettingsDefinitions(), RequestExecutorServiceSettings.getSettingsDefinitions() ).flatMap(Collection::stream).collect(Collectors.toSet()); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/SenderExecutableActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/SenderExecutableActionTests.java new file mode 100644 index 0000000000000..3b95c4ba86e59 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/SenderExecutableActionTests.java @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.action; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; +import org.elasticsearch.xpack.inference.external.http.sender.RequestManager; +import org.elasticsearch.xpack.inference.external.http.sender.Sender; +import org.junit.Before; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; + +public class SenderExecutableActionTests extends ESTestCase { + private static final String failedToSendRequestErrorMessage = "test failed"; + private Sender sender; + private RequestManager requestManager; + private SenderExecutableAction executableAction; + + @Before + public void setUpMocks() { + sender = mock(Sender.class); + requestManager = mock(RequestManager.class); + executableAction = new SenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage); + } + + public void testSendCompletesSuccessfully() { + var testRan = new AtomicBoolean(false); + + mockSender(listener -> listener.onResponse(mock(InferenceServiceResults.class))); + + executableAction.execute( + mock(InferenceInputs.class), + mock(TimeValue.class), + ActionListener.wrap(success -> testRan.set(true), e -> fail(e, "Test failed.")) + ); + + assertTrue("Test failed to call listener.", testRan.get()); + } + + @SuppressWarnings("unchecked") + public void testSendThrowingElasticsearchExceptionIsUnwrapped() { + var expectedException = new ElasticsearchException("test"); + var actualException = new AtomicReference(); + + doThrow(expectedException).when(sender) + .send(eq(requestManager), any(InferenceInputs.class), any(TimeValue.class), any(ActionListener.class)); + + execute(actualException); + + assertThat(actualException.get(), notNullValue()); + assertThat(actualException.get(), sameInstance(expectedException)); + } + + public void testSenderReturnedElasticsearchExceptionIsUnwrapped() { + var expectedException = new ElasticsearchException("test"); + var actualException = new AtomicReference(); + + mockSender(listener -> listener.onFailure(expectedException)); + + execute(actualException); + + assertThat(actualException.get(), notNullValue()); + assertThat(actualException.get(), sameInstance(expectedException)); + } + + @SuppressWarnings("unchecked") + public void testSendThrowingExceptionIsWrapped() { + var expectedException = new IllegalStateException("test"); + var actualException = new AtomicReference(); + + doThrow(expectedException).when(sender) + .send(eq(requestManager), any(InferenceInputs.class), any(TimeValue.class), any(ActionListener.class)); + + execute(actualException); + + assertThat(actualException.get(), notNullValue()); + assertThat(actualException.get().getMessage(), is(failedToSendRequestErrorMessage)); + assertThat(actualException.get(), instanceOf(ElasticsearchStatusException.class)); + assertThat(actualException.get().getCause(), sameInstance(expectedException)); + } + + public void testSenderReturnedExceptionIsWrapped() { + var expectedException = new IllegalStateException("test"); + var actualException = new AtomicReference(); + + mockSender(listener -> listener.onFailure(expectedException)); + + execute(actualException); + + assertThat(actualException.get(), notNullValue()); + assertThat(actualException.get().getMessage(), is(failedToSendRequestErrorMessage)); + assertThat(actualException.get(), instanceOf(ElasticsearchStatusException.class)); + assertThat(actualException.get().getCause(), sameInstance(expectedException)); + } + + @SuppressWarnings("unchecked") + private void mockSender(Consumer> listener) { + doAnswer(ans -> { + listener.accept(ans.getArgument(3, ActionListener.class)); + return null; // void + }).when(sender).send(eq(requestManager), any(InferenceInputs.class), any(TimeValue.class), any(ActionListener.class)); + } + + private void execute(AtomicReference actualException) { + executableAction.execute( + mock(InferenceInputs.class), + mock(TimeValue.class), + ActionListener.wrap(shouldNotSucceed -> fail("Test failed."), actualException::set) + ); + } + +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/SingleInputSenderExecutableActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/SingleInputSenderExecutableActionTests.java new file mode 100644 index 0000000000000..d4ab9b1f1e19a --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/SingleInputSenderExecutableActionTests.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.action; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; +import org.elasticsearch.xpack.inference.external.http.sender.RequestManager; +import org.elasticsearch.xpack.inference.external.http.sender.Sender; +import org.junit.Before; + +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class SingleInputSenderExecutableActionTests extends ESTestCase { + private static final String errorMessage = "test"; + private SingleInputSenderExecutableAction executableAction; + + @Before + @SuppressWarnings("unchecked") + public void setUpMocks() { + var sender = mock(Sender.class); + var requestManager = mock(RequestManager.class); + executableAction = new SingleInputSenderExecutableAction(sender, requestManager, errorMessage, errorMessage); + + doAnswer(ans -> { + ans.getArgument(3, ActionListener.class).onResponse(mock(InferenceServiceResults.class)); + return null; // void + }).when(sender).send(eq(requestManager), any(InferenceInputs.class), any(TimeValue.class), any(ActionListener.class)); + } + + public void testOneInputIsValid() { + var testRan = new AtomicBoolean(false); + + executableAction.execute( + mock(DocumentsOnlyInput.class), + mock(TimeValue.class), + ActionListener.wrap(success -> testRan.set(true), e -> fail(e, "Test failed.")) + ); + + assertTrue("Test failed to call listener.", testRan.get()); + } + + public void testInvalidInputType() { + var badInput = mock(InferenceInputs.class); + var actualException = new AtomicReference(); + + executableAction.execute( + badInput, + mock(TimeValue.class), + ActionListener.wrap(shouldNotSucceed -> fail("Test failed."), actualException::set) + ); + + assertThat(actualException.get(), notNullValue()); + assertThat(actualException.get().getMessage(), is("Invalid inference input type")); + assertThat(actualException.get(), instanceOf(ElasticsearchStatusException.class)); + assertThat(((ElasticsearchStatusException) actualException.get()).status(), is(RestStatus.INTERNAL_SERVER_ERROR)); + } + + public void testMoreThanOneInput() { + var badInput = mock(DocumentsOnlyInput.class); + when(badInput.getInputs()).thenReturn(List.of("one", "two")); + var actualException = new AtomicReference(); + + executableAction.execute( + badInput, + mock(TimeValue.class), + ActionListener.wrap(shouldNotSucceed -> fail("Test failed."), actualException::set) + ); + + assertThat(actualException.get(), notNullValue()); + assertThat(actualException.get().getMessage(), is("test only accepts 1 input")); + assertThat(actualException.get(), instanceOf(ElasticsearchStatusException.class)); + assertThat(((ElasticsearchStatusException) actualException.get()).status(), is(RestStatus.BAD_REQUEST)); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicChatCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicChatCompletionActionTests.java index ffa0ac307490e..fca2e316af17f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicChatCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicChatCompletionActionTests.java @@ -23,7 +23,10 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SingleInputSenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.sender.AnthropicCompletionRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; @@ -42,6 +45,7 @@ import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; import static org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests.createSender; @@ -229,14 +233,15 @@ public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOExc var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is("Anthropic completions only accepts 1 input")); + assertThat(thrownException.getMessage(), is("Anthropic chat completions only accepts 1 input")); assertThat(thrownException.status(), is(RestStatus.BAD_REQUEST)); } } - private AnthropicChatCompletionAction createAction(String url, String apiKey, String modelName, int maxTokens, Sender sender) { + private ExecutableAction createAction(String url, String apiKey, String modelName, int maxTokens, Sender sender) { var model = AnthropicChatCompletionModelTests.createChatCompletionModel(url, apiKey, modelName, maxTokens); - - return new AnthropicChatCompletionAction(sender, model, createWithEmptySettings(threadPool)); + var requestCreator = AnthropicCompletionRequestManager.of(model, threadPool); + var errorMessage = constructFailedToSendRequestMessage(model.getUri(), "Anthropic chat completions"); + return new SingleInputSenderExecutableAction(sender, requestCreator, errorMessage, "Anthropic chat completions"); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreatorTests.java index 72124a6221254..45a2fb0954c79 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreatorTests.java @@ -112,7 +112,7 @@ public void testCreate_AzureOpenAiEmbeddingsModel() throws IOException { model.setUri(new URI(getUrl(webServer))); var actionCreator = new AzureOpenAiActionCreator(sender, createWithEmptySettings(threadPool)); var overriddenTaskSettings = createRequestTaskSettingsMap("overridden_user"); - var action = (AzureOpenAiEmbeddingsAction) actionCreator.create(model, overriddenTaskSettings); + var action = actionCreator.create(model, overriddenTaskSettings); PlainActionFuture listener = new PlainActionFuture<>(); action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); @@ -162,7 +162,7 @@ public void testCreate_AzureOpenAiEmbeddingsModel_WithoutUser() throws IOExcepti model.setUri(new URI(getUrl(webServer))); var actionCreator = new AzureOpenAiActionCreator(sender, createWithEmptySettings(threadPool)); var overriddenTaskSettings = createRequestTaskSettingsMap(null); - var action = (AzureOpenAiEmbeddingsAction) actionCreator.create(model, overriddenTaskSettings); + var action = actionCreator.create(model, overriddenTaskSettings); PlainActionFuture listener = new PlainActionFuture<>(); action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); @@ -213,7 +213,7 @@ public void testCreate_AzureOpenAiEmbeddingsModel_FailsFromInvalidResponseFormat model.setUri(new URI(getUrl(webServer))); var actionCreator = new AzureOpenAiActionCreator(sender, createWithEmptySettings(threadPool)); var overriddenTaskSettings = createRequestTaskSettingsMap("overridden_user"); - var action = (AzureOpenAiEmbeddingsAction) actionCreator.create(model, overriddenTaskSettings); + var action = actionCreator.create(model, overriddenTaskSettings); PlainActionFuture listener = new PlainActionFuture<>(); action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); @@ -285,7 +285,7 @@ public void testExecute_ReturnsSuccessfulResponse_AfterTruncating_From413StatusC model.setUri(new URI(getUrl(webServer))); var actionCreator = new AzureOpenAiActionCreator(sender, createWithEmptySettings(threadPool)); var overriddenTaskSettings = createRequestTaskSettingsMap("overridden_user"); - var action = (AzureOpenAiEmbeddingsAction) actionCreator.create(model, overriddenTaskSettings); + var action = actionCreator.create(model, overriddenTaskSettings); PlainActionFuture listener = new PlainActionFuture<>(); action.execute(new DocumentsOnlyInput(List.of("abcd")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); @@ -361,7 +361,7 @@ public void testExecute_ReturnsSuccessfulResponse_AfterTruncating_From400StatusC model.setUri(new URI(getUrl(webServer))); var actionCreator = new AzureOpenAiActionCreator(sender, createWithEmptySettings(threadPool)); var overriddenTaskSettings = createRequestTaskSettingsMap("overridden_user"); - var action = (AzureOpenAiEmbeddingsAction) actionCreator.create(model, overriddenTaskSettings); + var action = actionCreator.create(model, overriddenTaskSettings); PlainActionFuture listener = new PlainActionFuture<>(); action.execute(new DocumentsOnlyInput(List.of("abcd")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); @@ -420,7 +420,7 @@ public void testExecute_TruncatesInputBeforeSending() throws IOException { model.setUri(new URI(getUrl(webServer))); var actionCreator = new AzureOpenAiActionCreator(sender, createWithEmptySettings(threadPool)); var overriddenTaskSettings = createRequestTaskSettingsMap("overridden_user"); - var action = (AzureOpenAiEmbeddingsAction) actionCreator.create(model, overriddenTaskSettings); + var action = actionCreator.create(model, overriddenTaskSettings); PlainActionFuture listener = new PlainActionFuture<>(); action.execute(new DocumentsOnlyInput(List.of("super long input")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); @@ -472,7 +472,7 @@ public void testInfer_AzureOpenAiCompletion_WithOverriddenUser() throws IOExcept model.setUri(new URI(getUrl(webServer))); var actionCreator = new AzureOpenAiActionCreator(sender, createWithEmptySettings(threadPool)); var taskSettingsWithUserOverride = createRequestTaskSettingsMap(overriddenUser); - var action = (AzureOpenAiCompletionAction) actionCreator.create(model, taskSettingsWithUserOverride); + var action = actionCreator.create(model, taskSettingsWithUserOverride); PlainActionFuture listener = new PlainActionFuture<>(); action.execute(new DocumentsOnlyInput(List.of(completionInput)), InferenceAction.Request.DEFAULT_TIMEOUT, listener); @@ -528,7 +528,7 @@ public void testInfer_AzureOpenAiCompletionModel_WithoutUser() throws IOExceptio model.setUri(new URI(getUrl(webServer))); var actionCreator = new AzureOpenAiActionCreator(sender, createWithEmptySettings(threadPool)); var requestTaskSettingsWithoutUser = createRequestTaskSettingsMap(null); - var action = (AzureOpenAiCompletionAction) actionCreator.create(model, requestTaskSettingsWithoutUser); + var action = actionCreator.create(model, requestTaskSettingsWithoutUser); PlainActionFuture listener = new PlainActionFuture<>(); action.execute(new DocumentsOnlyInput(List.of(completionInput)), InferenceAction.Request.DEFAULT_TIMEOUT, listener); @@ -586,7 +586,7 @@ public void testInfer_AzureOpenAiCompletionModel_FailsFromInvalidResponseFormat( model.setUri(new URI(getUrl(webServer))); var actionCreator = new AzureOpenAiActionCreator(sender, createWithEmptySettings(threadPool)); var requestTaskSettingsWithoutUser = createRequestTaskSettingsMap(userOverride); - var action = (AzureOpenAiCompletionAction) actionCreator.create(model, requestTaskSettingsWithoutUser); + var action = actionCreator.create(model, requestTaskSettingsWithoutUser); PlainActionFuture listener = new PlainActionFuture<>(); action.execute(new DocumentsOnlyInput(List.of(completionInput)), InferenceAction.Request.DEFAULT_TIMEOUT, listener); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionActionTests.java index 7d52616402405..4c7683c882816 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionActionTests.java @@ -22,7 +22,10 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.results.ChatCompletionResults; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SingleInputSenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.sender.AzureOpenAiCompletionRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; import org.elasticsearch.xpack.inference.external.http.sender.Sender; @@ -41,11 +44,11 @@ import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; import static org.elasticsearch.xpack.inference.external.action.azureopenai.AzureOpenAiActionCreatorTests.getContentOfMessageInRequestMap; import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; import static org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests.createSender; -import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings; import static org.elasticsearch.xpack.inference.services.azureopenai.completion.AzureOpenAiCompletionModelTests.createCompletionModel; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; @@ -181,7 +184,7 @@ public void testExecute_ThrowsException() { assertThat(thrownException.getMessage(), is(format("Failed to send Azure OpenAI completion request to [%s]", getUrl(webServer)))); } - private AzureOpenAiCompletionAction createAction( + private ExecutableAction createAction( String resourceName, String deploymentId, String apiVersion, @@ -193,7 +196,9 @@ private AzureOpenAiCompletionAction createAction( try { var model = createCompletionModel(resourceName, deploymentId, apiVersion, user, apiKey, null, inferenceEntityId); model.setUri(new URI(getUrl(webServer))); - return new AzureOpenAiCompletionAction(sender, model, createWithEmptySettings(threadPool)); + var requestCreator = new AzureOpenAiCompletionRequestManager(model, threadPool); + var errorMessage = constructFailedToSendRequestMessage(model.getUri(), "Azure OpenAI completion"); + return new SingleInputSenderExecutableAction(sender, requestCreator, errorMessage, "Azure OpenAI completion"); } catch (URISyntaxException e) { throw new RuntimeException(e); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsActionTests.java index 4cc7b7c0d9cfc..4c07ce81eb4cc 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsActionTests.java @@ -21,7 +21,11 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.inference.common.TruncatorTests; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.sender.AzureOpenAiEmbeddingsRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.Sender; @@ -41,11 +45,11 @@ import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; import static org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests.createSender; import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationFloat; -import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings; import static org.elasticsearch.xpack.inference.services.azureopenai.embeddings.AzureOpenAiEmbeddingsModelTests.createModel; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -197,7 +201,7 @@ public void testExecute_ThrowsException() { assertThat(thrownException.getMessage(), is(format("Failed to send Azure OpenAI embeddings request to [%s]", getUrl(webServer)))); } - private AzureOpenAiEmbeddingsAction createAction( + private ExecutableAction createAction( String resourceName, String deploymentId, String apiVersion, @@ -210,8 +214,9 @@ private AzureOpenAiEmbeddingsAction createAction( try { model = createModel(resourceName, deploymentId, apiVersion, user, apiKey, null, inferenceEntityId); model.setUri(new URI(getUrl(webServer))); - var action = new AzureOpenAiEmbeddingsAction(sender, model, createWithEmptySettings(threadPool)); - return action; + var requestCreator = new AzureOpenAiEmbeddingsRequestManager(model, TruncatorTests.createTruncator(), threadPool); + var errorMessage = constructFailedToSendRequestMessage(model.getUri(), "Azure OpenAI embeddings"); + return new SenderExecutableAction(sender, requestCreator, errorMessage); } catch (URISyntaxException e) { throw new RuntimeException(e); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionActionTests.java index 0a604980f6c83..ba839e0d7c5e9 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionActionTests.java @@ -23,7 +23,10 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SingleInputSenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.sender.CohereCompletionRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; import org.elasticsearch.xpack.inference.external.http.sender.Sender; @@ -41,6 +44,7 @@ import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; import static org.elasticsearch.xpack.inference.results.ChatCompletionResultsTests.buildExpectationCompletion; @@ -339,9 +343,10 @@ public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOExc } } - private CohereCompletionAction createAction(String url, String apiKey, @Nullable String modelName, Sender sender) { + private ExecutableAction createAction(String url, String apiKey, @Nullable String modelName, Sender sender) { var model = CohereCompletionModelTests.createModel(url, apiKey, modelName); - - return new CohereCompletionAction(sender, model, threadPool); + var requestManager = CohereCompletionRequestManager.of(model, threadPool); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.getServiceSettings().uri(), "Cohere completion"); + return new SingleInputSenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage, "Cohere completion"); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsActionTests.java index 9cf6de27b93bc..fe0eb782eddfc 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsActionTests.java @@ -22,8 +22,11 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.sender.CohereEmbeddingsRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; import org.elasticsearch.xpack.inference.external.http.sender.Sender; @@ -45,6 +48,7 @@ import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationByte; @@ -340,7 +344,7 @@ public void testExecute_ThrowsExceptionWithNullUrl() { MatcherAssert.assertThat(thrownException.getMessage(), is("Failed to send Cohere embeddings request")); } - private CohereEmbeddingsAction createAction( + private ExecutableAction createAction( String url, String apiKey, CohereEmbeddingsTaskSettings taskSettings, @@ -349,8 +353,12 @@ private CohereEmbeddingsAction createAction( Sender sender ) { var model = CohereEmbeddingsModelTests.createModel(url, apiKey, taskSettings, 1024, 1024, modelName, embeddingType); - - return new CohereEmbeddingsAction(sender, model, threadPool); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( + model.getServiceSettings().getCommonSettings().uri(), + "Cohere embeddings" + ); + var requestCreator = CohereEmbeddingsRequestManager.of(model, threadPool); + return new SenderExecutableAction(sender, requestCreator, failedToSendRequestErrorMessage); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java index 9dd465e0276f4..6fcd386702497 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java @@ -22,8 +22,11 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SingleInputSenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.GoogleAiStudioCompletionRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; @@ -39,6 +42,7 @@ import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; import static org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioServiceTests.buildExpectationCompletions; @@ -265,10 +269,16 @@ public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOExc } } - private GoogleAiStudioCompletionAction createAction(String url, String apiKey, String modelName, Sender sender) { + private ExecutableAction createAction(String url, String apiKey, String modelName, Sender sender) { var model = GoogleAiStudioCompletionModelTests.createModel(modelName, url, apiKey); - - return new GoogleAiStudioCompletionAction(sender, model, threadPool); + var requestManager = new GoogleAiStudioCompletionRequestManager(model, threadPool); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google AI Studio completion"); + return new SingleInputSenderExecutableAction( + sender, + requestManager, + failedToSendRequestErrorMessage, + "Google AI Studio completion" + ); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsActionTests.java index 7e98b9b31f6ed..27862f7309877 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsActionTests.java @@ -21,8 +21,12 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.inference.common.TruncatorTests; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.GoogleAiStudioEmbeddingsRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; @@ -37,6 +41,7 @@ import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationFloat; @@ -182,10 +187,11 @@ public void testExecute_ThrowsException() { ); } - private GoogleAiStudioEmbeddingsAction createAction(String url, String apiKey, String modelName, Sender sender) { + private ExecutableAction createAction(String url, String apiKey, String modelName, Sender sender) { var model = createModel(modelName, apiKey, url); - - return new GoogleAiStudioEmbeddingsAction(sender, model, createWithEmptySettings(threadPool)); + var requestManager = new GoogleAiStudioEmbeddingsRequestManager(model, TruncatorTests.createTruncator(), threadPool); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google AI Studio embeddings"); + return new SenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiEmbeddingsActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiEmbeddingsActionTests.java index 17a2c29e195f1..edfc447d5337e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiEmbeddingsActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiEmbeddingsActionTests.java @@ -17,8 +17,12 @@ import org.elasticsearch.test.http.MockWebServer; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.inference.common.TruncatorTests; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.GoogleVertexAiEmbeddingsRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; import org.junit.After; @@ -31,8 +35,8 @@ import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; -import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings; import static org.elasticsearch.xpack.inference.services.googlevertexai.embeddings.GoogleVertexAiEmbeddingsModelTests.createModel; import static org.hamcrest.Matchers.is; import static org.mockito.ArgumentMatchers.any; @@ -118,10 +122,11 @@ public void testExecute_ThrowsException() { ); } - private GoogleVertexAiEmbeddingsAction createAction(String url, String location, String projectId, String modelName, Sender sender) { + private ExecutableAction createAction(String url, String location, String projectId, String modelName, Sender sender) { var model = createModel(location, projectId, modelName, url, "{}"); - - return new GoogleVertexAiEmbeddingsAction(sender, model, createWithEmptySettings(threadPool)); + var requestManager = new GoogleVertexAiEmbeddingsRequestManager(model, TruncatorTests.createTruncator(), threadPool); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google Vertex AI embeddings"); + return new SenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiRerankActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiRerankActionTests.java index b84a6328e9882..491e17fc8c0a3 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiRerankActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googlevertexai/GoogleVertexAiRerankActionTests.java @@ -17,8 +17,11 @@ import org.elasticsearch.test.http.MockWebServer; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.GoogleVertexAiRerankRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; import org.elasticsearch.xpack.inference.services.googlevertexai.rerank.GoogleVertexAiRerankModelTests; @@ -32,6 +35,7 @@ import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; import static org.hamcrest.Matchers.is; import static org.mockito.ArgumentMatchers.any; @@ -110,9 +114,10 @@ public void testExecute_ThrowsException() { assertThat(thrownException.getMessage(), is(format("Failed to send Google Vertex AI rerank request to [%s]", getUrl(webServer)))); } - private GoogleVertexAiRerankAction createAction(String url, String projectId, Sender sender) { + private ExecutableAction createAction(String url, String projectId, Sender sender) { var model = GoogleVertexAiRerankModelTests.createModel(url, projectId, null); - - return new GoogleVertexAiRerankAction(sender, model, threadPool); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google Vertex AI rerank"); + var requestManager = GoogleVertexAiRerankRequestManager.of(model, threadPool); + return new SenderExecutableAction(sender, requestManager, failedToSendRequestErrorMessage); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceActionTests.java index 8ca0ce3daf0ba..848ca790d677d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceActionTests.java @@ -10,18 +10,19 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.inference.common.TruncatorTests; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.retry.AlwaysRetryingResponseHandler; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.HuggingFaceRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; -import org.elasticsearch.xpack.inference.logging.ThrottlerManager; -import org.elasticsearch.xpack.inference.services.ServiceComponents; +import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserModel; import org.junit.After; import org.junit.Before; @@ -108,27 +109,29 @@ public void testExecute_ThrowsException() { ); } - private HuggingFaceAction createAction(String url, Sender sender) { + private ExecutableAction createAction(String url, Sender sender) { var model = createModel(url, "secret"); + return createAction(model, sender); + } - return new HuggingFaceAction( - sender, + private ExecutableAction createAction(HuggingFaceElserModel model, Sender sender) { + var requestCreator = HuggingFaceRequestManager.of( model, - new ServiceComponents(threadPool, mock(ThrottlerManager.class), Settings.EMPTY, TruncatorTests.createTruncator()), new AlwaysRetryingResponseHandler("test", (result) -> null), - "test action" + TruncatorTests.createTruncator(), + threadPool + ); + var errorMessage = format( + "Failed to send Hugging Face %s request from inference entity id [%s]", + "test action", + model.getInferenceEntityId() ); + + return new SenderExecutableAction(sender, requestCreator, errorMessage); } - private HuggingFaceAction createAction(String url, Sender sender, String modelId) { + private ExecutableAction createAction(String url, Sender sender, String modelId) { var model = createModel(url, "secret", modelId); - - return new HuggingFaceAction( - sender, - model, - new ServiceComponents(threadPool, mock(ThrottlerManager.class), Settings.EMPTY, TruncatorTests.createTruncator()), - new AlwaysRetryingResponseHandler("test", (result) -> null), - "test action" - ); + return createAction(model, sender); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionActionTests.java index 42b062667f770..d84b2b5bb324a 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionActionTests.java @@ -24,10 +24,13 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SingleInputSenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; +import org.elasticsearch.xpack.inference.external.http.sender.OpenAiCompletionRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; import org.junit.After; @@ -41,6 +44,7 @@ import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; import static org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests.createSender; @@ -273,21 +277,15 @@ public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOExc var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); - assertThat(thrownException.getMessage(), is("OpenAI completions only accepts 1 input")); + assertThat(thrownException.getMessage(), is("OpenAI chat completions only accepts 1 input")); assertThat(thrownException.status(), is(RestStatus.BAD_REQUEST)); } } - private OpenAiChatCompletionAction createAction( - String url, - String org, - String apiKey, - String modelName, - @Nullable String user, - Sender sender - ) { + private ExecutableAction createAction(String url, String org, String apiKey, String modelName, @Nullable String user, Sender sender) { var model = createChatCompletionModel(url, org, apiKey, modelName, user); - - return new OpenAiChatCompletionAction(sender, model, createWithEmptySettings(threadPool)); + var requestCreator = OpenAiCompletionRequestManager.of(model, threadPool); + var errorMessage = constructFailedToSendRequestMessage(model.getServiceSettings().uri(), "OpenAI chat completions"); + return new SingleInputSenderExecutableAction(sender, requestCreator, errorMessage, "OpenAI chat completions"); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsActionTests.java index 03c0b4d146b2e..509dd144a1d1f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsActionTests.java @@ -21,9 +21,13 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.inference.common.TruncatorTests; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; +import org.elasticsearch.xpack.inference.external.http.sender.OpenAiEmbeddingsRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; import org.elasticsearch.xpack.inference.services.ServiceComponentsTests; @@ -37,11 +41,11 @@ import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; import static org.elasticsearch.xpack.inference.external.request.openai.OpenAiUtils.ORGANIZATION_HEADER; import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationFloat; -import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings; import static org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsModelTests.createModel; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -220,17 +224,11 @@ public void testExecute_ThrowsExceptionWithNullUrl() { assertThat(thrownException.getMessage(), is("Failed to send OpenAI embeddings request")); } - private OpenAiEmbeddingsAction createAction( - String url, - String org, - String apiKey, - String modelName, - @Nullable String user, - Sender sender - ) { + private ExecutableAction createAction(String url, String org, String apiKey, String modelName, @Nullable String user, Sender sender) { var model = createModel(url, org, apiKey, modelName, user); - - return new OpenAiEmbeddingsAction(sender, model, createWithEmptySettings(threadPool)); + var requestCreator = OpenAiEmbeddingsRequestManager.of(model, TruncatorTests.createTruncator(), threadPool); + var errorMessage = constructFailedToSendRequestMessage(model.getServiceSettings().uri(), "OpenAI embeddings"); + return new SenderExecutableAction(sender, requestCreator, errorMessage); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/model/TestModel.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/model/TestModel.java index 094952b8716b7..49f3f9d48f187 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/model/TestModel.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/model/TestModel.java @@ -168,6 +168,11 @@ public Integer dimensions() { public DenseVectorFieldMapper.ElementType elementType() { return elementType; } + + @Override + public String modelId() { + return model; + } } public record TestTaskSettings(Integer temperature) implements TaskSettings { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ServiceUtilsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ServiceUtilsTests.java index a14b42d51c6f8..76f095236af8a 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ServiceUtilsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ServiceUtilsTests.java @@ -21,6 +21,8 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingByteResults; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsFeatureFlag; +import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; import org.elasticsearch.xpack.inference.results.InferenceTextEmbeddingByteResultsTests; import org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests; @@ -42,8 +44,10 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.getEmbeddingSize; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -285,6 +289,50 @@ public void testRemoveAsOneOfTypesMissingReturnsNull() { assertThat(map.entrySet(), hasSize(3)); } + public void testRemoveAsAdaptiveAllocationsSettings() { + assumeTrue("Should only run if adaptive allocations feature flag is enabled", AdaptiveAllocationsFeatureFlag.isEnabled()); + + Map map = new HashMap<>( + Map.of("settings", new HashMap<>(Map.of("enabled", true, "min_number_of_allocations", 7, "max_number_of_allocations", 42))) + ); + ValidationException validationException = new ValidationException(); + assertThat( + ServiceUtils.removeAsAdaptiveAllocationsSettings(map, "settings", validationException), + equalTo(new AdaptiveAllocationsSettings(true, 7, 42)) + ); + assertThat(validationException.validationErrors(), empty()); + + assertThat(ServiceUtils.removeAsAdaptiveAllocationsSettings(map, "non-existent-key", validationException), nullValue()); + assertThat(validationException.validationErrors(), empty()); + + map = new HashMap<>(Map.of("settings", new HashMap<>(Map.of("enabled", false)))); + assertThat( + ServiceUtils.removeAsAdaptiveAllocationsSettings(map, "settings", validationException), + equalTo(new AdaptiveAllocationsSettings(false, null, null)) + ); + assertThat(validationException.validationErrors(), empty()); + } + + public void testRemoveAsAdaptiveAllocationsSettings_exceptions() { + assumeTrue("Should only run if adaptive allocations feature flag is enabled", AdaptiveAllocationsFeatureFlag.isEnabled()); + + Map map = new HashMap<>( + Map.of("settings", new HashMap<>(Map.of("enabled", "YES!", "blah", 42, "max_number_of_allocations", -7))) + ); + ValidationException validationException = new ValidationException(); + ServiceUtils.removeAsAdaptiveAllocationsSettings(map, "settings", validationException); + assertThat(validationException.validationErrors(), hasSize(3)); + assertThat( + validationException.validationErrors().get(0), + containsString("field [enabled] is not of the expected type. The value [YES!] cannot be converted to a [Boolean]") + ); + assertThat(validationException.validationErrors().get(1), containsString("[settings] does not allow the setting [blah]")); + assertThat( + validationException.validationErrors().get(2), + containsString("[max_number_of_allocations] must be a positive integer or null") + ); + } + public void testConvertToUri_CreatesUri() { var validation = new ValidationException(); var uri = convertToUri("www.elastic.co", "name", "scope", validation); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java index ae413fc17425c..d219e9d55312a 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java @@ -96,7 +96,7 @@ public void testParseRequestConfig_CreatesAnAmazonBedrockModel() throws IOExcept var settings = (AmazonBedrockEmbeddingsServiceSettings) model.getServiceSettings(); assertThat(settings.region(), is("region")); - assertThat(settings.model(), is("model")); + assertThat(settings.modelId(), is("model")); assertThat(settings.provider(), is(AmazonBedrockProvider.AMAZONTITAN)); var secretSettings = (AmazonBedrockSecretSettings) model.getSecretSettings(); assertThat(secretSettings.accessKey.toString(), is("access")); @@ -290,7 +290,7 @@ public void testParseRequestConfig_MovesModel() throws IOException { var settings = (AmazonBedrockEmbeddingsServiceSettings) model.getServiceSettings(); assertThat(settings.region(), is("region")); - assertThat(settings.model(), is("model")); + assertThat(settings.modelId(), is("model")); assertThat(settings.provider(), is(AmazonBedrockProvider.AMAZONTITAN)); var secretSettings = (AmazonBedrockSecretSettings) model.getSecretSettings(); assertThat(secretSettings.accessKey.toString(), is("access")); @@ -353,7 +353,7 @@ public void testParsePersistedConfigWithSecrets_CreatesAnAmazonBedrockEmbeddings var settings = (AmazonBedrockEmbeddingsServiceSettings) model.getServiceSettings(); assertThat(settings.region(), is("region")); - assertThat(settings.model(), is("model")); + assertThat(settings.modelId(), is("model")); assertThat(settings.provider(), is(AmazonBedrockProvider.AMAZONTITAN)); var secretSettings = (AmazonBedrockSecretSettings) model.getSecretSettings(); assertThat(secretSettings.accessKey.toString(), is("access")); @@ -404,7 +404,7 @@ public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExists var settings = (AmazonBedrockEmbeddingsServiceSettings) model.getServiceSettings(); assertThat(settings.region(), is("region")); - assertThat(settings.model(), is("model")); + assertThat(settings.modelId(), is("model")); assertThat(settings.provider(), is(AmazonBedrockProvider.AMAZONTITAN)); var secretSettings = (AmazonBedrockSecretSettings) model.getSecretSettings(); assertThat(secretSettings.accessKey.toString(), is("access")); @@ -431,7 +431,7 @@ public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExists var settings = (AmazonBedrockEmbeddingsServiceSettings) model.getServiceSettings(); assertThat(settings.region(), is("region")); - assertThat(settings.model(), is("model")); + assertThat(settings.modelId(), is("model")); assertThat(settings.provider(), is(AmazonBedrockProvider.AMAZONTITAN)); var secretSettings = (AmazonBedrockSecretSettings) model.getSecretSettings(); assertThat(secretSettings.accessKey.toString(), is("access")); @@ -458,7 +458,7 @@ public void testParsePersistedConfigWithSecrets_NotThrowWhenAnExtraKeyExistsInSe var settings = (AmazonBedrockEmbeddingsServiceSettings) model.getServiceSettings(); assertThat(settings.region(), is("region")); - assertThat(settings.model(), is("model")); + assertThat(settings.modelId(), is("model")); assertThat(settings.provider(), is(AmazonBedrockProvider.AMAZONTITAN)); var secretSettings = (AmazonBedrockSecretSettings) model.getSecretSettings(); assertThat(secretSettings.accessKey.toString(), is("access")); @@ -485,7 +485,7 @@ public void testParsePersistedConfigWithSecrets_NotThrowWhenAnExtraKeyExistsInSe var settings = (AmazonBedrockEmbeddingsServiceSettings) model.getServiceSettings(); assertThat(settings.region(), is("region")); - assertThat(settings.model(), is("model")); + assertThat(settings.modelId(), is("model")); assertThat(settings.provider(), is(AmazonBedrockProvider.AMAZONTITAN)); var secretSettings = (AmazonBedrockSecretSettings) model.getSecretSettings(); assertThat(secretSettings.accessKey.toString(), is("access")); @@ -513,7 +513,7 @@ public void testParsePersistedConfigWithSecrets_NotThrowWhenAnExtraKeyExistsInTa var settings = (AmazonBedrockChatCompletionServiceSettings) model.getServiceSettings(); assertThat(settings.region(), is("region")); - assertThat(settings.model(), is("model")); + assertThat(settings.modelId(), is("model")); assertThat(settings.provider(), is(AmazonBedrockProvider.ANTHROPIC)); var taskSettings = (AmazonBedrockChatCompletionTaskSettings) model.getTaskSettings(); assertThat(taskSettings.temperature(), is(1.0)); @@ -539,7 +539,7 @@ public void testParsePersistedConfig_CreatesAnAmazonBedrockEmbeddingsModel() thr var settings = (AmazonBedrockEmbeddingsServiceSettings) model.getServiceSettings(); assertThat(settings.region(), is("region")); - assertThat(settings.model(), is("model")); + assertThat(settings.modelId(), is("model")); assertThat(settings.provider(), is(AmazonBedrockProvider.AMAZONTITAN)); assertNull(model.getSecretSettings()); } @@ -558,7 +558,7 @@ public void testParsePersistedConfig_CreatesAnAmazonBedrockChatCompletionModel() var settings = (AmazonBedrockChatCompletionServiceSettings) model.getServiceSettings(); assertThat(settings.region(), is("region")); - assertThat(settings.model(), is("model")); + assertThat(settings.modelId(), is("model")); assertThat(settings.provider(), is(AmazonBedrockProvider.ANTHROPIC)); var taskSettings = (AmazonBedrockChatCompletionTaskSettings) model.getTaskSettings(); assertThat(taskSettings.temperature(), is(1.0)); @@ -602,7 +602,7 @@ public void testParsePersistedConfig_DoesNotThrowWhenAnExtraKeyExistsInConfig() var settings = (AmazonBedrockEmbeddingsServiceSettings) model.getServiceSettings(); assertThat(settings.region(), is("region")); - assertThat(settings.model(), is("model")); + assertThat(settings.modelId(), is("model")); assertThat(settings.provider(), is(AmazonBedrockProvider.AMAZONTITAN)); assertNull(model.getSecretSettings()); } @@ -623,7 +623,7 @@ public void testParsePersistedConfig_NotThrowWhenAnExtraKeyExistsInServiceSettin var settings = (AmazonBedrockEmbeddingsServiceSettings) model.getServiceSettings(); assertThat(settings.region(), is("region")); - assertThat(settings.model(), is("model")); + assertThat(settings.modelId(), is("model")); assertThat(settings.provider(), is(AmazonBedrockProvider.AMAZONTITAN)); assertNull(model.getSecretSettings()); } @@ -643,7 +643,7 @@ public void testParsePersistedConfig_NotThrowWhenAnExtraKeyExistsInTaskSettings( var settings = (AmazonBedrockChatCompletionServiceSettings) model.getServiceSettings(); assertThat(settings.region(), is("region")); - assertThat(settings.model(), is("model")); + assertThat(settings.modelId(), is("model")); assertThat(settings.provider(), is(AmazonBedrockProvider.ANTHROPIC)); var taskSettings = (AmazonBedrockChatCompletionTaskSettings) model.getTaskSettings(); assertThat(taskSettings.temperature(), is(1.0)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingTypeTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingTypeTests.java index ed13e5a87e71b..3aa423d5bbafd 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingTypeTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingTypeTests.java @@ -38,14 +38,14 @@ public void testTranslateToVersion_ReturnsFloat_WhenVersionIsBeforeByteEnumAddit public void testTranslateToVersion_ReturnsByte_WhenVersionOnByteEnumAddition_WhenSpecifyingByte() { assertThat( - CohereEmbeddingType.translateToVersion(CohereEmbeddingType.BYTE, TransportVersions.ML_INFERENCE_EMBEDDING_BYTE_ADDED), + CohereEmbeddingType.translateToVersion(CohereEmbeddingType.BYTE, TransportVersions.V_8_14_0), is(CohereEmbeddingType.BYTE) ); } public void testTranslateToVersion_ReturnsFloat_WhenVersionOnByteEnumAddition_WhenSpecifyingFloat() { assertThat( - CohereEmbeddingType.translateToVersion(CohereEmbeddingType.FLOAT, TransportVersions.ML_INFERENCE_EMBEDDING_BYTE_ADDED), + CohereEmbeddingType.translateToVersion(CohereEmbeddingType.FLOAT, TransportVersions.V_8_14_0), is(CohereEmbeddingType.FLOAT) ); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettingsTests.java index 1ce5a9fb12833..a729ac8e225b5 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettingsTests.java @@ -7,48 +7,58 @@ package org.elasticsearch.xpack.inference.services.cohere.rerank; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Nullable; -import org.elasticsearch.inference.SimilarityMeasure; -import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider; -import org.elasticsearch.xpack.inference.InferenceNamedWriteablesProvider; +import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; import org.elasticsearch.xpack.inference.services.cohere.CohereServiceSettings; import org.elasticsearch.xpack.inference.services.cohere.CohereServiceSettingsTests; import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettingsTests; import java.io.IOException; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; -import static org.hamcrest.Matchers.is; +import static org.elasticsearch.xpack.inference.MatchersUtils.equalToIgnoringWhitespaceInJsonString; -public class CohereRerankServiceSettingsTests extends AbstractWireSerializingTestCase { +public class CohereRerankServiceSettingsTests extends AbstractBWCWireSerializationTestCase { public static CohereRerankServiceSettings createRandom() { - var commonSettings = CohereServiceSettingsTests.createRandom(); + return createRandom(randomFrom(new RateLimitSettings[] { null, RateLimitSettingsTests.createRandom() })); + } - return new CohereRerankServiceSettings(commonSettings); + public static CohereRerankServiceSettings createRandom(@Nullable RateLimitSettings rateLimitSettings) { + return new CohereRerankServiceSettings( + randomFrom(new String[] { null, Strings.format("http://%s.com", randomAlphaOfLength(8)) }), + randomFrom(new String[] { null, randomAlphaOfLength(10) }), + rateLimitSettings + ); } public void testToXContent_WritesAllValues() throws IOException { - var serviceSettings = new CohereRerankServiceSettings( - new CohereServiceSettings("url", SimilarityMeasure.COSINE, 5, 10, "model_id", new RateLimitSettings(3)) - ); + var url = "http://www.abc.com"; + var model = "model"; + + var serviceSettings = new CohereRerankServiceSettings(url, model, null); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); serviceSettings.toXContent(builder, null); String xContentResult = Strings.toString(builder); - // TODO we probably shouldn't allow configuring these fields for reranking - assertThat(xContentResult, is(""" - {"url":"url","similarity":"cosine","dimensions":5,"max_input_tokens":10,"model_id":"model_id",""" + """ - "rate_limit":{"requests_per_minute":3}}""")); + + assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString(""" + { + "url":"http://www.abc.com", + "model_id":"model", + "rate_limit": { + "requests_per_minute": 10000 + } + } + """)); } @Override @@ -67,11 +77,12 @@ protected CohereRerankServiceSettings mutateInstance(CohereRerankServiceSettings } @Override - protected NamedWriteableRegistry getNamedWriteableRegistry() { - List entries = new ArrayList<>(); - entries.addAll(new MlInferenceNamedXContentProvider().getNamedWriteables()); - entries.addAll(InferenceNamedWriteablesProvider.getNamedWriteables()); - return new NamedWriteableRegistry(entries); + protected CohereRerankServiceSettings mutateInstanceForVersion(CohereRerankServiceSettings instance, TransportVersion version) { + if (version.before(TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED)) { + // We always default to the same rate limit settings, if a node is on a version before rate limits were introduced + return new CohereRerankServiceSettings(instance.uri(), instance.modelId(), CohereServiceSettings.DEFAULT_RATE_LIMIT_SETTINGS); + } + return instance; } public static Map getServiceSettingsMap(@Nullable String url, @Nullable String model) { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettingsTests.java index 8e8a1db76da14..ebb9c964e4c9a 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettingsTests.java @@ -23,8 +23,8 @@ import java.util.Map; import static org.elasticsearch.xpack.inference.services.ServiceFields.ELEMENT_TYPE; -import static org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings.NUM_ALLOCATIONS; -import static org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings.NUM_THREADS; +import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS; +import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings.NUM_THREADS; import static org.hamcrest.Matchers.is; public class CustomElandInternalTextEmbeddingServiceSettingsTests extends AbstractWireSerializingTestCase< @@ -216,8 +216,7 @@ public void testToXContent_WritesAllValues() throws IOException { String xContentResult = Strings.toString(builder); assertThat(xContentResult, is(""" - {"num_allocations":1,"num_threads":1,"model_id":"model_id","adaptive_allocations":null,"dimensions":100,""" + """ - "similarity":"cosine","element_type":"byte"}""")); + {"num_allocations":1,"num_threads":1,"model_id":"model_id","dimensions":100,"similarity":"cosine","element_type":"byte"}""")); } @Override diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettingsTests.java new file mode 100644 index 0000000000000..41afef88d22c6 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettingsTests.java @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elasticsearch; + +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; +import org.elasticsearch.xpack.inference.services.elser.ElserInternalServiceSettings; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.containsString; + +public class ElasticsearchInternalServiceSettingsTests extends AbstractWireSerializingTestCase { + + public static ElasticsearchInternalServiceSettings validInstance(String modelId) { + boolean useAdaptive = randomBoolean(); + if (useAdaptive) { + var adaptive = new AdaptiveAllocationsSettings(true, 1, randomIntBetween(2, 8)); + return new ElasticsearchInternalServiceSettings(randomBoolean() ? 1 : null, randomIntBetween(1, 16), modelId, adaptive); + } else { + return new ElasticsearchInternalServiceSettings(randomIntBetween(1, 10), randomIntBetween(1, 16), modelId, null); + } + } + + @Override + protected Writeable.Reader instanceReader() { + return ElasticsearchInternalServiceSettings::new; + } + + @Override + protected ElasticsearchInternalServiceSettings createTestInstance() { + return validInstance("my-model"); + } + + @Override + protected ElasticsearchInternalServiceSettings mutateInstance(ElasticsearchInternalServiceSettings instance) throws IOException { + return switch (randomIntBetween(0, 2)) { + case 0 -> new ElserInternalServiceSettings( + new ElasticsearchInternalServiceSettings( + instance.getNumAllocations() == null ? 1 : instance.getNumAllocations() + 1, + instance.getNumThreads(), + instance.modelId(), + instance.getAdaptiveAllocationsSettings() + ) + ); + case 1 -> new ElserInternalServiceSettings( + new ElasticsearchInternalServiceSettings( + instance.getNumAllocations(), + instance.getNumThreads() + 1, + instance.modelId(), + instance.getAdaptiveAllocationsSettings() + ) + ); + case 2 -> new ElserInternalServiceSettings( + new ElasticsearchInternalServiceSettings( + instance.getNumAllocations(), + instance.getNumThreads(), + instance.modelId() + "-bar", + instance.getAdaptiveAllocationsSettings() + ) + ); + default -> throw new IllegalStateException(); + }; + } + + public void testFromRequestMap_NoDefaultModel() { + var serviceSettingsBuilder = ElasticsearchInternalServiceSettings.fromRequestMap( + new HashMap<>( + Map.of(ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS, 1, ElasticsearchInternalServiceSettings.NUM_THREADS, 4) + ) + ); + assertNull(serviceSettingsBuilder.getModelId()); + } + + public void testFromMap() { + var serviceSettings = ElasticsearchInternalServiceSettings.fromRequestMap( + new HashMap<>( + Map.of( + ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS, + 1, + ElasticsearchInternalServiceSettings.NUM_THREADS, + 4, + ElasticsearchInternalServiceSettings.MODEL_ID, + ".elser_model_1" + ) + ) + ).build(); + assertEquals(new ElasticsearchInternalServiceSettings(1, 4, ".elser_model_1", null), serviceSettings); + } + + public void testFromMapMissingOptions() { + var e = expectThrows( + ValidationException.class, + () -> ElasticsearchInternalServiceSettings.fromRequestMap( + new HashMap<>(Map.of(ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS, 1)) + ) + ); + + assertThat(e.getMessage(), containsString("[service_settings] does not contain the required setting [num_threads]")); + + e = expectThrows( + ValidationException.class, + () -> ElasticsearchInternalServiceSettings.fromRequestMap( + new HashMap<>(Map.of(ElasticsearchInternalServiceSettings.NUM_THREADS, 1)) + ) + ); + + assertThat( + e.getMessage(), + containsString("[service_settings] does not contain one of the required settings [num_allocations, adaptive_allocations]") + ); + } + + public void testFromMapInvalidSettings() { + var settingsMap = new HashMap( + Map.of(ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS, 0, ElasticsearchInternalServiceSettings.NUM_THREADS, -1) + ); + var e = expectThrows(ValidationException.class, () -> ElasticsearchInternalServiceSettings.fromRequestMap(settingsMap)); + + assertThat(e.getMessage(), containsString("Invalid value [0]. [num_allocations] must be a positive integer")); + assertThat(e.getMessage(), containsString("Invalid value [-1]. [num_threads] must be a positive integer")); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java index ad1910cb9fc0a..e6fd725a50198 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java @@ -46,7 +46,6 @@ import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate; import org.elasticsearch.xpack.core.utils.FloatConversionUtils; import org.elasticsearch.xpack.inference.services.ServiceFields; -import org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings; import org.junit.After; import org.junit.Before; import org.mockito.ArgumentCaptor; @@ -125,7 +124,7 @@ public void testParseRequestConfig() { 1, ElasticsearchInternalServiceSettings.NUM_THREADS, 4, - InternalServiceSettings.MODEL_ID, + ElasticsearchInternalServiceSettings.MODEL_ID, ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID ) ) @@ -161,7 +160,7 @@ public void testParseRequestConfig() { ActionListener modelListener = ActionListener.wrap( model -> fail("Model parsing should have failed"), - e -> assertThat(e, instanceOf(IllegalArgumentException.class)) + e -> assertThat(e, instanceOf(ElasticsearchStatusException.class)) ); service.parseRequestConfig(randomInferenceEntityId, taskType, settings, Set.of(), modelListener); @@ -179,7 +178,7 @@ public void testParseRequestConfig() { 1, ElasticsearchInternalServiceSettings.NUM_THREADS, 4, - InternalServiceSettings.MODEL_ID, + ElasticsearchInternalServiceSettings.MODEL_ID, ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID, // we can't directly test the eland case until we mock // the threadpool within the client "not_a_valid_service_setting", @@ -208,7 +207,7 @@ public void testParseRequestConfig() { 1, ElasticsearchInternalServiceSettings.NUM_THREADS, 4, - InternalServiceSettings.MODEL_ID, + ElasticsearchInternalServiceSettings.MODEL_ID, ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID, // we can't directly test the eland case until we mock // the threadpool within the client "extra_setting_that_should_not_be_here", @@ -237,7 +236,7 @@ public void testParseRequestConfig() { 1, ElasticsearchInternalServiceSettings.NUM_THREADS, 4, - InternalServiceSettings.MODEL_ID, + ElasticsearchInternalServiceSettings.MODEL_ID, ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID // we can't directly test the eland case until we mock // the threadpool within the client ) @@ -279,7 +278,7 @@ public void testParseRequestConfig_Rerank() { 1, ElasticsearchInternalServiceSettings.NUM_THREADS, 4, - InternalServiceSettings.MODEL_ID, + ElasticsearchInternalServiceSettings.MODEL_ID, "foo" ) ) @@ -326,7 +325,7 @@ public void testParseRequestConfig_Rerank_DefaultTaskSettings() { 1, ElasticsearchInternalServiceSettings.NUM_THREADS, 4, - InternalServiceSettings.MODEL_ID, + ElasticsearchInternalServiceSettings.MODEL_ID, "foo" ) ) @@ -390,7 +389,7 @@ public void testParsePersistedConfig() { 1, ElasticsearchInternalServiceSettings.NUM_THREADS, 4, - InternalServiceSettings.MODEL_ID, + ElasticsearchInternalServiceSettings.MODEL_ID, "invalid" ) ) @@ -420,7 +419,7 @@ public void testParsePersistedConfig() { 1, ElasticsearchInternalServiceSettings.NUM_THREADS, 4, - InternalServiceSettings.MODEL_ID, + ElasticsearchInternalServiceSettings.MODEL_ID, ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID, ServiceFields.DIMENSIONS, 1 @@ -641,12 +640,12 @@ public void testParsePersistedConfig_Rerank() { 1, ElasticsearchInternalServiceSettings.NUM_THREADS, 4, - InternalServiceSettings.MODEL_ID, + ElasticsearchInternalServiceSettings.MODEL_ID, "foo" ) ) ); - settings.put(InternalServiceSettings.MODEL_ID, "foo"); + settings.put(ElasticsearchInternalServiceSettings.MODEL_ID, "foo"); var returnDocs = randomBoolean(); settings.put( ModelConfigurations.TASK_SETTINGS, @@ -670,12 +669,12 @@ public void testParsePersistedConfig_Rerank() { 1, ElasticsearchInternalServiceSettings.NUM_THREADS, 4, - InternalServiceSettings.MODEL_ID, + ElasticsearchInternalServiceSettings.MODEL_ID, "foo" ) ) ); - settings.put(InternalServiceSettings.MODEL_ID, "foo"); + settings.put(ElasticsearchInternalServiceSettings.MODEL_ID, "foo"); var model = service.parsePersistedConfig(randomInferenceEntityId, TaskType.RERANK, settings); assertThat(model.getTaskSettings(), instanceOf(CustomElandRerankTaskSettings.class)); @@ -706,7 +705,7 @@ public void testParseRequestConfigEland_PreservesTaskType() { 1, ElasticsearchInternalServiceSettings.NUM_THREADS, 4, - InternalServiceSettings.MODEL_ID, + ElasticsearchInternalServiceSettings.MODEL_ID, "custom-model" ) ) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallInternalServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallInternalServiceSettingsTests.java index 927d53360a2c5..f685eb3732a89 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallInternalServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallInternalServiceSettingsTests.java @@ -22,49 +22,17 @@ public class MultilingualE5SmallInternalServiceSettingsTests extends AbstractWir public static MultilingualE5SmallInternalServiceSettings createRandom() { return new MultilingualE5SmallInternalServiceSettings( - randomIntBetween(1, 4), - randomIntBetween(1, 4), - randomFrom(ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_VALID_IDS), - null - ); - } - - public void testFromMap_DefaultModelVersion() { - var serviceSettingsBuilder = MultilingualE5SmallInternalServiceSettings.fromMap( - new HashMap<>( - Map.of( - MultilingualE5SmallInternalServiceSettings.NUM_ALLOCATIONS, - 1, - MultilingualE5SmallInternalServiceSettings.NUM_THREADS, - 4 - ) + ElasticsearchInternalServiceSettingsTests.validInstance( + randomFrom(ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_VALID_IDS) ) ); - assertNull(serviceSettingsBuilder.getModelId()); - } - - public void testFromMap() { - String randomModelVariant = randomFrom(ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_VALID_IDS); - var serviceSettings = MultilingualE5SmallInternalServiceSettings.fromMap( - new HashMap<>( - Map.of( - MultilingualE5SmallInternalServiceSettings.NUM_ALLOCATIONS, - 1, - MultilingualE5SmallInternalServiceSettings.NUM_THREADS, - 4, - MultilingualE5SmallInternalServiceSettings.MODEL_ID, - randomModelVariant - ) - ) - ).build(); - assertEquals(new MultilingualE5SmallInternalServiceSettings(1, 4, randomModelVariant, null), serviceSettings); } public void testFromMapInvalidVersion() { String randomModelVariant = randomAlphaOfLength(10); var e = expectThrows( ValidationException.class, - () -> MultilingualE5SmallInternalServiceSettings.fromMap( + () -> MultilingualE5SmallInternalServiceSettings.fromRequestMap( new HashMap<>( Map.of( MultilingualE5SmallInternalServiceSettings.NUM_ALLOCATIONS, @@ -83,7 +51,7 @@ public void testFromMapInvalidVersion() { public void testFromMapMissingOptions() { var e = expectThrows( ValidationException.class, - () -> MultilingualE5SmallInternalServiceSettings.fromMap( + () -> MultilingualE5SmallInternalServiceSettings.fromRequestMap( new HashMap<>(Map.of(MultilingualE5SmallInternalServiceSettings.NUM_ALLOCATIONS, 1)) ) ); @@ -92,12 +60,15 @@ public void testFromMapMissingOptions() { e = expectThrows( ValidationException.class, - () -> MultilingualE5SmallInternalServiceSettings.fromMap( + () -> MultilingualE5SmallInternalServiceSettings.fromRequestMap( new HashMap<>(Map.of(MultilingualE5SmallInternalServiceSettings.NUM_THREADS, 1)) ) ); - assertThat(e.getMessage(), containsString("[service_settings] does not contain the required setting [num_allocations]")); + assertThat( + e.getMessage(), + containsString("[service_settings] does not contain one of the required settings [num_allocations, adaptive_allocations]") + ); } public void testFromMapInvalidSettings() { @@ -109,7 +80,7 @@ public void testFromMapInvalidSettings() { -1 ) ); - var e = expectThrows(ValidationException.class, () -> MultilingualE5SmallInternalServiceSettings.fromMap(settingsMap)); + var e = expectThrows(ValidationException.class, () -> MultilingualE5SmallInternalServiceSettings.fromRequestMap(settingsMap)); assertThat(e.getMessage(), containsString("Invalid value [0]. [num_allocations] must be a positive integer")); assertThat(e.getMessage(), containsString("Invalid value [-1]. [num_threads] must be a positive integer")); @@ -129,20 +100,20 @@ protected MultilingualE5SmallInternalServiceSettings createTestInstance() { protected MultilingualE5SmallInternalServiceSettings mutateInstance(MultilingualE5SmallInternalServiceSettings instance) { return switch (randomIntBetween(0, 2)) { case 0 -> new MultilingualE5SmallInternalServiceSettings( - instance.getNumAllocations() + 1, + instance.getNumAllocations() == null ? 1 : instance.getNumAllocations() + 1, instance.getNumThreads(), - instance.getModelId(), + instance.modelId(), null ); case 1 -> new MultilingualE5SmallInternalServiceSettings( instance.getNumAllocations(), instance.getNumThreads() + 1, - instance.getModelId(), + instance.modelId(), null ); case 2 -> { var versions = new HashSet<>(ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_VALID_IDS); - versions.remove(instance.getModelId()); + versions.remove(instance.modelId()); yield new MultilingualE5SmallInternalServiceSettings( instance.getNumAllocations(), instance.getNumThreads(), diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettingsTests.java index e7fbbffa2d3fe..ec753b9bec887 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettingsTests.java @@ -8,109 +8,35 @@ package org.elasticsearch.xpack.inference.services.elser; import org.elasticsearch.TransportVersions; -import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettingsTests; import java.io.IOException; -import java.util.HashMap; import java.util.HashSet; -import java.util.Map; - -import static org.hamcrest.Matchers.containsString; public class ElserInternalServiceSettingsTests extends AbstractWireSerializingTestCase { public static ElserInternalServiceSettings createRandom() { return new ElserInternalServiceSettings( - randomIntBetween(1, 4), - randomIntBetween(1, 2), - randomFrom(ElserInternalService.VALID_ELSER_MODEL_IDS), - null + ElasticsearchInternalServiceSettingsTests.validInstance(randomFrom(ElserInternalService.VALID_ELSER_MODEL_IDS)) ); } - public void testFromMap_DefaultModelVersion() { - var serviceSettingsBuilder = ElserInternalServiceSettings.fromMap( - new HashMap<>(Map.of(ElserInternalServiceSettings.NUM_ALLOCATIONS, 1, ElserInternalServiceSettings.NUM_THREADS, 4)) - ); - assertNull(serviceSettingsBuilder.getModelId()); - } - - public void testFromMap() { - var serviceSettings = ElserInternalServiceSettings.fromMap( - new HashMap<>( - Map.of( - ElserInternalServiceSettings.NUM_ALLOCATIONS, - 1, - ElserInternalServiceSettings.NUM_THREADS, - 4, - ElserInternalServiceSettings.MODEL_ID, - ".elser_model_1" - ) - ) - ).build(); - assertEquals(new ElserInternalServiceSettings(1, 4, ".elser_model_1", null), serviceSettings); - } - - public void testFromMapInvalidVersion() { - var e = expectThrows( - ValidationException.class, - () -> ElserInternalServiceSettings.fromMap( - new HashMap<>( - Map.of( - ElserInternalServiceSettings.NUM_ALLOCATIONS, - 1, - ElserInternalServiceSettings.NUM_THREADS, - 4, - "model_id", - ".elser_model_27" - ) - ) - ) - ); - assertThat(e.getMessage(), containsString("unknown ELSER model id [.elser_model_27]")); - } - - public void testFromMapMissingOptions() { - var e = expectThrows( - ValidationException.class, - () -> ElserInternalServiceSettings.fromMap(new HashMap<>(Map.of(ElserInternalServiceSettings.NUM_ALLOCATIONS, 1))) - ); - - assertThat(e.getMessage(), containsString("[service_settings] does not contain the required setting [num_threads]")); - - e = expectThrows( - ValidationException.class, - () -> ElserInternalServiceSettings.fromMap(new HashMap<>(Map.of(ElserInternalServiceSettings.NUM_THREADS, 1))) - ); - - assertThat(e.getMessage(), containsString("[service_settings] does not contain the required setting [num_allocations]")); - } - public void testBwcWrite() throws IOException { { - var settings = new ElserInternalServiceSettings(1, 1, ".elser_model_1", null); + var settings = new ElserInternalServiceSettings(new ElasticsearchInternalServiceSettings(1, 1, ".elser_model_1", null)); var copy = copyInstance(settings, TransportVersions.V_8_12_0); assertEquals(settings, copy); } { - var settings = new ElserInternalServiceSettings(1, 1, ".elser_model_1", null); + var settings = new ElserInternalServiceSettings(new ElasticsearchInternalServiceSettings(1, 1, ".elser_model_1", null)); var copy = copyInstance(settings, TransportVersions.V_8_11_X); assertEquals(settings, copy); } } - public void testFromMapInvalidSettings() { - var settingsMap = new HashMap( - Map.of(ElserInternalServiceSettings.NUM_ALLOCATIONS, 0, ElserInternalServiceSettings.NUM_THREADS, -1) - ); - var e = expectThrows(ValidationException.class, () -> ElserInternalServiceSettings.fromMap(settingsMap)); - - assertThat(e.getMessage(), containsString("Invalid value [0]. [num_allocations] must be a positive integer")); - assertThat(e.getMessage(), containsString("Invalid value [-1]. [num_threads] must be a positive integer")); - } - @Override protected Writeable.Reader instanceReader() { return ElserInternalServiceSettings::new; @@ -125,25 +51,31 @@ protected ElserInternalServiceSettings createTestInstance() { protected ElserInternalServiceSettings mutateInstance(ElserInternalServiceSettings instance) { return switch (randomIntBetween(0, 2)) { case 0 -> new ElserInternalServiceSettings( - instance.getNumAllocations() + 1, - instance.getNumThreads(), - instance.getModelId(), - null + new ElasticsearchInternalServiceSettings( + instance.getNumAllocations() == null ? 1 : instance.getNumAllocations() + 1, + instance.getNumThreads(), + instance.modelId(), + null + ) ); case 1 -> new ElserInternalServiceSettings( - instance.getNumAllocations(), - instance.getNumThreads() + 1, - instance.getModelId(), - null + new ElasticsearchInternalServiceSettings( + instance.getNumAllocations(), + instance.getNumThreads() + 1, + instance.modelId(), + null + ) ); case 2 -> { var versions = new HashSet<>(ElserInternalService.VALID_ELSER_MODEL_IDS); - versions.remove(instance.getModelId()); + versions.remove(instance.modelId()); yield new ElserInternalServiceSettings( - instance.getNumAllocations(), - instance.getNumThreads(), - versions.iterator().next(), - null + new ElasticsearchInternalServiceSettings( + instance.getNumAllocations(), + instance.getNumThreads(), + versions.iterator().next(), + null + ) ); } default -> throw new IllegalStateException(); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java index 5ee55003e7fe1..f950e515a5336 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java @@ -332,7 +332,7 @@ public void testParseRequestConfig_DefaultModel() { ); ActionListener modelActionListener = ActionListener.wrap((model) -> { - assertEquals(".elser_model_2", ((ElserInternalModel) model).getServiceSettings().getModelId()); + assertEquals(".elser_model_2", ((ElserInternalModel) model).getServiceSettings().modelId()); }, (e) -> { fail("Model verification should not fail"); }); service.parseRequestConfig("foo", TaskType.SPARSE_EMBEDDING, settings, Set.of(), modelActionListener); @@ -345,7 +345,7 @@ public void testParseRequestConfig_DefaultModel() { ); ActionListener modelActionListener = ActionListener.wrap((model) -> { - assertEquals(".elser_model_2_linux-x86_64", ((ElserInternalModel) model).getServiceSettings().getModelId()); + assertEquals(".elser_model_2_linux-x86_64", ((ElserInternalModel) model).getServiceSettings().modelId()); }, (e) -> { fail("Model verification should not fail"); }); service.parseRequestConfig("foo", TaskType.SPARSE_EMBEDDING, settings, Set.of("linux-x86_64"), modelActionListener); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java index 1e3dd1e348f55..c1eb66ac848ab 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java @@ -100,7 +100,7 @@ public void testParseRequestConfig_CreatesAMistralEmbeddingsModel() throws IOExc var embeddingsModel = (MistralEmbeddingsModel) model; var serviceSettings = (MistralEmbeddingsServiceSettings) model.getServiceSettings(); - assertThat(serviceSettings.model(), is("mistral-embed")); + assertThat(serviceSettings.modelId(), is("mistral-embed")); assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); }, exception -> fail("Unexpected exception: " + exception)); @@ -231,7 +231,7 @@ public void testParsePersistedConfig_CreatesAMistralEmbeddingsModel() throws IOE assertThat(model, instanceOf(MistralEmbeddingsModel.class)); var embeddingsModel = (MistralEmbeddingsModel) model; - assertThat(embeddingsModel.getServiceSettings().model(), is("mistral-embed")); + assertThat(embeddingsModel.getServiceSettings().modelId(), is("mistral-embed")); assertThat(embeddingsModel.getServiceSettings().dimensions(), is(1024)); assertThat(embeddingsModel.getServiceSettings().maxInputTokens(), is(512)); assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); @@ -354,7 +354,7 @@ public void testParsePersistedConfig_WithoutSecretsCreatesEmbeddingsModel() thro assertThat(model, instanceOf(MistralEmbeddingsModel.class)); var embeddingsModel = (MistralEmbeddingsModel) model; - assertThat(embeddingsModel.getServiceSettings().model(), is("mistral-embed")); + assertThat(embeddingsModel.getServiceSettings().modelId(), is("mistral-embed")); assertThat(embeddingsModel.getServiceSettings().dimensions(), is(1024)); assertThat(embeddingsModel.getServiceSettings().maxInputTokens(), is(512)); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/telemetry/StatsMapTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/telemetry/StatsMapTests.java new file mode 100644 index 0000000000000..fcd8d3d7cefbc --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/telemetry/StatsMapTests.java @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.telemetry; + +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsServiceSettingsTests; +import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsTaskSettingsTests; +import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsServiceSettingsTests; +import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsTaskSettingsTests; + +import java.util.Map; + +import static org.hamcrest.Matchers.is; + +public class StatsMapTests extends ESTestCase { + public void testAddingEntry_InitializesTheCountToOne() { + var stats = new StatsMap<>(InferenceStats::key, InferenceStats::new); + + stats.increment( + new OpenAiEmbeddingsModel( + "inference_id", + TaskType.TEXT_EMBEDDING, + "openai", + OpenAiEmbeddingsServiceSettingsTests.getServiceSettingsMap("modelId", null, null), + OpenAiEmbeddingsTaskSettingsTests.getTaskSettingsMap(null), + null, + ConfigurationParseContext.REQUEST + ) + ); + + var converted = stats.toSerializableMap(); + + assertThat( + converted, + is( + Map.of( + "openai:text_embedding:modelId", + new org.elasticsearch.xpack.core.inference.InferenceRequestStats("openai", TaskType.TEXT_EMBEDDING, "modelId", 1) + ) + ) + ); + } + + public void testIncrementingWithSeparateModels_IncrementsTheCounterToTwo() { + var stats = new StatsMap<>(InferenceStats::key, InferenceStats::new); + + var model1 = new OpenAiEmbeddingsModel( + "inference_id", + TaskType.TEXT_EMBEDDING, + "openai", + OpenAiEmbeddingsServiceSettingsTests.getServiceSettingsMap("modelId", null, null), + OpenAiEmbeddingsTaskSettingsTests.getTaskSettingsMap(null), + null, + ConfigurationParseContext.REQUEST + ); + + var model2 = new OpenAiEmbeddingsModel( + "inference_id", + TaskType.TEXT_EMBEDDING, + "openai", + OpenAiEmbeddingsServiceSettingsTests.getServiceSettingsMap("modelId", null, null), + OpenAiEmbeddingsTaskSettingsTests.getTaskSettingsMap(null), + null, + ConfigurationParseContext.REQUEST + ); + + stats.increment(model1); + stats.increment(model2); + + var converted = stats.toSerializableMap(); + + assertThat( + converted, + is( + Map.of( + "openai:text_embedding:modelId", + new org.elasticsearch.xpack.core.inference.InferenceRequestStats("openai", TaskType.TEXT_EMBEDDING, "modelId", 2) + ) + ) + ); + } + + public void testNullModelId_ResultsInKeyWithout() { + var stats = new StatsMap<>(InferenceStats::key, InferenceStats::new); + + stats.increment( + new CohereEmbeddingsModel( + "inference_id", + TaskType.TEXT_EMBEDDING, + "cohere", + CohereEmbeddingsServiceSettingsTests.getServiceSettingsMap(null, null, null), + CohereEmbeddingsTaskSettingsTests.getTaskSettingsMap(null, null), + null, + ConfigurationParseContext.REQUEST + ) + ); + + var converted = stats.toSerializableMap(); + + assertThat( + converted, + is( + Map.of( + "cohere:text_embedding", + new org.elasticsearch.xpack.core.inference.InferenceRequestStats("cohere", TaskType.TEXT_EMBEDDING, null, 1) + ) + ) + ); + } +} diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java index c32e7e583c787..4b7b27bf2cec3 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java @@ -124,7 +124,7 @@ public void testUsedInScript() throws IOException { SearchLookup lookup = new SearchLookup( searchExecutionContext::getFieldType, (mft, lookupSupplier, fdo) -> mft.fielddataBuilder( - new FieldDataContext("test", lookupSupplier, searchExecutionContext::sourcePath, fdo) + new FieldDataContext("test", null, lookupSupplier, searchExecutionContext::sourcePath, fdo) ).build(null, null), (ctx, doc) -> null ); diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index 1152d93f66b38..92aac7897bcfd 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentParsingException; @@ -240,6 +241,11 @@ public String indexName() { throw new UnsupportedOperationException(); } + @Override + public IndexSettings indexSettings() { + throw new UnsupportedOperationException(); + } + @Override public MappedFieldType.FieldExtractPreference fieldExtractPreference() { return MappedFieldType.FieldExtractPreference.NONE; diff --git a/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackage.java b/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackage.java index b0544806d52bd..cdc3205f4197c 100644 --- a/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackage.java +++ b/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackage.java @@ -27,6 +27,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskAwareRequest; +import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.threadpool.ThreadPool; @@ -141,26 +142,29 @@ static void importModel( try { final long relativeStartNanos = System.nanoTime(); - logAndWriteNotificationAtInfo(auditClient, modelId, "starting model import"); + logAndWriteNotificationAtLevel(auditClient, modelId, "starting model import", Level.INFO); modelImporter.doImport(); final long totalRuntimeNanos = System.nanoTime() - relativeStartNanos; - logAndWriteNotificationAtInfo( + logAndWriteNotificationAtLevel( auditClient, modelId, - format("finished model import after [%d] seconds", TimeUnit.NANOSECONDS.toSeconds(totalRuntimeNanos)) + format("finished model import after [%d] seconds", TimeUnit.NANOSECONDS.toSeconds(totalRuntimeNanos)), + Level.INFO ); + } catch (TaskCancelledException e) { + recordError(auditClient, modelId, exceptionRef, e, Level.WARNING); } catch (ElasticsearchException e) { - recordError(auditClient, modelId, exceptionRef, e); + recordError(auditClient, modelId, exceptionRef, e, Level.ERROR); } catch (MalformedURLException e) { - recordError(auditClient, modelId, "an invalid URL", exceptionRef, e, RestStatus.INTERNAL_SERVER_ERROR); + recordError(auditClient, modelId, "an invalid URL", exceptionRef, e, Level.ERROR, RestStatus.INTERNAL_SERVER_ERROR); } catch (URISyntaxException e) { - recordError(auditClient, modelId, "an invalid URL syntax", exceptionRef, e, RestStatus.INTERNAL_SERVER_ERROR); + recordError(auditClient, modelId, "an invalid URL syntax", exceptionRef, e, Level.ERROR, RestStatus.INTERNAL_SERVER_ERROR); } catch (IOException e) { - recordError(auditClient, modelId, "an IOException", exceptionRef, e, RestStatus.SERVICE_UNAVAILABLE); + recordError(auditClient, modelId, "an IOException", exceptionRef, e, Level.ERROR, RestStatus.SERVICE_UNAVAILABLE); } catch (Exception e) { - recordError(auditClient, modelId, "an Exception", exceptionRef, e, RestStatus.INTERNAL_SERVER_ERROR); + recordError(auditClient, modelId, "an Exception", exceptionRef, e, Level.ERROR, RestStatus.INTERNAL_SERVER_ERROR); } finally { taskManager.unregister(task); @@ -199,8 +203,15 @@ public ModelDownloadTask createTask(long id, String type, String action, TaskId }, false); } - private static void recordError(Client client, String modelId, AtomicReference exceptionRef, ElasticsearchException e) { - logAndWriteNotificationAtError(client, modelId, e.getDetailedMessage()); + private static void recordError( + Client client, + String modelId, + AtomicReference exceptionRef, + ElasticsearchException e, + Level level + ) { + String message = format("Model importing failed due to [%s]", e.getDetailedMessage()); + logAndWriteNotificationAtLevel(client, modelId, message, level); exceptionRef.set(e); } @@ -210,21 +221,17 @@ private static void recordError( String failureType, AtomicReference exceptionRef, Exception e, + Level level, RestStatus status ) { String message = format("Model importing failed due to %s [%s]", failureType, e); - logAndWriteNotificationAtError(client, modelId, message); + logAndWriteNotificationAtLevel(client, modelId, message, level); exceptionRef.set(new ElasticsearchStatusException(message, status, e)); } - private static void logAndWriteNotificationAtError(Client client, String modelId, String message) { - writeNotification(client, modelId, message, Level.ERROR); - logger.error(format("[%s] %s", modelId, message)); - } - - private static void logAndWriteNotificationAtInfo(Client client, String modelId, String message) { - writeNotification(client, modelId, message, Level.INFO); - logger.info(format("[%s] %s", modelId, message)); + private static void logAndWriteNotificationAtLevel(Client client, String modelId, String message, Level level) { + writeNotification(client, modelId, message, level); + logger.log(level.log4jLevel(), format("[%s] %s", modelId, message)); } private static void writeNotification(Client client, String modelId, String message, Level level) { diff --git a/x-pack/plugin/ml-package-loader/src/test/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackageTests.java b/x-pack/plugin/ml-package-loader/src/test/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackageTests.java index 1e10ea48d03db..a3f59e13f2f5b 100644 --- a/x-pack/plugin/ml-package-loader/src/test/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackageTests.java +++ b/x-pack/plugin/ml-package-loader/src/test/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackageTests.java @@ -7,14 +7,17 @@ package org.elasticsearch.xpack.ml.packageloader.action; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.common.notifications.Level; import org.elasticsearch.xpack.core.ml.action.AuditMlNotificationAction; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ModelPackageConfig; import org.elasticsearch.xpack.core.ml.packageloader.action.LoadTrainedModelPackageAction; @@ -62,36 +65,44 @@ public void testSendsFinishedUploadNotification() { public void testSendsErrorNotificationForInternalError() throws URISyntaxException, IOException { ElasticsearchStatusException exception = new ElasticsearchStatusException("exception", RestStatus.INTERNAL_SERVER_ERROR); + String message = format("Model importing failed due to [%s]", exception.toString()); - assertUploadCallsOnFailure(exception, exception.toString()); + assertUploadCallsOnFailure(exception, message, Level.ERROR); } public void testSendsErrorNotificationForMalformedURL() throws URISyntaxException, IOException { MalformedURLException exception = new MalformedURLException("exception"); String message = format(MODEL_IMPORT_FAILURE_MSG_FORMAT, "an invalid URL", exception.toString()); - assertUploadCallsOnFailure(exception, message, RestStatus.INTERNAL_SERVER_ERROR); + assertUploadCallsOnFailure(exception, message, RestStatus.INTERNAL_SERVER_ERROR, Level.ERROR); } public void testSendsErrorNotificationForURISyntax() throws URISyntaxException, IOException { URISyntaxException exception = mock(URISyntaxException.class); String message = format(MODEL_IMPORT_FAILURE_MSG_FORMAT, "an invalid URL syntax", exception.toString()); - assertUploadCallsOnFailure(exception, message, RestStatus.INTERNAL_SERVER_ERROR); + assertUploadCallsOnFailure(exception, message, RestStatus.INTERNAL_SERVER_ERROR, Level.ERROR); } public void testSendsErrorNotificationForIOException() throws URISyntaxException, IOException { IOException exception = mock(IOException.class); String message = format(MODEL_IMPORT_FAILURE_MSG_FORMAT, "an IOException", exception.toString()); - assertUploadCallsOnFailure(exception, message, RestStatus.SERVICE_UNAVAILABLE); + assertUploadCallsOnFailure(exception, message, RestStatus.SERVICE_UNAVAILABLE, Level.ERROR); } public void testSendsErrorNotificationForException() throws URISyntaxException, IOException { RuntimeException exception = mock(RuntimeException.class); String message = format(MODEL_IMPORT_FAILURE_MSG_FORMAT, "an Exception", exception.toString()); - assertUploadCallsOnFailure(exception, message, RestStatus.INTERNAL_SERVER_ERROR); + assertUploadCallsOnFailure(exception, message, RestStatus.INTERNAL_SERVER_ERROR, Level.ERROR); + } + + public void testSendsWarningNotificationForTaskCancelledException() throws URISyntaxException, IOException { + TaskCancelledException exception = new TaskCancelledException("cancelled"); + String message = format("Model importing failed due to [%s]", exception.toString()); + + assertUploadCallsOnFailure(exception, message, Level.WARNING); } public void testCallsOnResponseWithAcknowledgedResponse() throws URISyntaxException, IOException { @@ -123,18 +134,24 @@ public void testDoesNotCallListenerWhenNotWaitingForCompletion() { ); } - private void assertUploadCallsOnFailure(Exception exception, String message, RestStatus status) throws URISyntaxException, IOException { + private void assertUploadCallsOnFailure(Exception exception, String message, RestStatus status, Level level) throws URISyntaxException, + IOException { var esStatusException = new ElasticsearchStatusException(message, status, exception); - assertNotificationAndOnFailure(exception, esStatusException, message); + assertNotificationAndOnFailure(exception, esStatusException, message, level); } - private void assertUploadCallsOnFailure(ElasticsearchStatusException exception, String message) throws URISyntaxException, IOException { - assertNotificationAndOnFailure(exception, exception, message); + private void assertUploadCallsOnFailure(ElasticsearchException exception, String message, Level level) throws URISyntaxException, + IOException { + assertNotificationAndOnFailure(exception, exception, message, level); } - private void assertNotificationAndOnFailure(Exception thrownException, ElasticsearchStatusException onFailureException, String message) - throws URISyntaxException, IOException { + private void assertNotificationAndOnFailure( + Exception thrownException, + ElasticsearchException onFailureException, + String message, + Level level + ) throws URISyntaxException, IOException { var client = mock(Client.class); var taskManager = mock(TaskManager.class); var task = mock(Task.class); @@ -150,9 +167,11 @@ private void assertNotificationAndOnFailure(Exception thrownException, Elasticse var notificationArg = ArgumentCaptor.forClass(AuditMlNotificationAction.Request.class); // 2 notifications- the starting message and the failure verify(client, times(2)).execute(eq(AuditMlNotificationAction.INSTANCE), notificationArg.capture(), any()); - assertThat(notificationArg.getValue().getMessage(), is(message)); // the last message is captured + var notification = notificationArg.getValue(); + assertThat(notification.getMessage(), is(message)); // the last message is captured + assertThat(notification.getLevel(), is(level)); // the last message is captured - var receivedException = (ElasticsearchStatusException) failureRef.get(); + var receivedException = (ElasticsearchException) failureRef.get(); assertThat(receivedException.toString(), is(onFailureException.toString())); assertThat(receivedException.status(), is(onFailureException.status())); assertThat(receivedException.getCause(), is(onFailureException.getCause())); diff --git a/x-pack/plugin/ml/qa/no-bootstrap-tests/build.gradle b/x-pack/plugin/ml/qa/no-bootstrap-tests/build.gradle index 4d2fdd603dbbd..6a74f40777aae 100644 --- a/x-pack/plugin/ml/qa/no-bootstrap-tests/build.gradle +++ b/x-pack/plugin/ml/qa/no-bootstrap-tests/build.gradle @@ -3,4 +3,5 @@ apply plugin: 'elasticsearch.standalone-test' dependencies { testImplementation project(":x-pack:plugin:core") testImplementation project(path: xpackModule('ml')) + testImplementation "net.java.dev.jna:jna:${versions.jna}" } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsAction.java index 12a8e7aadee46..7c282d88aebfd 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsAction.java @@ -9,7 +9,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters; @@ -96,11 +95,29 @@ public TransportGetTrainedModelsStatsAction( TrainedModelProvider trainedModelProvider, Client client ) { - super(GetTrainedModelsStatsAction.NAME, actionFilters, transportService.getTaskManager()); + this( + transportService, + actionFilters, + clusterService, + trainedModelProvider, + client, + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME) + ); + } + + private TransportGetTrainedModelsStatsAction( + TransportService transportService, + ActionFilters actionFilters, + ClusterService clusterService, + TrainedModelProvider trainedModelProvider, + Client client, + Executor executor + ) { + super(GetTrainedModelsStatsAction.NAME, actionFilters, transportService.getTaskManager(), executor); this.client = client; this.clusterService = clusterService; this.trainedModelProvider = trainedModelProvider; - this.executor = threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME); + this.executor = executor; } @Override @@ -108,15 +125,6 @@ protected void doExecute( Task task, GetTrainedModelsStatsAction.Request request, ActionListener listener - ) { - // workaround for https://github.com/elastic/elasticsearch/issues/97916 - TODO remove this when we can - executor.execute(ActionRunnable.wrap(listener, l -> doExecuteForked(task, request, l))); - } - - protected void doExecuteForked( - Task task, - GetTrainedModelsStatsAction.Request request, - ActionListener listener ) { final TaskId parentTaskId = new TaskId(clusterService.localNode().getId(), task.getId()); final ModelAliasMetadata modelAliasMetadata = ModelAliasMetadata.fromState(clusterService.state()); @@ -406,7 +414,7 @@ static Map inferenceIngestStatsByModelId( static NodesStatsRequest nodeStatsRequest(ClusterState state, TaskId parentTaskId) { String[] ingestNodes = state.nodes().getIngestNodes().keySet().toArray(String[]::new); NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(ingestNodes).clear() - .addMetric(NodesStatsRequestParameters.Metric.INGEST.metricName()); + .addMetric(NodesStatsRequestParameters.Metric.INGEST); nodesStatsRequest.setIncludeShardsStats(false); nodesStatsRequest.setParentTask(parentTaskId); return nodesStatsRequest; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java index b33e86d434f95..15f647bc76697 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java @@ -21,6 +21,13 @@ public class AdaptiveAllocationsScaler { static final double SCALE_UP_THRESHOLD = 0.9; private static final double SCALE_DOWN_THRESHOLD = 0.85; + /** + * If the max_number_of_allocations is not set, use this value for now to prevent scaling up + * to high numbers due to possible bugs or unexpected behaviour in the scaler. + * TODO(jan): remove this safeguard when the scaler behaves as expected in production. + */ + private static final int MAX_NUMBER_OF_ALLOCATIONS_SAFEGUARD = 32; + private static final Logger logger = LogManager.getLogger(AdaptiveAllocationsScaler.class); private final String deploymentId; @@ -114,6 +121,9 @@ Integer scale() { numberOfAllocations--; } + if (maxNumberOfAllocations == null) { + numberOfAllocations = Math.min(numberOfAllocations, MAX_NUMBER_OF_ALLOCATIONS_SAFEGUARD); + } if (minNumberOfAllocations != null) { numberOfAllocations = Math.max(numberOfAllocations, minNumberOfAllocations); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearning.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearning.java index 2d7832d747de4..08766f8a054df 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearning.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearning.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.breaker.NoopCircuitBreaker; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.analysis.CharFilterFactory; import org.elasticsearch.index.analysis.TokenizerFactory; import org.elasticsearch.indices.analysis.AnalysisModule; @@ -121,7 +122,12 @@ public static class MockedRollupIndexCapsTransport extends TransportAction< @Inject public MockedRollupIndexCapsTransport(TransportService transportService) { - super(GetRollupIndexCapsAction.NAME, new ActionFilters(new HashSet<>()), transportService.getTaskManager()); + super( + GetRollupIndexCapsAction.NAME, + new ActionFilters(new HashSet<>()), + transportService.getTaskManager(), + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java index 9758d00627efe..08097357725d0 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java @@ -138,4 +138,12 @@ public void testEstimation_highVariance() { assertThat(averageLoadMean + averageLoadError, greaterThan(expectedLoad)); assertThat(averageLoadError / averageLoadMean, lessThan(1 - AdaptiveAllocationsScaler.SCALE_UP_THRESHOLD)); } + + public void testAutoscaling_maxAllocationsSafeguard() { + AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1); + adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(1_000_000, 10_000_000, 1, 0.05), 10, 1); + assertThat(adaptiveAllocationsScaler.scale(), equalTo(32)); + adaptiveAllocationsScaler.setMinMaxNumberOfAllocations(2, 77); + assertThat(adaptiveAllocationsScaler.scale(), equalTo(77)); + } } diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsCollector.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsCollector.java index e62f7113acbf4..8c6ec0c5aa487 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsCollector.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsCollector.java @@ -71,11 +71,11 @@ protected Collection doCollect(final MonitoringDoc.Node node, fin request.setIncludeShardsStats(false); request.indices(FLAGS); request.addMetrics( - NodesStatsRequestParameters.Metric.OS.metricName(), - NodesStatsRequestParameters.Metric.JVM.metricName(), - NodesStatsRequestParameters.Metric.PROCESS.metricName(), - NodesStatsRequestParameters.Metric.THREAD_POOL.metricName(), - NodesStatsRequestParameters.Metric.FS.metricName() + NodesStatsRequestParameters.Metric.OS, + NodesStatsRequestParameters.Metric.JVM, + NodesStatsRequestParameters.Metric.PROCESS, + NodesStatsRequestParameters.Metric.THREAD_POOL, + NodesStatsRequestParameters.Metric.FS ); request.timeout(getCollectionTimeout()); diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java index 35a2b0c8cd58a..cd05c9bf0d754 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java @@ -124,9 +124,14 @@ public void testExecuteWithGlobalBlock() throws Exception { ); final MonitoringBulkRequest request = randomRequest(); - final ClusterBlockException e = expectThrows(ClusterBlockException.class, () -> ActionTestUtils.executeBlocking(action, request)); - assertThat(e, hasToString(containsString("ClusterBlockException: blocked by: [SERVICE_UNAVAILABLE/2/no master]"))); + assertThat( + asInstanceOf( + ClusterBlockException.class, + safeAwaitFailure(MonitoringBulkResponse.class, l -> action.execute(null, request, l)) + ), + hasToString(containsString("ClusterBlockException: blocked by: [SERVICE_UNAVAILABLE/2/no master]")) + ); } public void testExecuteIgnoresRequestWhenCollectionIsDisabled() throws Exception { @@ -169,13 +174,13 @@ public void testExecuteEmptyRequest() { monitoringService ); - final MonitoringBulkRequest request = new MonitoringBulkRequest(); - final ActionRequestValidationException e = expectThrows( - ActionRequestValidationException.class, - () -> ActionTestUtils.executeBlocking(action, request) + assertThat( + asInstanceOf( + ActionRequestValidationException.class, + safeAwaitFailure(MonitoringBulkResponse.class, l -> action.execute(null, new MonitoringBulkRequest(), l)) + ), + hasToString(containsString("no monitoring documents added")) ); - - assertThat(e, hasToString(containsString("no monitoring documents added"))); } @SuppressWarnings("unchecked") diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetFlamegraphAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetFlamegraphAction.java index 4f3778081563b..59f5ce1d7cbf5 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetFlamegraphAction.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetFlamegraphAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.client.internal.ParentTaskAssigningClient; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; @@ -33,7 +34,7 @@ public class TransportGetFlamegraphAction extends TransportAction AzureHttpFixture.class.getResourceAsStream("azure-http-fixture.pem") + ); private static ElasticsearchCluster cluster = ElasticsearchCluster.local() .module("repository-azure") @@ -44,13 +54,14 @@ public class AzureRepositoriesMeteringIT extends AbstractRepositoriesMeteringAPI ) .setting( "azure.client.repositories_metering.endpoint_suffix", - () -> "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=" + fixture.getAddress(), + () -> "ignored;DefaultEndpointsProtocol=https;BlobEndpoint=" + fixture.getAddress(), s -> USE_FIXTURE ) + .systemProperty("javax.net.ssl.trustStore", () -> trustStore.getTrustStorePath().toString(), s -> USE_FIXTURE) .build(); @ClassRule - public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster); + public static TestRule ruleChain = RuleChain.outerRule(fixture).around(trustStore).around(cluster); @Override protected String repositoryType() { diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java index 4108b0f6d3c83..0b17302ffd6a5 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java @@ -95,7 +95,7 @@ public TransportRollupSearchAction( ClusterService clusterService, IndexNameExpressionResolver resolver ) { - super(RollupSearchAction.NAME, actionFilters, transportService.getTaskManager()); + super(RollupSearchAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.client = client; this.registry = registry; this.bigArrays = bigArrays; diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java index 7a947fcb5ce02..ad5e6a0cf9b40 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java @@ -41,6 +41,7 @@ import java.util.function.Function; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.startsWith; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -562,7 +563,6 @@ public void testMultipleJobTriggering() throws Exception { assertThat(indexer.getState(), equalTo(IndexerState.INDEXING)); latch.countDown(); assertBusy(() -> assertThat(indexer.getState(), equalTo(IndexerState.STARTED))); - assertThat(indexer.getStats().getNumInvocations(), equalTo((long) i + 1)); assertThat(indexer.getStats().getNumPages(), equalTo((long) i + 1)); } final CountDownLatch latch = indexer.newLatch(); @@ -572,6 +572,7 @@ public void testMultipleJobTriggering() throws Exception { latch.countDown(); assertBusy(() -> assertThat(indexer.getState(), equalTo(IndexerState.STOPPED))); assertTrue(indexer.abort()); + assertThat(indexer.getStats().getNumInvocations(), greaterThanOrEqualTo(6L)); } finally { ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); } diff --git a/x-pack/plugin/searchable-snapshots/build.gradle b/x-pack/plugin/searchable-snapshots/build.gradle index c5cd000ef7774..4e309499445e6 100644 --- a/x-pack/plugin/searchable-snapshots/build.gradle +++ b/x-pack/plugin/searchable-snapshots/build.gradle @@ -15,6 +15,7 @@ base { dependencies { compileOnly project(path: xpackModule('core')) compileOnly project(path: xpackModule('blob-cache')) + compileOnly project(path: ':libs:elasticsearch-native') testImplementation(testArtifact(project(xpackModule('blob-cache')))) internalClusterTestImplementation(testArtifact(project(xpackModule('core')))) internalClusterTestImplementation(project(path: xpackModule('shutdown'))) diff --git a/x-pack/plugin/searchable-snapshots/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/searchablesnapshots/AzureSearchableSnapshotsIT.java b/x-pack/plugin/searchable-snapshots/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/searchablesnapshots/AzureSearchableSnapshotsIT.java index 35f00f8f0e022..4d7aabe489b9c 100644 --- a/x-pack/plugin/searchable-snapshots/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/searchablesnapshots/AzureSearchableSnapshotsIT.java +++ b/x-pack/plugin/searchable-snapshots/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/searchablesnapshots/AzureSearchableSnapshotsIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Booleans; +import org.elasticsearch.test.TestTrustStore; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.ESRestTestCase; import org.junit.ClassRule; @@ -27,7 +28,16 @@ public class AzureSearchableSnapshotsIT extends AbstractSearchableSnapshotsRestT private static final String AZURE_TEST_KEY = System.getProperty("test.azure.key"); private static final String AZURE_TEST_SASTOKEN = System.getProperty("test.azure.sas_token"); - private static AzureHttpFixture fixture = new AzureHttpFixture(USE_FIXTURE, AZURE_TEST_ACCOUNT, AZURE_TEST_CONTAINER); + private static AzureHttpFixture fixture = new AzureHttpFixture( + USE_FIXTURE ? AzureHttpFixture.Protocol.HTTPS : AzureHttpFixture.Protocol.NONE, + AZURE_TEST_ACCOUNT, + AZURE_TEST_CONTAINER, + AzureHttpFixture.sharedKeyForAccountPredicate(AZURE_TEST_ACCOUNT) + ); + + private static TestTrustStore trustStore = new TestTrustStore( + () -> AzureHttpFixture.class.getResourceAsStream("azure-http-fixture.pem") + ); private static ElasticsearchCluster cluster = ElasticsearchCluster.local() .module("repository-azure") @@ -53,10 +63,11 @@ public class AzureSearchableSnapshotsIT extends AbstractSearchableSnapshotsRestT .setting("xpack.searchable.snapshot.shared_cache.size", "16MB") .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB") .setting("xpack.searchable_snapshots.cache_fetch_async_thread_pool.keep_alive", "0ms") + .systemProperty("javax.net.ssl.trustStore", () -> trustStore.getTrustStorePath().toString(), s -> USE_FIXTURE) .build(); @ClassRule - public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster); + public static TestRule ruleChain = RuleChain.outerRule(fixture).around(trustStore).around(cluster); @Override protected final Settings restClientSettings() { diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java index 931e8790f98c6..56efc72f2f6f7 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java @@ -146,7 +146,8 @@ private void readWithoutBlobCacheSlow(ByteBuffer b, long position, int length) t final int read = SharedBytes.readCacheFile(channel, pos, relativePos, len, byteBufferReference); stats.addCachedBytesRead(read); return read; - }, (channel, channelPos, relativePos, len, progressUpdater) -> { + }, (channel, channelPos, streamFactory, relativePos, len, progressUpdater) -> { + assert streamFactory == null : streamFactory; final long startTimeNanos = stats.currentTimeNanos(); try (InputStream input = openInputStreamFromBlobStore(rangeToWrite.start() + relativePos, len)) { assert ThreadPool.assertCurrentThreadPool(SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME); diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheFileTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheFileTests.java index 92ec94963c0c6..372dddc6eca71 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheFileTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheFileTests.java @@ -14,11 +14,11 @@ import org.elasticsearch.blobcache.BlobCacheTestUtils; import org.elasticsearch.blobcache.common.ByteRange; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.filesystem.FileSystemNatives; import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue; import org.elasticsearch.core.PathUtils; import org.elasticsearch.core.PathUtilsForTesting; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.nativeaccess.NativeAccess; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFile.EvictionListener; @@ -393,15 +393,7 @@ public void testFSyncFailure() throws Exception { } } - private static void assumeLinux64bitsOrWindows() { - assumeTrue( - "This test uses native methods implemented only for Windows & Linux 64bits", - Constants.WINDOWS || Constants.LINUX && Constants.JRE_IS_64BIT - ); - } - public void testCacheFileCreatedAsSparseFile() throws Exception { - assumeLinux64bitsOrWindows(); final long fourKb = 4096L; final long oneMb = 1 << 20; @@ -420,7 +412,7 @@ public void testCacheFileCreatedAsSparseFile() throws Exception { final FileChannel fileChannel = cacheFile.getChannel(); assertTrue(Files.exists(file)); - OptionalLong sizeOnDisk = FileSystemNatives.allocatedSizeInBytes(file); + OptionalLong sizeOnDisk = NativeAccess.instance().allocatedSizeInBytes(file); assertTrue(sizeOnDisk.isPresent()); assertThat(sizeOnDisk.getAsLong(), equalTo(0L)); @@ -430,7 +422,7 @@ public void testCacheFileCreatedAsSparseFile() throws Exception { fill(fileChannel, Math.toIntExact(cacheFile.getLength() - 1L), Math.toIntExact(cacheFile.getLength())); fileChannel.force(false); - sizeOnDisk = FileSystemNatives.allocatedSizeInBytes(file); + sizeOnDisk = NativeAccess.instance().allocatedSizeInBytes(file); assertTrue(sizeOnDisk.isPresent()); assertThat( "Cache file should be sparse and not fully allocated on disk", @@ -445,7 +437,7 @@ public void testCacheFileCreatedAsSparseFile() throws Exception { fill(fileChannel, 0, Math.toIntExact(cacheFile.getLength())); fileChannel.force(false); - sizeOnDisk = FileSystemNatives.allocatedSizeInBytes(file); + sizeOnDisk = NativeAccess.instance().allocatedSizeInBytes(file); assertTrue(sizeOnDisk.isPresent()); assertThat( "Cache file should be fully allocated on disk (maybe more given cluster/block size)", diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index ffa4d1082c7e6..9eee5b0bd7a6f 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -39,6 +39,9 @@ public class Constants { "cluster:admin/indices/dangling/find", "cluster:admin/indices/dangling/import", "cluster:admin/indices/dangling/list", + "cluster:admin/ingest/geoip/database/delete", + "cluster:admin/ingest/geoip/database/get", + "cluster:admin/ingest/geoip/database/put", "cluster:admin/ingest/pipeline/delete", "cluster:admin/ingest/pipeline/get", "cluster:admin/ingest/pipeline/put", @@ -353,6 +356,7 @@ public class Constants { "cluster:monitor/main", "cluster:monitor/nodes/capabilities", "cluster:monitor/nodes/data_tier_usage", + "cluster:monitor/nodes/features", "cluster:monitor/nodes/hot_threads", "cluster:monitor/nodes/info", "cluster:monitor/nodes/stats", @@ -552,6 +556,7 @@ public class Constants { "indices:data/read/eql/async/get", "indices:data/read/esql", "indices:data/read/esql/async/get", + "indices:data/read/esql/resolve_fields", "indices:data/read/explain", "indices:data/read/field_caps", "indices:data/read/get", diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java index 704d8b75d9ed3..c0866fa7ea694 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java @@ -1013,6 +1013,10 @@ public void testZeroMinDocAggregation() throws Exception { prepareIndex("test").setId("2").setSource("color", "yellow", "fruit", "banana", "count", -2).setRefreshPolicy(IMMEDIATE).get(); prepareIndex("test").setId("3").setSource("color", "green", "fruit", "grape", "count", -3).setRefreshPolicy(IMMEDIATE).get(); prepareIndex("test").setId("4").setSource("color", "red", "fruit", "grape", "count", -4).setRefreshPolicy(IMMEDIATE).get(); + prepareIndex("test").setId("5") + .setSource("color", new String[] { "green", "black" }, "fruit", "grape", "count", -5) + .setRefreshPolicy(IMMEDIATE) + .get(); indicesAdmin().prepareForceMerge("test").get(); assertResponse( diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGrantAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGrantAction.java index fffcb476abaa4..ea3cdab66beff 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGrantAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGrantAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; @@ -50,7 +51,7 @@ public TransportGrantAction( AuthorizationService authorizationService, ThreadContext threadContext ) { - super(actionName, actionFilters, transportService.getTaskManager()); + super(actionName, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.authenticationService = authenticationService; this.authorizationService = authorizationService; this.threadContext = threadContext; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportBaseUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportBaseUpdateApiKeyAction.java index 33b1e44004454..95aa383faa246 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportBaseUpdateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportBaseUpdateApiKeyAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; @@ -32,7 +33,7 @@ protected TransportBaseUpdateApiKeyAction( final ActionFilters actionFilters, final SecurityContext context ) { - super(actionName, actionFilters, transportService.getTaskManager()); + super(actionName, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.securityContext = context; } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateApiKeyAction.java index 268afc7f0b32f..a2a68204e0d39 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateApiKeyAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -42,7 +43,7 @@ public TransportCreateApiKeyAction( CompositeRolesStore rolesStore, NamedXContentRegistry xContentRegistry ) { - super(CreateApiKeyAction.NAME, actionFilters, transportService.getTaskManager()); + super(CreateApiKeyAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.apiKeyService = apiKeyService; this.resolver = new ApiKeyUserRoleDescriptorResolver(rolesStore, xContentRegistry); this.securityContext = context; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyAction.java index eeccd4b833a23..59ee0a597ef00 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; @@ -37,7 +38,7 @@ public TransportCreateCrossClusterApiKeyAction( ApiKeyService apiKeyService, SecurityContext context ) { - super(CreateCrossClusterApiKeyAction.NAME, actionFilters, transportService.getTaskManager()); + super(CreateCrossClusterApiKeyAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.apiKeyService = apiKeyService; this.securityContext = context; } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportGetApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportGetApiKeyAction.java index 2ae790aaa94e7..80d9562627813 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportGetApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportGetApiKeyAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; @@ -36,7 +37,7 @@ public TransportGetApiKeyAction( SecurityContext context, ProfileService profileService ) { - super(GetApiKeyAction.NAME, actionFilters, transportService.getTaskManager()); + super(GetApiKeyAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.apiKeyService = apiKeyService; this.securityContext = context; this.profileService = profileService; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyAction.java index 1454b9e480a39..867e3b220deb6 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; @@ -58,7 +59,7 @@ public TransportQueryApiKeyAction( SecurityContext context, ProfileService profileService ) { - super(QueryApiKeyAction.NAME, actionFilters, transportService.getTaskManager()); + super(QueryApiKeyAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.apiKeyService = apiKeyService; this.securityContext = context; this.profileService = profileService; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetBuiltinPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetBuiltinPrivilegesAction.java index 62f0087c1a2d2..f4784d14e639a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetBuiltinPrivilegesAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetBuiltinPrivilegesAction.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.privilege.GetBuiltinPrivilegesAction; @@ -28,7 +29,7 @@ public class TransportGetBuiltinPrivilegesAction extends TransportAction getMappings() { if (enabled == false) { return Set.of(); } else { - return RoleMappingMetadata.getFromClusterState(clusterService.state()).getRoleMappings(); + final Set mappings = RoleMappingMetadata.getFromClusterState(clusterService.state()).getRoleMappings(); + logger.trace("Retrieved [{}] mapping(s) from cluster state", mappings.size()); + return mappings; } } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java index 7f35415d6f630..beabf93e80e0d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java @@ -397,6 +397,7 @@ public void onSecurityIndexStateChange(SecurityIndexManager.State previousState, @Override public void resolveRoles(UserData user, ActionListener> listener) { getRoleMappings(null, ActionListener.wrap(mappings -> { + logger.trace("Retrieved [{}] role mapping(s) from security index", mappings.size()); listener.onResponse(ExpressionRoleMapping.resolveRoles(user, mappings, scriptService, logger)); }, listener::onFailure)); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java index 1aa40a48ecc97..400bc35b93fd5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java @@ -748,7 +748,7 @@ public void testLicenseUpdateFailureHandlerUpdate() throws Exception { // On trial license, kerberos is allowed and the WWW-Authenticate response header should reflect that verifyHasAuthenticationHeaderValue( e, - "Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\"", + "Basic realm=\"" + XPackField.SECURITY + "\", charset=\"UTF-8\"", "Negotiate", "ApiKey" ); @@ -760,7 +760,7 @@ public void testLicenseUpdateFailureHandlerUpdate() throws Exception { request.getHttpRequest(), ActionListener.wrap(result -> { assertTrue(completed.compareAndSet(false, true)); }, e -> { // On basic or gold license, kerberos is not allowed and the WWW-Authenticate response header should also reflect that - verifyHasAuthenticationHeaderValue(e, "Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\"", "ApiKey"); + verifyHasAuthenticationHeaderValue(e, "Basic realm=\"" + XPackField.SECURITY + "\", charset=\"UTF-8\"", "ApiKey"); }) ); if (completed.get()) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/reservedstate/ReservedRoleMappingActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/reservedstate/ReservedRoleMappingActionTests.java index cac7c91f73ed1..978a7a44b08a5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/reservedstate/ReservedRoleMappingActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/reservedstate/ReservedRoleMappingActionTests.java @@ -21,7 +21,6 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.nullValue; /** * Tests that the ReservedRoleMappingAction does validation, can add and remove role mappings @@ -31,9 +30,7 @@ public class ReservedRoleMappingActionTests extends ESTestCase { private TransformState processJSON(ReservedRoleMappingAction action, TransformState prevState, String json) throws Exception { try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, json)) { var content = action.fromXContent(parser); - var state = action.transform(content, prevState); - assertThat(state.nonStateTransform(), nullValue()); - return state; + return action.transform(content, prevState); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index 62b72b4f9750c..85a1dc1aa029d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -1500,7 +1500,7 @@ public void testRealmAuthenticateTerminateAuthenticationProcessWithException() { final boolean throwElasticsearchSecurityException = randomBoolean(); final boolean withAuthenticateHeader = throwElasticsearchSecurityException && randomBoolean(); Exception throwE = new Exception("general authentication error"); - final String basicScheme = "Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\""; + final String basicScheme = "Basic realm=\"" + XPackField.SECURITY + "\", charset=\"UTF-8\""; String selectedScheme = randomFrom(basicScheme, "Negotiate IOJoj"); if (throwElasticsearchSecurityException) { throwE = new ElasticsearchSecurityException("authentication error", RestStatus.UNAUTHORIZED); @@ -1547,7 +1547,7 @@ public void testRealmAuthenticateGracefulTerminateAuthenticationProcess() { when(token.principal()).thenReturn(principal); when(firstRealm.token(threadContext)).thenReturn(token); when(firstRealm.supports(token)).thenReturn(true); - final String basicScheme = "Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\""; + final String basicScheme = "Basic realm=\"" + XPackField.SECURITY + "\", charset=\"UTF-8\""; mockAuthenticate(firstRealm, token, null, true); ElasticsearchSecurityException e = expectThrows( diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SimpleSecurityNetty4ServerTransportTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SimpleSecurityNetty4ServerTransportTests.java index 74b02c1d63bbf..888e858f2b039 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SimpleSecurityNetty4ServerTransportTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SimpleSecurityNetty4ServerTransportTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.TestPlainActionFuture; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; @@ -34,6 +35,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.Releasable; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.TestEnvironment; @@ -80,6 +82,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -167,19 +170,15 @@ protected Set> getSupportedSettings() { } public void testConnectException() throws UnknownHostException { - try { - connectToNode( - serviceA, - DiscoveryNodeUtils.create("C", new TransportAddress(InetAddress.getByName("localhost"), 9876), emptyMap(), emptySet()) - ); - fail("Expected ConnectTransportException"); - } catch (ConnectTransportException e) { - assertThat(e.getMessage(), containsString("connect_exception")); - assertThat(e.getMessage(), containsString("[127.0.0.1:9876]")); - Throwable cause = ExceptionsHelper.unwrap(e, IOException.class); - assertThat(cause, instanceOf(IOException.class)); - } - + final ConnectTransportException e = connectToNodeExpectFailure( + serviceA, + DiscoveryNodeUtils.create("C", new TransportAddress(InetAddress.getByName("localhost"), 9876), emptyMap(), emptySet()), + null + ); + assertThat(e.getMessage(), containsString("connect_exception")); + assertThat(e.getMessage(), containsString("[127.0.0.1:9876]")); + Throwable cause = ExceptionsHelper.unwrap(e, IOException.class); + assertThat(cause, instanceOf(IOException.class)); } @Override @@ -314,11 +313,8 @@ public boolean matches(SNIServerName sniServerName) { ); new Thread(() -> { - try { - connectToNode(serviceC, node, TestProfiles.LIGHT_PROFILE); - } catch (ConnectTransportException ex) { - // Ignore. The other side is not setup to do the ES handshake. So this will fail. - } + // noinspection ThrowableNotThrown + connectToNodeExpectFailure(serviceC, node, TestProfiles.LIGHT_PROFILE); }).start(); latch.await(); @@ -360,12 +356,10 @@ public void testInvalidSNIServerName() throws Exception { DiscoveryNodeRole.roles() ); - ConnectTransportException connectException = expectThrows( - ConnectTransportException.class, - () -> connectToNode(serviceC, node, TestProfiles.LIGHT_PROFILE) + assertThat( + connectToNodeExpectFailure(serviceC, node, TestProfiles.LIGHT_PROFILE).getMessage(), + containsString("invalid DiscoveryNode server_name [invalid_hostname]") ); - - assertThat(connectException.getMessage(), containsString("invalid DiscoveryNode server_name [invalid_hostname]")); } } } @@ -574,10 +568,7 @@ public void testClientChannelUsesSeparateSslConfigurationForRemoteCluster() thro // 1. Connection will fail because FC server certificate is not trusted by default final Settings qcSettings1 = Settings.builder().build(); try (MockTransportService qcService = buildService("QC", VersionInformation.CURRENT, TransportVersion.current(), qcSettings1)) { - final ConnectTransportException e = expectThrows( - ConnectTransportException.class, - () -> openConnection(qcService, node, connectionProfile) - ); + final ConnectTransportException e = openConnectionExpectFailure(qcService, node, connectionProfile); assertThat( e.getRootCause().getMessage(), anyOf(containsString("unable to find valid certification path"), containsString("Unable to find certificate chain")) @@ -897,11 +888,10 @@ public void testTcpHandshakeTimeout() throws IOException { builder.setHandshakeTimeout(TimeValue.timeValueMillis(1)); Settings settings = Settings.builder().put("xpack.security.transport.ssl.verification_mode", "none").build(); try (MockTransportService serviceC = buildService("TS_C", version0, transportVersion0, settings)) { - ConnectTransportException ex = expectThrows( - ConnectTransportException.class, - () -> connectToNode(serviceC, dummy, builder.build()) + assertEquals( + "[][" + dummy.getAddress() + "] handshake_timeout[1ms]", + connectToNodeExpectFailure(serviceC, dummy, builder.build()).getMessage() ); - assertEquals("[][" + dummy.getAddress() + "] handshake_timeout[1ms]", ex.getMessage()); } } finally { doneLatch.countDown(); @@ -934,10 +924,9 @@ public void testTlsHandshakeTimeout() throws IOException { TransportRequestOptions.Type.REG, TransportRequestOptions.Type.STATE ); - ConnectTransportException ex = expectThrows( - ConnectTransportException.class, - () -> connectToNode(serviceA, dummy, builder.build()) - ); + final var future = new TestPlainActionFuture(); + serviceA.connectToNode(dummy, builder.build(), future); + final var ex = expectThrows(ExecutionException.class, ConnectTransportException.class, future::get); // long wait assertEquals("[][" + dummy.getAddress() + "] connect_exception", ex.getMessage()); assertNotNull(ExceptionsHelper.unwrap(ex, SslHandshakeTimeoutException.class)); } finally { @@ -982,10 +971,7 @@ public void testTcpHandshakeConnectionReset() throws IOException, InterruptedExc builder.setHandshakeTimeout(TimeValue.timeValueHours(1)); Settings settings = Settings.builder().put("xpack.security.transport.ssl.verification_mode", "none").build(); try (MockTransportService serviceC = buildService("TS_C", version0, transportVersion0, settings)) { - ConnectTransportException ex = expectThrows( - ConnectTransportException.class, - () -> connectToNode(serviceC, dummy, builder.build()) - ); + ConnectTransportException ex = connectToNodeExpectFailure(serviceC, dummy, builder.build()); assertEquals("[][" + dummy.getAddress() + "] general node connection failure", ex.getMessage()); assertThat(ex.getCause().getMessage(), startsWith("handshake failed")); } diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java index 20dedbaa161be..621836cea9f89 100644 --- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java +++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java @@ -76,7 +76,7 @@ public List getRestHandlers( @UpdateForV9 // always true in v9 so can be removed static boolean serializesWithParentTaskAndTimeouts(TransportVersion transportVersion) { return transportVersion.isPatchFrom(TransportVersions.V_8_13_4) - || transportVersion.isPatchFrom(TransportVersions.SHUTDOWN_REQUEST_TIMEOUTS_FIX_8_14) + || transportVersion.isPatchFrom(TransportVersions.V_8_14_0) || transportVersion.onOrAfter(TransportVersions.SHUTDOWN_REQUEST_TIMEOUTS_FIX); } } diff --git a/x-pack/plugin/slm/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/slm/11_basic_slm.yml b/x-pack/plugin/slm/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/slm/11_basic_slm.yml index e9a1ca0bcecf1..2b3880f902c3d 100644 --- a/x-pack/plugin/slm/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/slm/11_basic_slm.yml +++ b/x-pack/plugin/slm/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/slm/11_basic_slm.yml @@ -99,3 +99,67 @@ setup: - match: { daily-snapshots.policy.schedule: "1 1 1 1 1 ?" } - is_true: daily-snapshots.next_execution_millis - is_true: daily-snapshots.stats + +--- +"Test version/modified_date is unchanged if updated policy is the same": + - do: + snapshot.create_repository: + repository: repo + body: + type: fs + settings: + location: "my-snaps" + + - do: + slm.put_lifecycle: + policy_id: "daily-snapshots" + body: | + { + "schedule": "0 1 2 3 4 ?", + "name": "", + "repository": "repo", + "config": { + "indices": ["foo-*", "important"], + "ignore_unavailable": false, + "include_global_state": false + }, + "retention": { + "expire_after": "30d", + "min_count": 1, + "max_count": 50 + } + } + + - do: + slm.get_lifecycle: + policy_id: "daily-snapshots" + - match: { daily-snapshots.policy.name: "" } + - match: { daily-snapshots.version: 1 } + - set: { daily-snapshots.modified_date_millis: modified } + + - do: + slm.put_lifecycle: + policy_id: "daily-snapshots" + body: | + { + "schedule": "0 1 2 3 4 ?", + "name": "", + "repository": "repo", + "config": { + "indices": ["foo-*", "important"], + "ignore_unavailable": false, + "include_global_state": false + }, + "retention": { + "expire_after": "30d", + "min_count": 1, + "max_count": 50 + } + } + + - do: + slm.get_lifecycle: + policy_id: "daily-snapshots" + - match: { daily-snapshots.policy.name: "" } + - match: { daily-snapshots.version: 1 } + - match: { daily-snapshots.modified_date_millis: $modified } diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/TransportSLMGetExpiredSnapshotsAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/TransportSLMGetExpiredSnapshotsAction.java index cf3a114fc5803..1401a9522b52d 100644 --- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/TransportSLMGetExpiredSnapshotsAction.java +++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/TransportSLMGetExpiredSnapshotsAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Tuple; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; @@ -70,7 +71,7 @@ public TransportSLMGetExpiredSnapshotsAction( RepositoriesService repositoriesService, ActionFilters actionFilters ) { - super(INSTANCE.name(), actionFilters, transportService.getTaskManager()); + super(INSTANCE.name(), actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.repositoriesService = repositoriesService; this.retentionExecutor = transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT); } diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java index 1e7f58b02e2ac..9b3f261976125 100644 --- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java +++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.tasks.Task; @@ -32,13 +33,12 @@ import org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata; import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy; import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadata; -import org.elasticsearch.xpack.core.slm.SnapshotLifecycleStats; import org.elasticsearch.xpack.core.slm.action.PutSnapshotLifecycleAction; import org.elasticsearch.xpack.slm.SnapshotLifecycleService; import java.time.Instant; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -88,6 +88,7 @@ protected void masterOperation( // same context, and therefore does not have access to the appropriate security headers. final Map filteredHeaders = ClientHelper.getPersistableSafeSecurityHeaders(threadPool.getThreadContext(), state); LifecyclePolicy.validatePolicyName(request.getLifecycleId()); + submitUnbatchedTask( "put-snapshot-lifecycle-" + request.getLifecycleId(), new UpdateSnapshotPolicyTask(request, listener, filteredHeaders) @@ -119,49 +120,46 @@ public static class UpdateSnapshotPolicyTask extends AckedClusterStateUpdateTask UpdateSnapshotPolicyTask(PutSnapshotLifecycleAction.Request request) { super(request, null); this.request = request; - this.filteredHeaders = Collections.emptyMap(); + this.filteredHeaders = Map.of(); } @Override public ClusterState execute(ClusterState currentState) { - SnapshotLifecycleMetadata snapMeta = currentState.metadata().custom(SnapshotLifecycleMetadata.TYPE); + SnapshotLifecycleMetadata snapMeta = currentState.metadata() + .custom(SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata.EMPTY); var currentMode = LifecycleOperationMetadata.currentSLMMode(currentState); + final SnapshotLifecyclePolicyMetadata existingPolicyMetadata = snapMeta.getSnapshotConfigurations() + .get(request.getLifecycleId()); + + // check for no-op in the state update task, in case it was changed/reset in the meantime + if (isNoopUpdate(existingPolicyMetadata, request.getLifecycle(), filteredHeaders)) { + return currentState; + } - String id = request.getLifecycleId(); - final SnapshotLifecycleMetadata lifecycleMetadata; - if (snapMeta == null) { - SnapshotLifecyclePolicyMetadata meta = SnapshotLifecyclePolicyMetadata.builder() - .setPolicy(request.getLifecycle()) - .setHeaders(filteredHeaders) - .setModifiedDate(Instant.now().toEpochMilli()) - .build(); - lifecycleMetadata = new SnapshotLifecycleMetadata( - Collections.singletonMap(id, meta), - currentMode, - new SnapshotLifecycleStats() - ); - logger.info("adding new snapshot lifecycle [{}]", id); + long nextVersion = (existingPolicyMetadata == null) ? 1L : existingPolicyMetadata.getVersion() + 1L; + Map snapLifecycles = new HashMap<>(snapMeta.getSnapshotConfigurations()); + SnapshotLifecyclePolicyMetadata newLifecycle = SnapshotLifecyclePolicyMetadata.builder(existingPolicyMetadata) + .setPolicy(request.getLifecycle()) + .setHeaders(filteredHeaders) + .setVersion(nextVersion) + .setModifiedDate(Instant.now().toEpochMilli()) + .build(); + + SnapshotLifecyclePolicyMetadata oldPolicy = snapLifecycles.put(newLifecycle.getId(), newLifecycle); + if (oldPolicy == null) { + logger.info("adding new snapshot lifecycle [{}]", newLifecycle.getId()); } else { - Map snapLifecycles = new HashMap<>(snapMeta.getSnapshotConfigurations()); - SnapshotLifecyclePolicyMetadata oldLifecycle = snapLifecycles.get(id); - SnapshotLifecyclePolicyMetadata newLifecycle = SnapshotLifecyclePolicyMetadata.builder(oldLifecycle) - .setPolicy(request.getLifecycle()) - .setHeaders(filteredHeaders) - .setVersion(oldLifecycle == null ? 1L : oldLifecycle.getVersion() + 1) - .setModifiedDate(Instant.now().toEpochMilli()) - .build(); - snapLifecycles.put(id, newLifecycle); - lifecycleMetadata = new SnapshotLifecycleMetadata(snapLifecycles, currentMode, snapMeta.getStats()); - if (oldLifecycle == null) { - logger.info("adding new snapshot lifecycle [{}]", id); - } else { - logger.info("updating existing snapshot lifecycle [{}]", id); - } + logger.info("updating existing snapshot lifecycle [{}]", newLifecycle.getId()); } - Metadata currentMeta = currentState.metadata(); return ClusterState.builder(currentState) - .metadata(Metadata.builder(currentMeta).putCustom(SnapshotLifecycleMetadata.TYPE, lifecycleMetadata)) + .metadata( + Metadata.builder(currentState.metadata()) + .putCustom( + SnapshotLifecycleMetadata.TYPE, + new SnapshotLifecycleMetadata(snapLifecycles, currentMode, snapMeta.getStats()) + ) + ) .build(); } } @@ -185,4 +183,19 @@ public Optional reservedStateHandlerName() { public Set modifiedKeys(PutSnapshotLifecycleAction.Request request) { return Set.of(request.getLifecycleId()); } + + /** + * Returns 'true' if the SLM is effectually the same (same policy and headers), and thus can be a no-op update. + */ + static boolean isNoopUpdate( + @Nullable SnapshotLifecyclePolicyMetadata existingPolicyMeta, + SnapshotLifecyclePolicy newPolicy, + Map filteredHeaders + ) { + if (existingPolicyMeta == null) { + return false; + } else { + return newPolicy.equals(existingPolicyMeta.getPolicy()) && filteredHeaders.equals(existingPolicyMeta.getHeaders()); + } + } } diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java index 207c683de0f49..71346ebc495d4 100644 --- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java +++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java @@ -38,16 +38,21 @@ import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata; +import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy; +import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadataTests; import org.elasticsearch.xpack.core.slm.action.DeleteSnapshotLifecycleAction; import org.elasticsearch.xpack.core.slm.action.PutSnapshotLifecycleAction; import org.junit.Assert; import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import static org.elasticsearch.xpack.core.slm.SnapshotInvocationRecordTests.randomSnapshotInvocationRecord; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; @@ -424,4 +429,29 @@ public void testPutSLMReservedStateHandler() throws Exception { } } + public void testIsNoop() { + SnapshotLifecyclePolicy existingPolicy = SnapshotLifecyclePolicyMetadataTests.randomSnapshotLifecyclePolicy("id1"); + SnapshotLifecyclePolicy newPolicy = randomValueOtherThan( + existingPolicy, + () -> SnapshotLifecyclePolicyMetadataTests.randomSnapshotLifecyclePolicy("id2") + ); + + Map existingHeaders = Map.of("foo", "bar"); + Map newHeaders = Map.of("foo", "eggplant"); + + SnapshotLifecyclePolicyMetadata existingPolicyMeta = new SnapshotLifecyclePolicyMetadata( + existingPolicy, + existingHeaders, + randomNonNegativeLong(), + randomNonNegativeLong(), + randomSnapshotInvocationRecord(), + randomSnapshotInvocationRecord(), + randomNonNegativeLong() + ); + + assertTrue(TransportPutSnapshotLifecycleAction.isNoopUpdate(existingPolicyMeta, existingPolicy, existingHeaders)); + assertFalse(TransportPutSnapshotLifecycleAction.isNoopUpdate(existingPolicyMeta, newPolicy, existingHeaders)); + assertFalse(TransportPutSnapshotLifecycleAction.isNoopUpdate(existingPolicyMeta, existingPolicy, newHeaders)); + assertFalse(TransportPutSnapshotLifecycleAction.isNoopUpdate(null, existingPolicy, existingHeaders)); + } } diff --git a/x-pack/plugin/snapshot-based-recoveries/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/AzureSnapshotBasedRecoveryIT.java b/x-pack/plugin/snapshot-based-recoveries/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/AzureSnapshotBasedRecoveryIT.java index 233851919ebe4..8cebe9fafdb52 100644 --- a/x-pack/plugin/snapshot-based-recoveries/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/AzureSnapshotBasedRecoveryIT.java +++ b/x-pack/plugin/snapshot-based-recoveries/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/AzureSnapshotBasedRecoveryIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Booleans; +import org.elasticsearch.test.TestTrustStore; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.junit.ClassRule; import org.junit.rules.RuleChain; @@ -26,7 +27,16 @@ public class AzureSnapshotBasedRecoveryIT extends AbstractSnapshotBasedRecoveryR private static final String AZURE_TEST_KEY = System.getProperty("test.azure.key"); private static final String AZURE_TEST_SASTOKEN = System.getProperty("test.azure.sas_token"); - private static AzureHttpFixture fixture = new AzureHttpFixture(USE_FIXTURE, AZURE_TEST_ACCOUNT, AZURE_TEST_CONTAINER); + private static AzureHttpFixture fixture = new AzureHttpFixture( + USE_FIXTURE ? AzureHttpFixture.Protocol.HTTPS : AzureHttpFixture.Protocol.NONE, + AZURE_TEST_ACCOUNT, + AZURE_TEST_CONTAINER, + AzureHttpFixture.sharedKeyForAccountPredicate(AZURE_TEST_ACCOUNT) + ); + + private static TestTrustStore trustStore = new TestTrustStore( + () -> AzureHttpFixture.class.getResourceAsStream("azure-http-fixture.pem") + ); private static ElasticsearchCluster cluster = ElasticsearchCluster.local() .nodes(3) @@ -49,10 +59,11 @@ public class AzureSnapshotBasedRecoveryIT extends AbstractSnapshotBasedRecoveryR s -> USE_FIXTURE ) .setting("xpack.license.self_generated.type", "trial") + .systemProperty("javax.net.ssl.trustStore", () -> trustStore.getTrustStorePath().toString(), s -> USE_FIXTURE) .build(); @ClassRule - public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster); + public static TestRule ruleChain = RuleChain.outerRule(fixture).around(trustStore).around(cluster); @Override protected String getTestRestCluster() { diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/azure/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/AzureSnapshotRepoTestKitIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/azure/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/AzureSnapshotRepoTestKitIT.java index bae37bd43b32b..afe17d2dd6f2d 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/azure/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/AzureSnapshotRepoTestKitIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/azure/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/AzureSnapshotRepoTestKitIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Booleans; +import org.elasticsearch.test.TestTrustStore; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.junit.ClassRule; import org.junit.rules.RuleChain; @@ -25,7 +26,16 @@ public class AzureSnapshotRepoTestKitIT extends AbstractSnapshotRepoTestKitRestT private static final String AZURE_TEST_KEY = System.getProperty("test.azure.key"); private static final String AZURE_TEST_SASTOKEN = System.getProperty("test.azure.sas_token"); - private static AzureHttpFixture fixture = new AzureHttpFixture(USE_FIXTURE, AZURE_TEST_ACCOUNT, AZURE_TEST_CONTAINER); + private static AzureHttpFixture fixture = new AzureHttpFixture( + USE_FIXTURE ? AzureHttpFixture.Protocol.HTTPS : AzureHttpFixture.Protocol.NONE, + AZURE_TEST_ACCOUNT, + AZURE_TEST_CONTAINER, + AzureHttpFixture.sharedKeyForAccountPredicate(AZURE_TEST_ACCOUNT) + ); + + private static TestTrustStore trustStore = new TestTrustStore( + () -> AzureHttpFixture.class.getResourceAsStream("azure-http-fixture.pem") + ); private static ElasticsearchCluster cluster = ElasticsearchCluster.local() .module("repository-azure") @@ -52,10 +62,11 @@ public class AzureSnapshotRepoTestKitIT extends AbstractSnapshotRepoTestKitRestT c.systemProperty("test.repository_test_kit.skip_cas", "true"); } }) + .systemProperty("javax.net.ssl.trustStore", () -> trustStore.getTrustStorePath().toString(), s -> USE_FIXTURE) .build(); @ClassRule - public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster); + public static TestRule ruleChain = RuleChain.outerRule(fixture).around(trustStore).around(cluster); @Override protected String getTestRestCluster() { diff --git a/x-pack/plugin/sql/sql-cli/build.gradle b/x-pack/plugin/sql/sql-cli/build.gradle index 2492086ac1fbb..1d3a63ec13c98 100644 --- a/x-pack/plugin/sql/sql-cli/build.gradle +++ b/x-pack/plugin/sql/sql-cli/build.gradle @@ -30,7 +30,7 @@ dependencies { api project(':x-pack:plugin:sql:sql-client') api project(":libs:elasticsearch-cli") - runtimeOnly "net.java.dev.jna:jna:${versions.jna}" + implementation "net.java.dev.jna:jna:${versions.jna}" testImplementation project(":test:framework") } diff --git a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java index d093332e48422..258a738a076c8 100644 --- a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java +++ b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java @@ -61,7 +61,7 @@ public void testParseMissingAuth() throws IOException { assertEquals("missing authentication token for REST request [/?pretty&error_trace]", failure.reason()); assertThat(failure.remoteTrace(), containsString("DefaultAuthenticationFailureHandler.missingToken")); assertNull(failure.cause()); - assertEquals(singletonMap("WWW-Authenticate", "Basic realm=\"security\" charset=\"UTF-8\""), failure.headers()); + assertEquals(singletonMap("WWW-Authenticate", "Basic realm=\"security\", charset=\"UTF-8\""), failure.headers()); } public void testNoError() { diff --git a/x-pack/plugin/sql/sql-client/src/test/resources/remote_failure/missing_auth.json b/x-pack/plugin/sql/sql-client/src/test/resources/remote_failure/missing_auth.json index 3d2927f85d6b1..d21fece75f7ac 100644 --- a/x-pack/plugin/sql/sql-client/src/test/resources/remote_failure/missing_auth.json +++ b/x-pack/plugin/sql/sql-client/src/test/resources/remote_failure/missing_auth.json @@ -5,7 +5,7 @@ "type" : "security_exception", "reason" : "missing authentication token for REST request [/?pretty&error_trace]", "header" : { - "WWW-Authenticate" : "Basic realm=\"security\" charset=\"UTF-8\"" + "WWW-Authenticate" : "Basic realm=\"security\", charset=\"UTF-8\"" }, "stack_trace" : "ElasticsearchSecurityException[missing authentication token for REST request [/?pretty&error_trace]]\n\tat org.elasticsearch.xpack.security.support.Exceptions.authenticationError(Exceptions.java:36)\n\tat org.elasticsearch.xpack.security.authc.DefaultAuthenticationFailureHandler.missingToken(DefaultAuthenticationFailureHandler.java:69)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$AuditableRestRequest.anonymousAccessDenied(AuthenticationService.java:603)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$handleNullToken$17(AuthenticationService.java:357)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.handleNullToken(AuthenticationService.java:362)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.consumeToken(AuthenticationService.java:277)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$extractToken$7(AuthenticationService.java:249)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.extractToken(AuthenticationService.java:266)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$null$0(AuthenticationService.java:201)\n\tat org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:59)\n\tat org.elasticsearch.xpack.security.authc.TokenService.getAndValidateToken(TokenService.java:230)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:197)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$4(AuthenticationService.java:228)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:239)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:193)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:147)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:99)\n\tat org.elasticsearch.xpack.security.rest.SecurityRestFilter.handleRequest(SecurityRestFilter.java:69)\n\tat org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:240)\n\tat org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:336)\n\tat org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:174)\n\tat org.elasticsearch.http.netty4.Netty4HttpServerTransport.dispatchRequest(Netty4HttpServerTransport.java:469)\n\tat org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:80)\n\tat io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)\n\tat org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler.channelRead(HttpPipeliningHandler.java:68)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)\n\tat io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102)\n\tat io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)\n\tat io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)\n\tat io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)\n\tat io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)\n\tat io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)\n\tat io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)\n\tat io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)\n\tat io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)\n\tat java.lang.Thread.run(Thread.java:748)\n" } @@ -13,7 +13,7 @@ "type" : "security_exception", "reason" : "missing authentication token for REST request [/?pretty&error_trace]", "header" : { - "WWW-Authenticate" : "Basic realm=\"security\" charset=\"UTF-8\"" + "WWW-Authenticate" : "Basic realm=\"security\", charset=\"UTF-8\"" }, "stack_trace" : "ElasticsearchSecurityException[missing authentication token for REST request [/?pretty&error_trace]]\n\tat org.elasticsearch.xpack.security.support.Exceptions.authenticationError(Exceptions.java:36)\n\tat org.elasticsearch.xpack.security.authc.DefaultAuthenticationFailureHandler.missingToken(DefaultAuthenticationFailureHandler.java:69)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$AuditableRestRequest.anonymousAccessDenied(AuthenticationService.java:603)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$handleNullToken$17(AuthenticationService.java:357)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.handleNullToken(AuthenticationService.java:362)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.consumeToken(AuthenticationService.java:277)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$extractToken$7(AuthenticationService.java:249)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.extractToken(AuthenticationService.java:266)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$null$0(AuthenticationService.java:201)\n\tat org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:59)\n\tat org.elasticsearch.xpack.security.authc.TokenService.getAndValidateToken(TokenService.java:230)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:197)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$4(AuthenticationService.java:228)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:239)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:193)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:147)\n\tat org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:99)\n\tat org.elasticsearch.xpack.security.rest.SecurityRestFilter.handleRequest(SecurityRestFilter.java:69)\n\tat org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:240)\n\tat org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:336)\n\tat org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:174)\n\tat org.elasticsearch.http.netty4.Netty4HttpServerTransport.dispatchRequest(Netty4HttpServerTransport.java:469)\n\tat org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:80)\n\tat io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)\n\tat org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler.channelRead(HttpPipeliningHandler.java:68)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)\n\tat io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102)\n\tat io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)\n\tat io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)\n\tat io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)\n\tat io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)\n\tat io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)\n\tat io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)\n\tat io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)\n\tat io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926)\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)\n\tat java.lang.Thread.run(Thread.java:748)\n" }, diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/enrich/20_standard_index.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/enrich/20_standard_index.yml index c767e3baac38f..971e276aab32a 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/enrich/20_standard_index.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/enrich/20_standard_index.yml @@ -187,3 +187,61 @@ enrich documents over _bulk via an alias: - do: enrich.delete_policy: name: test_alias_policy + +--- +enrich stats REST response structure: + - requires: + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_enrich/stats + capabilities: + - size-in-bytes + reason: "Capability required to run test" + + - do: + ingest.simulate: + id: test_pipeline + body: > + { + "docs": [ + { + "_index": "enrich-cache-stats-index", + "_id": "1", + "_source": {"baz": "quick", "c": 1} + }, + { + "_index": "enrich-cache-stats-index", + "_id": "2", + "_source": {"baz": "lazy", "c": 2} + }, + { + "_index": "enrich-cache-stats-index", + "_id": "3", + "_source": {"baz": "slow", "c": 3} + } + ] + } + - length: { docs: 3 } + + # This test's main purpose is to verify the REST response structure. + # So, rather than assessing specific values, we only assess the existence of fields. + - do: + enrich.stats: {} + - exists: executing_policies + - is_true: coordinator_stats + # We know there will be at least one node, but we don't want to be dependent on the exact number of nodes. + - is_true: coordinator_stats.0.node_id + - exists: coordinator_stats.0.queue_size + - exists: coordinator_stats.0.remote_requests_current + - exists: coordinator_stats.0.remote_requests_total + - exists: coordinator_stats.0.executed_searches_total + - is_true: cache_stats + - is_true: cache_stats.0.node_id + - exists: cache_stats.0.count + - exists: cache_stats.0.hits + - exists: cache_stats.0.misses + - exists: cache_stats.0.evictions + - exists: cache_stats.0.hits_time_in_millis + - exists: cache_stats.0.misses_time_in_millis + - exists: cache_stats.0.size_in_bytes diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/140_metadata.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/140_metadata.yml index d6c1c6c97944a..33c9cc7558672 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/140_metadata.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/140_metadata.yml @@ -5,7 +5,7 @@ setup: - method: POST path: /_query parameters: [method, path, parameters, capabilities] - capabilities: [metadata_fields, metadata_field_ignored] + capabilities: [metadata_fields, metadata_ignored_field] reason: "Ignored metadata field capability required" - do: @@ -140,3 +140,18 @@ setup: - match: {columns.0.name: "count_distinct(_ignored)"} - match: {columns.0.type: "long"} - match: {values.0.0: 2} + +--- +"Count_distinct on _source is a 400 error": + - requires: + capabilities: + - method: POST + path: /_query + parameters: [method, path, parameters, capabilities] + capabilities: [fix_count_distinct_source_error] + reason: "Capability marks fixing this bug" + - do: + catch: bad_request + esql.query: + body: + query: 'FROM test [metadata _source] | STATS COUNT_DISTINCT(_source)' diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml index aac60d9aaa8d0..003b1d0651d11 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml @@ -4,8 +4,8 @@ setup: - method: POST path: /_query parameters: [method, path, parameters, capabilities] - capabilities: [union_types] - reason: "Union types introduced in 8.15.0" + capabilities: [union_types, union_types_remove_fields, casting_operator] + reason: "Union types and casting operator introduced in 8.15.0" test_runner_features: [capabilities, allowed_warnings_regex] - do: @@ -204,13 +204,6 @@ load single index keyword_keyword: --- load single index ip_long and aggregate by client_ip: - - requires: - capabilities: - - method: POST - path: /_query - parameters: [method, path, parameters, capabilities] - capabilities: [casting_operator] - reason: "Casting operator and introduced in 8.15.0" - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" @@ -234,13 +227,6 @@ load single index ip_long and aggregate by client_ip: --- load single index ip_long and aggregate client_ip my message: - - requires: - capabilities: - - method: POST - path: /_query - parameters: [method, path, parameters, capabilities] - capabilities: [casting_operator] - reason: "Casting operator and introduced in 8.15.0" - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" @@ -266,13 +252,6 @@ load single index ip_long and aggregate client_ip my message: --- load single index ip_long stats invalid grouping: - - requires: - capabilities: - - method: POST - path: /_query - parameters: [method, path, parameters, capabilities] - capabilities: [casting_operator] - reason: "Casting operator and introduced in 8.15.0" - do: catch: '/Unknown column \[x\]/' esql.query: @@ -591,13 +570,6 @@ load two indices, convert, rename but not drop ambiguous field client_ip: --- load two indexes and group by converted client_ip: - - requires: - capabilities: - - method: POST - path: /_query - parameters: [method, path, parameters, capabilities] - capabilities: [casting_operator, union_types_agg_cast] - reason: "Casting operator and Union types introduced in 8.15.0" - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" @@ -621,13 +593,6 @@ load two indexes and group by converted client_ip: --- load two indexes and aggregate converted client_ip: - - requires: - capabilities: - - method: POST - path: /_query - parameters: [method, path, parameters, capabilities] - capabilities: [casting_operator, union_types_agg_cast] - reason: "Casting operator and Union types introduced in 8.15.0" - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" @@ -653,13 +618,6 @@ load two indexes and aggregate converted client_ip: --- load two indexes, convert client_ip and group by something invalid: - - requires: - capabilities: - - method: POST - path: /_query - parameters: [method, path, parameters, capabilities] - capabilities: [casting_operator, union_types_agg_cast] - reason: "Casting operator and Union types introduced in 8.15.0" - do: catch: '/Unknown column \[x\]/' esql.query: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/161_union_types_subfields.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/161_union_types_subfields.yml index 99bd1d6508895..ccf6512ca1ff7 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/161_union_types_subfields.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/161_union_types_subfields.yml @@ -4,7 +4,7 @@ setup: - method: POST path: /_query parameters: [ method, path, parameters, capabilities ] - capabilities: [ union_types ] + capabilities: [ union_types, union_types_remove_fields ] reason: "Union types introduced in 8.15.0" test_runner_features: [ capabilities, allowed_warnings_regex ] diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java index 4d2789dbb8591..62d22c0c0a9cc 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java @@ -54,7 +54,7 @@ public class LegacyStackTemplateRegistry extends IndexTemplateRegistry { private static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of( "xpack.stack.template.deprecated", "true", - "xpack.stack.template.logs.index.mode", + "xpack.stack.template.logsdb.index.mode", "standard" ); diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 648146ccdcc61..6a9936f4f27d3 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.features.FeatureService; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -58,7 +59,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { ); /** - * if index.mode "logs" is applied by default in logs@settings for 'logs-*-*' + * if index.mode "logsdb" is applied by default in logs@settings for 'logs-*-*' */ public static final Setting CLUSTER_LOGSDB_ENABLED = Setting.boolSetting( "cluster.logsdb.enabled", @@ -146,7 +147,7 @@ private Map loadComponentTemplateConfigs(boolean logs ), new IndexTemplateConfig( LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, - logsDbEnabled ? "/logs@mappings-logsdb.json" : "/logs@mappings.json", + "/logs@mappings.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE, ADDITIONAL_TEMPLATE_VARIABLES @@ -166,8 +167,8 @@ private Map loadComponentTemplateConfigs(boolean logs Map.of( "xpack.stack.template.deprecated", "false", - "xpack.stack.template.logs.index.mode", - logsDbEnabled ? "logs" : "standard" + "xpack.stack.template.logsdb.index.mode", + logsDbEnabled ? IndexMode.LOGSDB.getName() : IndexMode.STANDARD.getName() ) ), new IndexTemplateConfig( diff --git a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportTestGrokPatternAction.java b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportTestGrokPatternAction.java index f8ce7a1099952..a3532e47a21ce 100644 --- a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportTestGrokPatternAction.java +++ b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportTestGrokPatternAction.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.common.inject.Inject; @@ -32,18 +31,15 @@ public class TransportTestGrokPatternAction extends TransportAction listener) { - // As matching a regular expression might take a while, we run in a different thread to avoid blocking the network thread. - threadPool.generic().execute(ActionRunnable.supply(listener, () -> getResponse(request))); + listener.onResponse(getResponse(request)); } private TestGrokPatternAction.Response getResponse(TestGrokPatternAction.Request request) { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPutTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPutTransformAction.java index 4c978b1504a0f..ef42a2781962a 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPutTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPutTransformAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.ParentTaskAssigningClient; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; @@ -25,6 +26,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ClientHelper; @@ -110,9 +112,11 @@ protected void masterOperation(Task task, Request request, ClusterState clusterS ); // <2> Validate source and destination indices + + var parentTaskId = new TaskId(clusterService.localNode().getId(), task.getId()); ActionListener checkPrivilegesListener = validateTransformListener.delegateFailureAndWrap( (l, aVoid) -> ClientHelper.executeAsyncWithOrigin( - client, + new ParentTaskAssigningClient(client, parentTaskId), ClientHelper.TRANSFORM_ORIGIN, ValidateTransformAction.INSTANCE, new ValidateTransformAction.Request(config, request.isDeferValidation(), request.ackTimeout()), diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java index 23212636dc33c..59df3fa67074d 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.ParentTaskAssigningClient; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; @@ -31,6 +32,7 @@ import org.elasticsearch.persistent.PersistentTasksService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ClientHelper; @@ -126,23 +128,25 @@ protected TransportStartTransformAction( @Override protected void masterOperation( - Task ignoredTask, + Task task, StartTransformAction.Request request, ClusterState state, ActionListener listener ) { TransformNodes.warnIfNoTransformNodes(state); - final SetOnce transformTaskParamsHolder = new SetOnce<>(); - final SetOnce transformConfigHolder = new SetOnce<>(); + var transformTaskParamsHolder = new SetOnce(); + var transformConfigHolder = new SetOnce(); + var parentTaskId = new TaskId(clusterService.localNode().getId(), task.getId()); + var parentClient = new ParentTaskAssigningClient(client, parentTaskId); // <5> Wait for the allocated task's state to STARTED ActionListener> newPersistentTaskActionListener = ActionListener - .wrap(task -> { + .wrap(t -> { TransformTaskParams transformTask = transformTaskParamsHolder.get(); assert transformTask != null; waitForTransformTaskStarted( - task.getId(), + t.getId(), transformTask, request.ackTimeout(), ActionListener.wrap(taskStarted -> listener.onResponse(new StartTransformAction.Response(true)), listener::onFailure) @@ -196,7 +200,7 @@ protected void masterOperation( return; } TransformIndex.createDestinationIndex( - client, + parentClient, auditor, indexNameExpressionResolver, state, @@ -257,7 +261,7 @@ protected void masterOperation( ) ); ClientHelper.executeAsyncWithOrigin( - client, + parentClient, ClientHelper.TRANSFORM_ORIGIN, ValidateTransformAction.INSTANCE, new ValidateTransformAction.Request(config, false, request.ackTimeout()), diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportValidateTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportValidateTransformAction.java index 71593d416577e..7041f18df1e4a 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportValidateTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportValidateTransformAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.ParentTaskAssigningClient; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -23,6 +24,7 @@ import org.elasticsearch.license.License; import org.elasticsearch.license.RemoteClusterLicenseChecker; import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.common.validation.SourceDestValidator; import org.elasticsearch.xpack.core.transform.TransformDeprecations; @@ -30,8 +32,6 @@ import org.elasticsearch.xpack.core.transform.action.ValidateTransformAction; import org.elasticsearch.xpack.core.transform.action.ValidateTransformAction.Request; import org.elasticsearch.xpack.core.transform.action.ValidateTransformAction.Response; -import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; -import org.elasticsearch.xpack.transform.transforms.Function; import org.elasticsearch.xpack.transform.transforms.FunctionFactory; import org.elasticsearch.xpack.transform.transforms.TransformNodes; import org.elasticsearch.xpack.transform.utils.SourceDestValidations; @@ -99,8 +99,10 @@ protected void doExecute(Task task, Request request, ActionListener li TransformNodes.warnIfNoTransformNodes(clusterState); - final TransformConfig config = request.getConfig(); - final Function function = FunctionFactory.create(config); + var config = request.getConfig(); + var function = FunctionFactory.create(config); + var parentTaskId = new TaskId(clusterService.localNode().getId(), task.getId()); + var parentClient = new ParentTaskAssigningClient(client, parentTaskId); if (config.getVersion() == null || config.getVersion().before(TransformDeprecations.MIN_TRANSFORM_VERSION)) { listener.onFailure( @@ -130,7 +132,7 @@ protected void doExecute(Task task, Request request, ActionListener li if (request.isDeferValidation()) { deduceMappingsListener.onResponse(emptyMap()); } else { - function.deduceMappings(client, config.getHeaders(), config.getId(), config.getSource(), deduceMappingsListener); + function.deduceMappings(parentClient, config.getHeaders(), config.getId(), config.getSource(), deduceMappingsListener); } }, listener::onFailure); @@ -139,7 +141,7 @@ protected void doExecute(Task task, Request request, ActionListener li if (request.isDeferValidation()) { validateQueryListener.onResponse(true); } else { - function.validateQuery(client, config.getHeaders(), config.getSource(), request.ackTimeout(), validateQueryListener); + function.validateQuery(parentClient, config.getHeaders(), config.getSource(), request.ackTimeout(), validateQueryListener); } }, listener::onFailure); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestPutTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestPutTransformAction.java index 78bcb9a12ffc0..e80d61589fed4 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestPutTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestPutTransformAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; @@ -66,6 +67,10 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient PutTransformAction.Request request = PutTransformAction.Request.fromXContent(parser, id, deferValidation, timeout); - return channel -> client.execute(PutTransformAction.INSTANCE, request, new RestToXContentListener<>(channel)); + return channel -> new RestCancellableNodeClient(client, restRequest.getHttpChannel()).execute( + PutTransformAction.INSTANCE, + request, + new RestToXContentListener<>(channel) + ); } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestStartTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestStartTransformAction.java index fdfe2fe1744e7..9f2f310d7a9b9 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestStartTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestStartTransformAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.core.transform.TransformField; @@ -45,7 +46,11 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient TimeValue timeout = restRequest.paramAsTime(TransformField.TIMEOUT.getPreferredName(), AcknowledgedRequest.DEFAULT_ACK_TIMEOUT); StartTransformAction.Request request = new StartTransformAction.Request(id, from, timeout); - return channel -> client.execute(StartTransformAction.INSTANCE, request, new RestToXContentListener<>(channel)); + return channel -> new RestCancellableNodeClient(client, restRequest.getHttpChannel()).execute( + StartTransformAction.INSTANCE, + request, + new RestToXContentListener<>(channel) + ); } private static Instant parseDateOrThrow(String date, ParseField paramName, LongSupplier now) { diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java index e8bda244271c0..19cd37400a01c 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.watcher.test.integration; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.sort.SortBuilders; @@ -15,6 +16,7 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.watcher.actions.ActionStatus; import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder; +import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField; import org.elasticsearch.xpack.core.watcher.input.Input; import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource; import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder; @@ -71,10 +73,7 @@ public void testThatHistoryIsWrittenWithChainedInput() throws Exception { new ExecuteWatchRequestBuilder(client()).setId("test_watch").setRecordExecution(true).get(); - assertBusy(() -> { - flushAndRefresh(".watcher-history-*"); - assertHitCount(prepareSearch(".watcher-history-*"), 1); - }); + assertBusy(() -> { assertHitCount(getWatchHistory(), 1); }); } // See https://github.com/elastic/x-plugins/issues/2913 @@ -104,10 +103,7 @@ public void testFailedInputResultWithDotsInFieldNameGetsStored() throws Exceptio new ExecuteWatchRequestBuilder(client()).setId("test_watch").setRecordExecution(true).get(); - assertBusy(() -> { - refresh(".watcher-history*"); - assertHitCount(prepareSearch(".watcher-history*").setSize(0), 1); - }); + assertBusy(() -> { assertHitCount(getWatchHistory(), 1); }); // as fields with dots are allowed in 5.0 again, the mapping must be checked in addition GetMappingsResponse response = indicesAdmin().prepareGetMappings(".watcher-history*").get(); @@ -149,10 +145,7 @@ public void testPayloadInputWithDotsInFieldNameWorks() throws Exception { new ExecuteWatchRequestBuilder(client()).setId("test_watch").setRecordExecution(true).get(); - assertBusy(() -> { - refresh(".watcher-history*"); - assertHitCount(prepareSearch(".watcher-history*").setSize(0), 1); - }); + assertBusy(() -> { assertHitCount(getWatchHistory(), 1); }); // as fields with dots are allowed in 5.0 again, the mapping must be checked in addition GetMappingsResponse response = indicesAdmin().prepareGetMappings(".watcher-history*").get(); @@ -186,8 +179,7 @@ public void testThatHistoryContainsStatus() throws Exception { WatchStatus status = new GetWatchRequestBuilder(client()).setId("test_watch").get().getStatus(); assertBusy(() -> { - refresh(".watcher-history*"); - assertResponse(prepareSearch(".watcher-history*").setSize(1), searchResponse -> { + assertResponse(getWatchHistory(), searchResponse -> { assertHitCount(searchResponse, 1); SearchHit hit = searchResponse.getHits().getAt(0); @@ -233,4 +225,14 @@ public void testThatHistoryContainsStatus() throws Exception { }); } + /* + * Returns a SearchRequestBuilder containing up to the default number of watch history records (10) if the .watcher-history* is ready. + * Otherwise it throws an AssertionError. + */ + private SearchRequestBuilder getWatchHistory() { + ensureGreen(HistoryStoreField.DATA_STREAM); + flushAndRefresh(".watcher-history-*"); + return prepareSearch(".watcher-history-*"); + } + } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java index 2f5df9b51ab7e..7b2300ed6e892 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java @@ -45,10 +45,12 @@ import java.time.Instant; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.util.concurrent.atomic.AtomicBoolean; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; @@ -83,11 +85,11 @@ public void testPut() throws Exception { WatchRecord watchRecord = new WatchRecord.MessageWatchRecord(wid, event, ExecutionState.EXECUTED, null, randomAlphaOfLength(10)); IndexResponse indexResponse = mock(IndexResponse.class); - + AtomicBoolean historyItemIndexed = new AtomicBoolean(false); doAnswer(invocation -> { - BulkRequest request = (BulkRequest) invocation.getArguments()[1]; + BulkRequest request = (BulkRequest) invocation.getArguments()[0]; @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; + ActionListener listener = (ActionListener) invocation.getArguments()[1]; IndexRequest indexRequest = (IndexRequest) request.requests().get(0); if (indexRequest.id().equals(wid.value()) @@ -96,14 +98,17 @@ public void testPut() throws Exception { listener.onResponse( new BulkResponse(new BulkItemResponse[] { BulkItemResponse.success(1, OpType.CREATE, indexResponse) }, 1) ); + historyItemIndexed.set(true); } else { listener.onFailure(new ElasticsearchException("test issue")); + fail("Unexpected indexRequest"); } return null; }).when(client).bulk(any(), any()); historyStore.put(watchRecord); verify(client).bulk(any(), any()); + assertThat(historyItemIndexed.get(), equalTo(true)); } public void testStoreWithHideSecrets() throws Exception { @@ -148,12 +153,14 @@ public void testStoreWithHideSecrets() throws Exception { watchRecord.result().actionsResults().put(JiraAction.TYPE, result); ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(BulkRequest.class); + AtomicBoolean historyItemIndexed = new AtomicBoolean(false); doAnswer(invocation -> { @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[2]; + ActionListener listener = (ActionListener) invocation.getArguments()[1]; IndexResponse indexResponse = mock(IndexResponse.class); listener.onResponse(new BulkResponse(new BulkItemResponse[] { BulkItemResponse.success(1, OpType.CREATE, indexResponse) }, 1)); + historyItemIndexed.set(true); return null; }).when(client).bulk(requestCaptor.capture(), any()); @@ -162,7 +169,7 @@ public void testStoreWithHideSecrets() throws Exception { } else { historyStore.forcePut(watchRecord); } - + assertThat(historyItemIndexed.get(), equalTo(true)); assertThat(requestCaptor.getAllValues(), hasSize(1)); assertThat(requestCaptor.getValue().requests().get(0), instanceOf(IndexRequest.class)); IndexRequest capturedIndexRequest = (IndexRequest) requestCaptor.getValue().requests().get(0); diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index 80f1c706a34af..9de999a0616e5 100644 --- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -1061,11 +1061,21 @@ public void testDisableFieldNameField() throws IOException { "query": "FROM nofnf | LIMIT 1" }"""); // {"columns":[{"name":"dv","type":"keyword"},{"name":"no_dv","type":"keyword"}],"values":[["test",null]]} - assertMap( - entityAsMap(client().performRequest(esql)), - matchesMap().entry("columns", List.of(Map.of("name", "dv", "type", "keyword"), Map.of("name", "no_dv", "type", "keyword"))) - .entry("values", List.of(List.of("test", "test"))) - ); + try { + assertMap( + entityAsMap(client().performRequest(esql)), + matchesMap().entry( + "columns", + List.of(Map.of("name", "dv", "type", "keyword"), Map.of("name", "no_dv", "type", "keyword")) + ).entry("values", List.of(List.of("test", "test"))) + ); + } catch (ResponseException e) { + logger.error( + "failed to query index without field name field. Existing indices:\n{}", + EntityUtils.toString(client().performRequest(new Request("GET", "_cat/indices")).getEntity()) + ); + throw e; + } } }