diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/LogsDbDocumentParsingBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/LogsDbDocumentParsingBenchmark.java deleted file mode 100644 index 8924f84fdc908..0000000000000 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/LogsDbDocumentParsingBenchmark.java +++ /dev/null @@ -1,400 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.benchmark.index.mapper; - -import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.logging.LogConfigurator; -import org.elasticsearch.index.mapper.LuceneDocument; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.SourceToParse; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentType; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Warmup; - -import java.io.IOException; -import java.util.List; -import java.util.Random; -import java.util.concurrent.TimeUnit; - -@Fork(value = 1) -@Warmup(iterations = 5) -@Measurement(iterations = 5) -@BenchmarkMode(Mode.Throughput) -@OutputTimeUnit(TimeUnit.SECONDS) -@State(Scope.Benchmark) -public class LogsDbDocumentParsingBenchmark { - private Random random; - private MapperService mapperServiceEnabled; - private MapperService mapperServiceEnabledWithStoreArrays; - private MapperService mapperServiceDisabled; - private SourceToParse[] documents; - - static { - LogConfigurator.configureESLogging(); // doc values implementations need logging - } - - private static String SAMPLE_LOGS_MAPPING_ENABLED = """ - { - "_source": { - "mode": "synthetic" - }, - "properties": { - "kafka": { - "properties": { - "log": { - "properties": { - "component": { - "ignore_above": 1024, - "type": "keyword" - }, - "trace": { - "properties": { - "message": { - "type": "text" - }, - "class": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "thread": { - "ignore_above": 1024, - "type": "keyword" - }, - "class": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - """; - - private static String SAMPLE_LOGS_MAPPING_ENABLED_WITH_STORE_ARRAYS = """ - { - "_source": { - "mode": "synthetic" - }, - "properties": { - "kafka": { - "properties": { - "log": { - "properties": { - "component": { - "ignore_above": 1024, - "type": "keyword" - }, - "trace": { - "synthetic_source_keep": "arrays", - "properties": { - "message": { - "type": "text" - }, - "class": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "thread": { - "ignore_above": 1024, - "type": "keyword" - }, - "class": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - """; - - private static String SAMPLE_LOGS_MAPPING_DISABLED = """ - { - "_source": { - "mode": "synthetic" - }, - "enabled": false - } - """; - - @Setup - public void setUp() throws IOException { - this.random = new Random(); - this.mapperServiceEnabled = MapperServiceFactory.create(SAMPLE_LOGS_MAPPING_ENABLED); - this.mapperServiceEnabledWithStoreArrays = MapperServiceFactory.create(SAMPLE_LOGS_MAPPING_ENABLED_WITH_STORE_ARRAYS); - this.mapperServiceDisabled = MapperServiceFactory.create(SAMPLE_LOGS_MAPPING_DISABLED); - this.documents = generateRandomDocuments(10_000); - } - - @Benchmark - public List benchmarkEnabledObject() { - return mapperServiceEnabled.documentMapper().parse(randomFrom(documents)).docs(); - } - - @Benchmark - public List benchmarkEnabledObjectWithStoreArrays() { - return mapperServiceEnabledWithStoreArrays.documentMapper().parse(randomFrom(documents)).docs(); - } - - @Benchmark - public List benchmarkDisabledObject() { - return mapperServiceDisabled.documentMapper().parse(randomFrom(documents)).docs(); - } - - @SafeVarargs - @SuppressWarnings("varargs") - private T randomFrom(T... items) { - return items[random.nextInt(items.length)]; - } - - private SourceToParse[] generateRandomDocuments(int count) throws IOException { - var docs = new SourceToParse[count]; - for (int i = 0; i < count; i++) { - docs[i] = generateRandomDocument(); - } - return docs; - } - - private SourceToParse generateRandomDocument() throws IOException { - var builder = XContentBuilder.builder(XContentType.JSON.xContent()); - - builder.startObject(); - - builder.startObject("kafka"); - { - builder.startObject("log"); - { - builder.field("component", randomString(10)); - builder.startArray("trace"); - { - builder.startObject(); - { - builder.field("message", randomString(50)); - builder.field("class", randomString(10)); - } - builder.endObject(); - builder.startObject(); - { - builder.field("message", randomString(50)); - builder.field("class", randomString(10)); - } - builder.endObject(); - } - builder.endArray(); - builder.field("thread", randomString(10)); - builder.field("class", randomString(10)); - - } - builder.endObject(); - } - builder.endObject(); - - builder.startObject("host"); - { - builder.field("hostname", randomString(10)); - builder.startObject("os"); - { - builder.field("name", randomString(10)); - } - builder.endObject(); - - builder.field("domain", randomString(10)); - builder.field("ip", randomIp()); - builder.field("name", randomString(10)); - } - - builder.endObject(); - - builder.endObject(); - - return new SourceToParse(UUIDs.randomBase64UUID(), BytesReference.bytes(builder), XContentType.JSON); - } - - private String randomIp() { - return "" + random.nextInt(255) + '.' + random.nextInt(255) + '.' + random.nextInt(255) + '.' + random.nextInt(255); - } - - private String randomString(int maxLength) { - var length = random.nextInt(maxLength); - var builder = new StringBuilder(length); - for (int i = 0; i < length; i++) { - builder.append((byte) (32 + random.nextInt(94))); - } - return builder.toString(); - } -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java index 211718c151ba9..08e3c92307d72 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java @@ -31,6 +31,7 @@ import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.dsl.DependencyHandler; import org.gradle.api.artifacts.type.ArtifactTypeDefinition; +import org.gradle.api.file.FileCollection; import org.gradle.api.plugins.JavaPluginExtension; import org.gradle.api.provider.Provider; import org.gradle.api.specs.Specs; @@ -88,8 +89,8 @@ public void apply(Project project) { Map> versionTasks = versionTasks(project, "destructiveDistroUpgradeTest", buildParams.getBwcVersions()); TaskProvider destructiveDistroTest = project.getTasks().register("destructiveDistroTest"); - Configuration examplePlugin = configureExamplePlugin(project); - + Configuration examplePluginConfiguration = configureExamplePlugin(project); + FileCollection examplePluginFileCollection = examplePluginConfiguration; List> windowsTestTasks = new ArrayList<>(); Map>> linuxTestTasks = new HashMap<>(); @@ -102,9 +103,9 @@ public void apply(Project project) { t2 -> distribution.isDocker() == false || dockerSupport.get().getDockerAvailability().isAvailable() ); addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath); - addDistributionSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePlugin.getSingleFile().toString()); + addDistributionSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePluginFileCollection.getSingleFile().toString()); t.exclude("**/PackageUpgradeTests.class"); - }, distribution, examplePlugin.getDependencies()); + }, distribution, examplePluginConfiguration.getDependencies()); if (distribution.getPlatform() == Platform.WINDOWS) { windowsTestTasks.add(destructiveTask); diff --git a/build.gradle b/build.gradle index b85742be9632f..e6fc1f4eba28c 100644 --- a/build.gradle +++ b/build.gradle @@ -365,7 +365,7 @@ tasks.register("verifyBwcTestsEnabled") { tasks.register("branchConsistency") { description = 'Ensures this branch is internally consistent. For example, that versions constants match released versions.' - group 'Verification' + group = 'Verification' dependsOn ":verifyVersions", ":verifyBwcTestsEnabled" } diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index db7797ca23ec0..204cfc18950a8 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -45,7 +45,7 @@ if (useDra == false) { ivy { name = 'beats' if (useLocalArtifacts) { - url getLayout().getBuildDirectory().dir("artifacts").get().asFile + url = getLayout().getBuildDirectory().dir("artifacts").get().asFile patternLayout { artifact '/[organisation]/[module]-[revision]-[classifier].[ext]' } @@ -127,7 +127,7 @@ ext.expansions = { Architecture architecture, DockerBase base -> 'bin_dir' : base == DockerBase.IRON_BANK ? 'scripts' : 'bin', 'build_date' : buildDate, 'config_dir' : base == DockerBase.IRON_BANK ? 'scripts' : 'config', - 'git_revision' : buildParams.gitRevision, + 'git_revision' : buildParams.gitRevision.get(), 'license' : base == DockerBase.IRON_BANK ? 'Elastic License 2.0' : 'Elastic-License-2.0', 'package_manager' : base.packageManager, 'docker_base' : base.name().toLowerCase(), @@ -551,6 +551,7 @@ subprojects { Project subProject -> inputs.file("${parent.projectDir}/build/markers/${buildTaskName}.marker") executable = 'docker' outputs.file(tarFile) + outputs.doNotCacheIf("Build cache is disabled for export tasks") { true } args "save", "-o", tarFile, diff --git a/docs/changelog/117519.yaml b/docs/changelog/117519.yaml new file mode 100644 index 0000000000000..f228278983785 --- /dev/null +++ b/docs/changelog/117519.yaml @@ -0,0 +1,20 @@ +pr: 117519 +summary: Remove `data_frame_transforms` roles +area: Transform +type: breaking +issues: [] +breaking: + title: Remove `data_frame_transforms` roles + area: Transform + details: >- + `data_frame_transforms_admin` and `data_frame_transforms_user` were deprecated in + Elasticsearch 7 and are being removed in Elasticsearch 9. + `data_frame_transforms_admin` is now `transform_admin`. + `data_frame_transforms_user` is now `transform_user`. + Users must call the `_update` API to replace the permissions on the Transform before the + Transform can be started. + impact: >- + Transforms created with either the `data_frame_transforms_admin` or the + `data_frame_transforms_user` role will fail to start. The Transform will remain + in a `stopped` state, and its health will be red while displaying permission failures. + notable: false diff --git a/docs/changelog/117949.yaml b/docs/changelog/117949.yaml new file mode 100644 index 0000000000000..b67f36a224094 --- /dev/null +++ b/docs/changelog/117949.yaml @@ -0,0 +1,5 @@ +pr: 117949 +summary: Move `SlowLogFieldProvider` instantiation to node construction +area: Infra/Logging +type: bug +issues: [] diff --git a/docs/changelog/118804.yaml b/docs/changelog/118804.yaml new file mode 100644 index 0000000000000..1548367a5485f --- /dev/null +++ b/docs/changelog/118804.yaml @@ -0,0 +1,15 @@ +pr: 118804 +summary: Add new experimental `rank_vectors` mapping for late-interaction second order + ranking +area: Vector Search +type: feature +issues: [] +highlight: + title: Add new experimental `rank_vectors` mapping for late-interaction second order + ranking + body: + Late-interaction models are powerful rerankers. While their size and overall + cost doesn't lend itself for HNSW indexing, utilizing them as second order reranking + can provide excellent boosts in relevance. The new `rank_vectors` mapping allows for rescoring + over new and novel multi-vector late-interaction models like ColBERT or ColPali. + notable: true diff --git a/docs/changelog/119054.yaml b/docs/changelog/119054.yaml new file mode 100644 index 0000000000000..720f2e0ab02ed --- /dev/null +++ b/docs/changelog/119054.yaml @@ -0,0 +1,6 @@ +pr: 119054 +summary: "[Security Solution] allows `kibana_system` user to manage .reindexed-v8-*\ + \ Security Solution indices" +area: Authorization +type: enhancement +issues: [] diff --git a/docs/changelog/119233.yaml b/docs/changelog/119233.yaml new file mode 100644 index 0000000000000..ef89c011ce4f6 --- /dev/null +++ b/docs/changelog/119233.yaml @@ -0,0 +1,5 @@ +pr: 119233 +summary: Fixing `GetDatabaseConfigurationAction` response serialization +area: Ingest Node +type: bug +issues: [] diff --git a/docs/changelog/119474.yaml b/docs/changelog/119474.yaml new file mode 100644 index 0000000000000..e37561277d220 --- /dev/null +++ b/docs/changelog/119474.yaml @@ -0,0 +1,5 @@ +pr: 119474 +summary: "Add ES|QL cross-cluster query telemetry collection" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/119476.yaml b/docs/changelog/119476.yaml new file mode 100644 index 0000000000000..c275e6965d4a1 --- /dev/null +++ b/docs/changelog/119476.yaml @@ -0,0 +1,6 @@ +pr: 119476 +summary: Fix TopN row size estimate +area: ES|QL +type: bug +issues: + - 106956 diff --git a/docs/changelog/119495.yaml b/docs/changelog/119495.yaml new file mode 100644 index 0000000000000..b3e8f7e79d984 --- /dev/null +++ b/docs/changelog/119495.yaml @@ -0,0 +1,5 @@ +pr: 119495 +summary: Add mapping for `event_name` for OTel logs +area: Data streams +type: enhancement +issues: [] diff --git a/docs/changelog/119516.yaml b/docs/changelog/119516.yaml new file mode 100644 index 0000000000000..06dd5168a0823 --- /dev/null +++ b/docs/changelog/119516.yaml @@ -0,0 +1,5 @@ +pr: 119516 +summary: "Fix: do not let `_resolve/cluster` hang if remote is unresponsive" +area: Search +type: bug +issues: [] diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc index 44acba4752aaa..164e3398d7a4f 100644 --- a/docs/plugins/discovery-ec2.asciidoc +++ b/docs/plugins/discovery-ec2.asciidoc @@ -241,7 +241,7 @@ The `discovery-ec2` plugin can automatically set the `aws_availability_zone` node attribute to the availability zone of each node. This node attribute allows you to ensure that each shard has copies allocated redundantly across multiple availability zones by using the -{ref}/modules-cluster.html#shard-allocation-awareness[Allocation Awareness] +{ref}/shard-allocation-awareness.html#[Allocation Awareness] feature. In order to enable the automatic definition of the `aws_availability_zone` @@ -333,7 +333,7 @@ labelled as `Moderate` or `Low`. * It is a good idea to distribute your nodes across multiple https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html[availability -zones] and use {ref}/modules-cluster.html#shard-allocation-awareness[shard +zones] and use {ref}/shard-allocation-awareness.html[shard allocation awareness] to ensure that each shard has copies in more than one availability zone. diff --git a/docs/reference/analysis.asciidoc b/docs/reference/analysis.asciidoc index 72ab42d22b911..e8fbc3bd81b6d 100644 --- a/docs/reference/analysis.asciidoc +++ b/docs/reference/analysis.asciidoc @@ -9,8 +9,7 @@ -- _Text analysis_ is the process of converting unstructured text, like -the body of an email or a product description, into a structured format that's -optimized for search. +the body of an email or a product description, into a structured format that's <>. [discrete] [[when-to-configure-analysis]] diff --git a/docs/reference/analysis/tokenizers.asciidoc b/docs/reference/analysis/tokenizers.asciidoc index 38e4ebfcabc39..89928f07b5638 100644 --- a/docs/reference/analysis/tokenizers.asciidoc +++ b/docs/reference/analysis/tokenizers.asciidoc @@ -1,6 +1,14 @@ [[analysis-tokenizers]] == Tokenizer reference +.Difference between {es} tokenization and neural tokenization +[NOTE] +==== +{es}'s tokenization process produces linguistic tokens, optimized for search and retrieval. +This differs from neural tokenization in the context of machine learning and natural language processing. Neural tokenizers translate strings into smaller, subword tokens, which are encoded into vectors for consumptions by neural networks. +{es} does not have built-in neural tokenizers. +==== + A _tokenizer_ receives a stream of characters, breaks it up into individual _tokens_ (usually individual words), and outputs a stream of _tokens_. For instance, a <> tokenizer breaks diff --git a/docs/reference/cat/nodeattrs.asciidoc b/docs/reference/cat/nodeattrs.asciidoc index 0d354ab570e93..6c8093846030c 100644 --- a/docs/reference/cat/nodeattrs.asciidoc +++ b/docs/reference/cat/nodeattrs.asciidoc @@ -17,7 +17,7 @@ console. They are _not_ intended for use by applications. For application consumption, use the <>. ==== -Returns information about <>. +Returns information about <>. [[cat-nodeattrs-api-request]] ==== {api-request-title} diff --git a/docs/reference/cluster.asciidoc b/docs/reference/cluster.asciidoc index b4359c2bf4fbb..398ece616fe07 100644 --- a/docs/reference/cluster.asciidoc +++ b/docs/reference/cluster.asciidoc @@ -35,7 +35,7 @@ one of the following: master-eligible nodes, all data nodes, all ingest nodes, all voting-only nodes, all machine learning nodes, and all coordinating-only nodes. * a pair of patterns, using `*` wildcards, of the form `attrname:attrvalue`, - which adds to the subset all nodes with a custom node attribute whose name + which adds to the subset all nodes with a <> whose name and value match the respective patterns. Custom node attributes are configured by setting properties in the configuration file of the form `node.attr.attrname: attrvalue`. diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc index 4a7a54a5b290d..f078fd2b7f2ee 100644 --- a/docs/reference/cluster/stats.asciidoc +++ b/docs/reference/cluster/stats.asciidoc @@ -25,7 +25,6 @@ Returns cluster statistics. * If the {es} {security-features} are enabled, you must have the `monitor` or `manage` <> to use this API. - [[cluster-stats-api-desc]] ==== {api-description-title} @@ -1397,7 +1396,7 @@ as a human-readable string. `_search`::: -(object) Contains the information about the <> usage in the cluster. +(object) Contains information about <> usage. + .Properties of `_search` [%collapsible%open] @@ -1528,7 +1527,11 @@ This may include requests where partial results were returned, but not requests ======= + ====== +`_esql`::: +(object) Contains information about <> usage. +The structure of the object is the same as the `_search` object above. ===== diff --git a/docs/reference/commands/node-tool.asciidoc b/docs/reference/commands/node-tool.asciidoc index cdd2bb8f0f9d7..265006aa3df17 100644 --- a/docs/reference/commands/node-tool.asciidoc +++ b/docs/reference/commands/node-tool.asciidoc @@ -23,8 +23,8 @@ bin/elasticsearch-node repurpose|unsafe-bootstrap|detach-cluster|override-versio This tool has a number of modes: * `elasticsearch-node repurpose` can be used to delete unwanted data from a - node if it used to be a <> or a - <> but has been repurposed not to have one + node if it used to be a <> or a + <> but has been repurposed not to have one or other of these roles. * `elasticsearch-node remove-settings` can be used to remove persistent settings diff --git a/docs/reference/data-management.asciidoc b/docs/reference/data-management.asciidoc index 7ef021dc6370b..849a14e00e698 100644 --- a/docs/reference/data-management.asciidoc +++ b/docs/reference/data-management.asciidoc @@ -43,7 +43,7 @@ Data older than this period can be deleted by {es} at a later time. **Elastic Curator** is a tool that allows you to manage your indices and snapshots using user-defined filters and predefined actions. If ILM provides the functionality to manage your index lifecycle, and you have at least a Basic license, consider using ILM in place of Curator. Many stack components make use of ILM by default. {curator-ref-current}/ilm.html[Learn more]. -NOTE: <> is a deprecated Elasticsearch feature that allows you to manage the amount of data that is stored in your cluster, similar to the downsampling functionality of {ilm-init} and data stream lifecycle. This feature should not be used for new deployments. +NOTE: <> is a deprecated {es} feature that allows you to manage the amount of data that is stored in your cluster, similar to the downsampling functionality of {ilm-init} and data stream lifecycle. This feature should not be used for new deployments. [TIP] ==== diff --git a/docs/reference/data-management/migrate-index-allocation-filters.asciidoc b/docs/reference/data-management/migrate-index-allocation-filters.asciidoc index 85d42e4105a92..ee7d5640d53df 100644 --- a/docs/reference/data-management/migrate-index-allocation-filters.asciidoc +++ b/docs/reference/data-management/migrate-index-allocation-filters.asciidoc @@ -2,7 +2,7 @@ [[migrate-index-allocation-filters]] == Migrate index allocation filters to node roles -If you currently use custom node attributes and +If you currently use <> and <> to move indices through <> in a https://www.elastic.co/blog/implementing-hot-warm-cold-in-elasticsearch-with-index-lifecycle-management[hot-warm-cold architecture], diff --git a/docs/reference/data-store-architecture.asciidoc b/docs/reference/data-store-architecture.asciidoc index 4ee75c15562ea..a0d504eb117c8 100644 --- a/docs/reference/data-store-architecture.asciidoc +++ b/docs/reference/data-store-architecture.asciidoc @@ -9,10 +9,16 @@ from any node. The topics in this section provides information about the architecture of {es} and how it stores and retrieves data: * <>: Learn about the basic building blocks of an {es} cluster, including nodes, shards, primaries, and replicas. +* <>: Learn about the different roles that nodes can have in an {es} cluster. * <>: Learn how {es} replicates read and write operations across shards and shard copies. * <>: Learn how {es} allocates and balances shards across nodes. +** <>: Learn how to use custom node attributes to distribute shards across different racks or availability zones. +* <>: Learn how {es} caches search requests to improve performance. -- include::nodes-shards.asciidoc[] +include::node-roles.asciidoc[] include::docs/data-replication.asciidoc[leveloffset=-1] -include::modules/shard-ops.asciidoc[] \ No newline at end of file +include::modules/shard-ops.asciidoc[] +include::modules/cluster/allocation_awareness.asciidoc[leveloffset=+1] +include::shard-request-cache.asciidoc[leveloffset=-1] diff --git a/docs/reference/data-streams/downsampling.asciidoc b/docs/reference/data-streams/downsampling.asciidoc index 0b08b0972f9a1..10a0241cf0732 100644 --- a/docs/reference/data-streams/downsampling.asciidoc +++ b/docs/reference/data-streams/downsampling.asciidoc @@ -72,6 +72,45 @@ the granularity of `cold` archival data to monthly or less. .Downsampled metrics series image::images/data-streams/time-series-downsampled.png[align="center"] +[discrete] +[[downsample-api-process]] +==== The downsampling process + +The downsampling operation traverses the source TSDS index and performs the +following steps: + +. Creates a new document for each value of the `_tsid` field and each +`@timestamp` value, rounded to the `fixed_interval` defined in the downsample +configuration. +. For each new document, copies all <> from the source index to the target index. Dimensions in a +TSDS are constant, so this is done only once per bucket. +. For each <> field, computes aggregations +for all documents in the bucket. Depending on the metric type of each metric +field a different set of pre-aggregated results is stored: + +** `gauge`: The `min`, `max`, `sum`, and `value_count` are stored; `value_count` +is stored as type `aggregate_metric_double`. +** `counter`: The `last_value` is stored. +. For all other fields, the most recent value is copied to the target index. + +[discrete] +[[downsample-api-mappings]] +==== Source and target index field mappings + +Fields in the target, downsampled index are created based on fields in the +original source index, as follows: + +. All fields mapped with the `time-series-dimension` parameter are created in +the target downsample index with the same mapping as in the source index. +. All fields mapped with the `time_series_metric` parameter are created +in the target downsample index with the same mapping as in the source +index. An exception is that for fields mapped as `time_series_metric: gauge` +the field type is changed to `aggregate_metric_double`. +. All other fields that are neither dimensions nor metrics (that is, label +fields), are created in the target downsample index with the same mapping +that they had in the source index. + [discrete] [[running-downsampling]] === Running downsampling on time series data diff --git a/docs/reference/datatiers.asciidoc b/docs/reference/datatiers.asciidoc index 65e029d876e6f..066765368ec5e 100644 --- a/docs/reference/datatiers.asciidoc +++ b/docs/reference/datatiers.asciidoc @@ -189,7 +189,7 @@ tier]. [[configure-data-tiers-on-premise]] ==== Self-managed deployments -For self-managed deployments, each node's <> is configured +For self-managed deployments, each node's <> is configured in `elasticsearch.yml`. For example, the highest-performance nodes in a cluster might be assigned to both the hot and content tiers: diff --git a/docs/reference/high-availability/cluster-design.asciidoc b/docs/reference/high-availability/cluster-design.asciidoc index 105c8b236b0b1..d187db83c43f9 100644 --- a/docs/reference/high-availability/cluster-design.asciidoc +++ b/docs/reference/high-availability/cluster-design.asciidoc @@ -87,7 +87,7 @@ the same thing, but it's not necessary to use this feature in such a small cluster. We recommend you set only one of your two nodes to be -<>. This means you can be certain which of your +<>. This means you can be certain which of your nodes is the elected master of the cluster. The cluster can tolerate the loss of the other master-ineligible node. If you set both nodes to master-eligible, two nodes are required for a master election. Since the election will fail if either @@ -164,12 +164,12 @@ cluster that is suitable for production deployments. [[high-availability-cluster-design-three-nodes]] ==== Three-node clusters -If you have three nodes, we recommend they all be <> and +If you have three nodes, we recommend they all be <> and every index that is not a <> should have at least one replica. Nodes are data nodes by default. You may prefer for some indices to have two replicas so that each node has a copy of each shard in those indices. You should also configure each node to be -<> so that any two of them can hold a master +<> so that any two of them can hold a master election without needing to communicate with the third node. Nodes are master-eligible by default. This cluster will be resilient to the loss of any single node. @@ -188,8 +188,8 @@ service provides such a load balancer. Once your cluster grows to more than three nodes, you can start to specialise these nodes according to their responsibilities, allowing you to scale their -resources independently as needed. You can have as many <>, <>, <>, etc. as needed to +resources independently as needed. You can have as many <>, <>, <>, etc. as needed to support your workload. As your cluster grows larger, we recommend using dedicated nodes for each role. This allows you to independently scale resources for each task. diff --git a/docs/reference/ilm/apis/migrate-to-data-tiers.asciidoc b/docs/reference/ilm/apis/migrate-to-data-tiers.asciidoc index 76810170daa19..bbcdd71c45f1a 100644 --- a/docs/reference/ilm/apis/migrate-to-data-tiers.asciidoc +++ b/docs/reference/ilm/apis/migrate-to-data-tiers.asciidoc @@ -11,7 +11,7 @@ For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs]. -- -Switches the indices, ILM policies, and legacy, composable and component templates from using custom node attributes and +Switches the indices, ILM policies, and legacy, composable and component templates from using <> and <> to using <>, and optionally deletes one legacy index template. Using node roles enables {ilm-init} to <> between diff --git a/docs/reference/images/search/full-text-search-overview.svg b/docs/reference/images/search/full-text-search-overview.svg new file mode 100644 index 0000000000000..e7a1c5ba14cfa --- /dev/null +++ b/docs/reference/images/search/full-text-search-overview.svg @@ -0,0 +1,81 @@ + + + + + + + + + + + + + + + Full-text search with Elasticsearch + + + + + Source documents + + + + Analysis pipeline + Transforms text to normalized terms + + + + Inverted index + Search-optimized data structure + + + + Search query + + + + Relevance scoring + Similarity algorithm scores documents + + + + Search results + Most relevant first + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc b/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc index d08af21007622..2d59e9be31cd4 100644 --- a/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc +++ b/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc @@ -13,7 +13,7 @@ This setting corresponds to the data node roles: * <> * <> -NOTE: The <> role is not a valid data tier and cannot be used +NOTE: The <> role is not a valid data tier and cannot be used with the `_tier_preference` setting. The frozen tier stores <> exclusively. diff --git a/docs/reference/index-modules/allocation/filtering.asciidoc b/docs/reference/index-modules/allocation/filtering.asciidoc index 07a2455ca1eff..5da8e254cb4f2 100644 --- a/docs/reference/index-modules/allocation/filtering.asciidoc +++ b/docs/reference/index-modules/allocation/filtering.asciidoc @@ -6,7 +6,7 @@ a particular index. These per-index filters are applied in conjunction with <> and <>. -Shard allocation filters can be based on custom node attributes or the built-in +Shard allocation filters can be based on <> or the built-in `_name`, `_host_ip`, `_publish_ip`, `_ip`, `_host`, `_id`, `_tier` and `_tier_preference` attributes. <> uses filters based on custom node attributes to determine how to reallocate shards when moving @@ -114,7 +114,7 @@ The index allocation settings support the following built-in attributes: NOTE: `_tier` filtering is based on <> roles. Only a subset of roles are <> roles, and the generic -<> will match any tier filtering. +<> will match any tier filtering. You can use wildcards when specifying attribute values, for example: diff --git a/docs/reference/indices/downsample-data-stream.asciidoc b/docs/reference/indices/downsample-data-stream.asciidoc index a99d7b246ba43..6354f8e30d254 100644 --- a/docs/reference/indices/downsample-data-stream.asciidoc +++ b/docs/reference/indices/downsample-data-stream.asciidoc @@ -81,6 +81,8 @@ DELETE _index_template/* //// // end::downsample-example[] +Check the <> documentation for an overview, details about the downsampling process, and examples of running downsampling manually and as part of an ILM policy. + [[downsample-api-request]] ==== {api-request-title} @@ -121,44 +123,4 @@ to aggregate the original time series index. For example, `60m` produces a document for each 60 minute (hourly) interval. This follows standard time formatting syntax as used elsewhere in {es}. + -NOTE: Smaller, more granular intervals take up proportionally more space. - -[[downsample-api-process]] -==== The downsampling process - -The downsampling operation traverses the source TSDS index and performs the -following steps: - -. Creates a new document for each value of the `_tsid` field and each -`@timestamp` value, rounded to the `fixed_interval` defined in the downsample -configuration. -. For each new document, copies all <> from the source index to the target index. Dimensions in a -TSDS are constant, so this is done only once per bucket. -. For each <> field, computes aggregations -for all documents in the bucket. Depending on the metric type of each metric -field a different set of pre-aggregated results is stored: - -** `gauge`: The `min`, `max`, `sum`, and `value_count` are stored; `value_count` -is stored as type `aggregate_metric_double`. -** `counter`: The `last_value` is stored. -. For all other fields, the most recent value is copied to the target index. - -[[downsample-api-mappings]] -==== Source and target index field mappings - -Fields in the target, downsampled index are created based on fields in the -original source index, as follows: - -. All fields mapped with the `time-series-dimension` parameter are created in -the target downsample index with the same mapping as in the source index. -. All fields mapped with the `time_series_metric` parameter are created -in the target downsample index with the same mapping as in the source -index. An exception is that for fields mapped as `time_series_metric: gauge` -the field type is changed to `aggregate_metric_double`. -. All other fields that are neither dimensions nor metrics (that is, label -fields), are created in the target downsample index with the same mapping -that they had in the source index. - -Check the <> documentation for an overview and -examples of running downsampling manually and as part of an ILM policy. +NOTE: Smaller, more granular intervals take up proportionally more space. \ No newline at end of file diff --git a/docs/reference/indices/index-templates.asciidoc b/docs/reference/indices/index-templates.asciidoc index 5b152ecf177ec..90c4a6952446e 100644 --- a/docs/reference/indices/index-templates.asciidoc +++ b/docs/reference/indices/index-templates.asciidoc @@ -61,7 +61,7 @@ applying the templates, do one or more of the following: - Use a non-overlapping index pattern. -- Assign templates with an overlapping pattern a `priority` higher than `200`. +- Assign templates with an overlapping pattern a `priority` higher than `500`. For example, if you don't use {fleet} or {agent} and want to create a template for the `logs-*` index pattern, assign your template a priority of `500`. This ensures your template is applied instead of the built-in template for diff --git a/docs/reference/inference/service-elasticsearch.asciidoc b/docs/reference/inference/service-elasticsearch.asciidoc index 19e3f7a5dcffe..8870fbed357a6 100644 --- a/docs/reference/inference/service-elasticsearch.asciidoc +++ b/docs/reference/inference/service-elasticsearch.asciidoc @@ -9,8 +9,11 @@ For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[ Creates an {infer} endpoint to perform an {infer} task with the `elasticsearch` service. -NOTE: If you use the ELSER or the E5 model through the `elasticsearch` service, the API request will automatically download and deploy the model if it isn't downloaded yet. - +[NOTE] +==== +* Your {es} deployment contains <>, you only need to create the enpoints using the API if you want to customize the settings. +* If you use the ELSER or the E5 model through the `elasticsearch` service, the API request will automatically download and deploy the model if it isn't downloaded yet. +==== [discrete] [[infer-service-elasticsearch-api-request]] diff --git a/docs/reference/inference/service-elser.asciidoc b/docs/reference/inference/service-elser.asciidoc index 56e56215124af..47aaa58814602 100644 --- a/docs/reference/inference/service-elser.asciidoc +++ b/docs/reference/inference/service-elser.asciidoc @@ -10,14 +10,17 @@ For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[ Creates an {infer} endpoint to perform an {infer} task with the `elser` service. You can also deploy ELSER by using the <>. -NOTE: The API request will automatically download and deploy the ELSER model if -it isn't already downloaded. +[NOTE] +==== +* Your {es} deployment contains <>, you only need to create the enpoint using the API if you want to customize the settings. +* The API request will automatically download and deploy the ELSER model if it isn't already downloaded. +==== [WARNING] .Deprecated in 8.16 ==== -The elser service is deprecated and will be removed in a future release. -Use the <> instead, with model_id included in the service_settings. +The `elser` service is deprecated and will be removed in a future release. +Use the <> instead, with `model_id` included in the `service_settings`. ==== [discrete] diff --git a/docs/reference/intro.asciidoc b/docs/reference/intro.asciidoc index 391439df2ae85..2fd0722bcd660 100644 --- a/docs/reference/intro.asciidoc +++ b/docs/reference/intro.asciidoc @@ -260,7 +260,7 @@ Refer to <> for a hands-on examp *{esql}* is a new piped query language and compute engine which was first added in version *8.11*. -{esql} does not yet support all the features of Query DSL, like full-text search and semantic search. +{esql} does not yet support all the features of Query DSL. Look forward to new {esql} features and functionalities in each release. Refer to <> for a full overview of the query languages available in {es}. @@ -280,7 +280,7 @@ The <> accepts queries written in Query DS Query DSL support a wide range of search techniques, including the following: -* <>: Search text that has been analyzed and indexed to support phrase or proximity queries, fuzzy matches, and more. +* <>: Search text that has been analyzed and indexed to support phrase or proximity queries, fuzzy matches, and more. * <>: Search for exact matches using `keyword` fields. * <>: Search `semantic_text` fields using dense or sparse vector search on embeddings generated in your {es} cluster. * <>: Search for similar dense vectors using the kNN algorithm for embeddings generated outside of {es}. @@ -328,8 +328,7 @@ directly executed within {es} itself. The <> accepts queries written in {esql} syntax. -Today, it supports a subset of the features available in Query DSL, like aggregations, filters, and transformations. -It does not yet support full-text search or semantic search. +Today, it supports a subset of the features available in Query DSL, but it is rapidly evolving. It comes with a comprehensive set of <> for working with data and has robust integration with {kib}'s Discover, dashboards and visualizations. diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index babe4f508b5f0..e5155b7d4ce5b 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -180,6 +180,8 @@ include::types/rank-feature.asciidoc[] include::types/rank-features.asciidoc[] +include::types/rank-vectors.asciidoc[] + include::types/search-as-you-type.asciidoc[] include::types/semantic-text.asciidoc[] diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc index 199a59a5b143c..c16b979043a57 100644 --- a/docs/reference/mapping/types/dense-vector.asciidoc +++ b/docs/reference/mapping/types/dense-vector.asciidoc @@ -1,4 +1,3 @@ -[role="xpack"] [[dense-vector]] === Dense vector field type ++++ diff --git a/docs/reference/mapping/types/rank-vectors.asciidoc b/docs/reference/mapping/types/rank-vectors.asciidoc new file mode 100644 index 0000000000000..a718a5e47ec85 --- /dev/null +++ b/docs/reference/mapping/types/rank-vectors.asciidoc @@ -0,0 +1,201 @@ +[role="xpack"] +[[rank-vectors]] +=== Rank Vectors +++++ + Rank Vectors +++++ +experimental::[] + +The `rank_vectors` field type enables late-interaction dense vector scoring in Elasticsearch. The number of vectors +per field can vary, but they must all share the same number of dimensions and element type. + +The purpose of vectors stored in this field is second order ranking documents with max-sim similarity. + +Here is a simple example of using this field with `float` elements. + +[source,console] +-------------------------------------------------- +PUT my-rank-vectors-float +{ + "mappings": { + "properties": { + "my_vector": { + "type": "rank_vectors" + } + } + } +} + +PUT my-rank-vectors-float/_doc/1 +{ + "my_vector" : [[0.5, 10, 6], [-0.5, 10, 10]] +} + +-------------------------------------------------- +// TESTSETUP + +In addition to the `float` element type, `byte` and `bit` element types are also supported. + +Here is an example of using this field with `byte` elements. + +[source,console] +-------------------------------------------------- +PUT my-rank-vectors-byte +{ + "mappings": { + "properties": { + "my_vector": { + "type": "rank_vectors", + "element_type": "byte" + } + } + } +} + +PUT my-rank-vectors-byte/_doc/1 +{ + "my_vector" : [[1, 2, 3], [4, 5, 6]] +} +-------------------------------------------------- + +Here is an example of using this field with `bit` elements. + +[source,console] +-------------------------------------------------- +PUT my-rank-vectors-bit +{ + "mappings": { + "properties": { + "my_vector": { + "type": "rank_vectors", + "element_type": "bit" + } + } + } +} + +POST /my-rank-vectors-bit/_bulk?refresh +{"index": {"_id" : "1"}} +{"my_vector": [127, -127, 0, 1, 42]} +{"index": {"_id" : "2"}} +{"my_vector": "8100012a7f"} +-------------------------------------------------- + +[role="child_attributes"] +[[rank-vectors-params]] +==== Parameters for rank vectors fields + +The `rank_vectors` field type supports the following parameters: + +[[rank-vectors-element-type]] +`element_type`:: +(Optional, string) +The data type used to encode vectors. The supported data types are +`float` (default), `byte`, and bit. + +.Valid values for `element_type` +[%collapsible%open] +==== +`float`::: +indexes a 4-byte floating-point +value per dimension. This is the default value. + +`byte`::: +indexes a 1-byte integer value per dimension. + +`bit`::: +indexes a single bit per dimension. Useful for very high-dimensional vectors or models that specifically support bit vectors. +NOTE: when using `bit`, the number of dimensions must be a multiple of 8 and must represent the number of bits. + +==== + +`dims`:: +(Optional, integer) +Number of vector dimensions. Can't exceed `4096`. If `dims` is not specified, +it will be set to the length of the first vector added to the field. + +[[rank-vectors-synthetic-source]] +==== Synthetic `_source` + +IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices +(indices that have `index.mode` set to `time_series`). For other indices +synthetic `_source` is in technical preview. Features in technical preview may +be changed or removed in a future release. Elastic will work to fix +any issues, but features in technical preview are not subject to the support SLA +of official GA features. + +`rank_vectors` fields support <> . + +[[rank-vectors-scoring]] +==== Scoring with rank vectors + +Rank vectors can be accessed and used in <>. + +For example, the following query scores documents based on the maxSim similarity between the query vector and the vectors stored in the `my_vector` field: + +[source,console] +-------------------------------------------------- +GET my-rank-vectors-float/_search +{ + "query": { + "script_score": { + "query": { + "match_all": {} + }, + "script": { + "source": "maxSimDotProduct(params.query_vector, 'my_vector')", + "params": { + "query_vector": [[0.5, 10, 6], [-0.5, 10, 10]] + } + } + } + } +} +-------------------------------------------------- + +Additionally, asymmetric similarity functions can be used to score against `bit` vectors. For example, the following query scores documents based on the maxSimDotProduct similarity between a floating point query vector and bit vectors stored in the `my_vector` field: + +[source,console] +-------------------------------------------------- +PUT my-rank-vectors-bit +{ + "mappings": { + "properties": { + "my_vector": { + "type": "rank_vectors", + "element_type": "bit" + } + } + } +} + +POST /my-rank-vectors-bit/_bulk?refresh +{"index": {"_id" : "1"}} +{"my_vector": [127, -127, 0, 1, 42]} +{"index": {"_id" : "2"}} +{"my_vector": "8100012a7f"} + +GET my-rank-vectors-bit/_search +{ + "query": { + "script_score": { + "query": { + "match_all": {} + }, + "script": { + "source": "maxSimDotProduct(params.query_vector, 'my_vector')", + "params": { + "query_vector": [ + [0.35, 0.77, 0.95, 0.15, 0.11, 0.08, 0.58, 0.06, 0.44, 0.52, 0.21, + 0.62, 0.65, 0.16, 0.64, 0.39, 0.93, 0.06, 0.93, 0.31, 0.92, 0.0, + 0.66, 0.86, 0.92, 0.03, 0.81, 0.31, 0.2 , 0.92, 0.95, 0.64, 0.19, + 0.26, 0.77, 0.64, 0.78, 0.32, 0.97, 0.84] + ] <1> + } + } + } + } +} +-------------------------------------------------- +<1> Note that the query vector has 40 elements, matching the number of bits in the bit vectors. + diff --git a/docs/reference/modules/cluster.asciidoc b/docs/reference/modules/cluster.asciidoc index b3eaa5b47c238..cf8e97de5e188 100644 --- a/docs/reference/modules/cluster.asciidoc +++ b/docs/reference/modules/cluster.asciidoc @@ -27,7 +27,23 @@ include::cluster/shards_allocation.asciidoc[] include::cluster/disk_allocator.asciidoc[] -include::cluster/allocation_awareness.asciidoc[] +[[shard-allocation-awareness-settings]] +==== Shard allocation awareness settings + +You can use <> as _awareness attributes_ to enable {es} +to take your physical hardware configuration into account when allocating shards. +If {es} knows which nodes are on the same physical server, in the same rack, or +in the same zone, it can distribute the primary shard and its replica shards to +minimize the risk of losing all shard copies in the event of a failure. <>. + +`cluster.routing.allocation.awareness.attributes`:: + (<>) + The node attributes that {es} should use as awareness attributes. For example, if you have a `rack_id` attribute that specifies the rack in which each node resides, you can set this setting to `rack_id` to ensure that primary and replica shards are not allocated on the same rack. You can specify multiple attributes as a comma-separated list. + +`cluster.routing.allocation.awareness.force.*`:: + (<>) + The shard allocation awareness values that must exist for shards to be reallocated in case of location failure. Learn more about <>. + include::cluster/allocation_filtering.asciidoc[] diff --git a/docs/reference/modules/cluster/allocation_awareness.asciidoc b/docs/reference/modules/cluster/allocation_awareness.asciidoc index 9c6197f9ba40d..34164cd364fc5 100644 --- a/docs/reference/modules/cluster/allocation_awareness.asciidoc +++ b/docs/reference/modules/cluster/allocation_awareness.asciidoc @@ -1,5 +1,5 @@ [[shard-allocation-awareness]] -==== Shard allocation awareness +== Shard allocation awareness You can use custom node attributes as _awareness attributes_ to enable {es} to take your physical hardware configuration into account when allocating shards. @@ -7,12 +7,7 @@ If {es} knows which nodes are on the same physical server, in the same rack, or in the same zone, it can distribute the primary shard and its replica shards to minimize the risk of losing all shard copies in the event of a failure. -When shard allocation awareness is enabled with the -<> -`cluster.routing.allocation.awareness.attributes` setting, shards are only -allocated to nodes that have values set for the specified awareness attributes. -If you use multiple awareness attributes, {es} considers each attribute -separately when allocating shards. +When shard allocation awareness is enabled with the `cluster.routing.allocation.awareness.attributes` setting, shards are only allocated to nodes that have values set for the specified awareness attributes. If you use multiple awareness attributes, {es} considers each attribute separately when allocating shards. NOTE: The number of attribute values determines how many shard copies are allocated in each location. If the number of nodes in each location is @@ -22,11 +17,11 @@ unassigned. TIP: Learn more about <>. [[enabling-awareness]] -===== Enabling shard allocation awareness +=== Enabling shard allocation awareness To enable shard allocation awareness: -. Specify the location of each node with a custom node attribute. For example, +. Specify the location of each node with a <>. For example, if you want Elasticsearch to distribute shards across different racks, you might use an awareness attribute called `rack_id`. + @@ -94,7 +89,7 @@ copies of a particular shard from being allocated in the same location, you can enable forced awareness. [[forced-awareness]] -===== Forced awareness +=== Forced awareness By default, if one location fails, {es} spreads its shards across the remaining locations. This might be undesirable if the cluster does not have sufficient diff --git a/docs/reference/modules/cluster/allocation_filtering.asciidoc b/docs/reference/modules/cluster/allocation_filtering.asciidoc index e70e43682973b..d0d2652059048 100644 --- a/docs/reference/modules/cluster/allocation_filtering.asciidoc +++ b/docs/reference/modules/cluster/allocation_filtering.asciidoc @@ -6,7 +6,7 @@ allocates shards from any index. These cluster wide filters are applied in conjunction with <> and <>. -Shard allocation filters can be based on custom node attributes or the built-in +Shard allocation filters can be based on <> or the built-in `_name`, `_host_ip`, `_publish_ip`, `_ip`, `_host`, `_id` and `_tier` attributes. The `cluster.routing.allocation` settings are <>, enabling live indices to @@ -59,9 +59,9 @@ The cluster allocation settings support the following built-in attributes: NOTE: `_tier` filtering is based on <> roles. Only a subset of roles are <> roles, and the generic -<> will match any tier filtering. +<> will match any tier filtering. a subset of roles that are <> roles, but the generic -<> will match any tier filtering. +<> will match any tier filtering. You can use wildcards when specifying attribute values, for example: diff --git a/docs/reference/modules/cluster/disk_allocator.asciidoc b/docs/reference/modules/cluster/disk_allocator.asciidoc index 02cc48c6e27fc..8efe4c0132e86 100644 --- a/docs/reference/modules/cluster/disk_allocator.asciidoc +++ b/docs/reference/modules/cluster/disk_allocator.asciidoc @@ -41,6 +41,23 @@ on the affected node drops below the high watermark, {es} automatically removes the write block. Refer to <> to resolve persistent watermark errors. +[NOTE] +.Max headroom settings +=================================================== + +Max headroom settings apply only when watermark settings are percentages or ratios. + +A max headroom value is intended to cap the required free disk space before hitting +the respective watermark. This is useful for servers with larger disks, where a percentage or ratio watermark could translate to an overly large free disk space requirement. In this case, the max headroom can be used to cap the required free disk space amount. + +For example, where `cluster.routing.allocation.disk.watermark.flood_stage` is 95% and `cluster.routing.allocation.disk.watermark.flood_stage.max_headroom` is 100GB, this means that: + +* For a smaller disk, e.g., of 100GB, the flood watermark will hit at 95%, meaning at 5GB of free space, since 5GB is smaller than the 100GB max headroom value. +* For a larger disk, e.g., of 100TB, the flood watermark will hit at 100GB of free space. That is because the 95% flood watermark alone would require 5TB of free disk space, but is capped by the max headroom setting to 100GB. + +Max headroom settings have their default values only if their respective watermark settings are not explicitly set. If watermarks are explicitly set, then the max headroom settings do not have their default values, and need to be explicitly set if they are needed. +=================================================== + [[disk-based-shard-allocation-does-not-balance]] [TIP] ==== @@ -100,18 +117,7 @@ is now `true`. The setting will be removed in a future release. + -- (<>) -Controls the flood stage watermark, which defaults to 95%. {es} enforces a read-only index block (`index.blocks.read_only_allow_delete`) on every index that has one or more shards allocated on the node, and that has at least one disk exceeding the flood stage. This setting is a last resort to prevent nodes from running out of disk space. The index block is automatically released when the disk utilization falls below the high watermark. Similarly to the low and high watermark values, it can alternatively be set to a ratio value, e.g., `0.95`, or an absolute byte value. - -An example of resetting the read-only index block on the `my-index-000001` index: - -[source,console] --------------------------------------------------- -PUT /my-index-000001/_settings -{ - "index.blocks.read_only_allow_delete": null -} --------------------------------------------------- -// TEST[setup:my_index] +Controls the flood stage watermark, which defaults to 95%. {es} enforces a read-only index block (<>) on every index that has one or more shards allocated on the node, and that has at least one disk exceeding the flood stage. This setting is a last resort to prevent nodes from running out of disk space. The index block is automatically released when the disk utilization falls below the high watermark. Similarly to the low and high watermark values, it can alternatively be set to a ratio value, e.g., `0.95`, or an absolute byte value. -- // end::cluster-routing-flood-stage-tag[] @@ -121,10 +127,10 @@ Defaults to 100GB when `cluster.routing.allocation.disk.watermark.flood_stage` is not explicitly set. This caps the amount of free space required. -NOTE: You cannot mix the usage of percentage/ratio values and byte values across +NOTE: You can't mix the usage of percentage/ratio values and byte values across the `cluster.routing.allocation.disk.watermark.low`, `cluster.routing.allocation.disk.watermark.high`, and `cluster.routing.allocation.disk.watermark.flood_stage` settings. Either all values -are set to percentage/ratio values, or all are set to byte values. This enforcement is +must be set to percentage/ratio values, or all must be set to byte values. This is required so that {es} can validate that the settings are internally consistent, ensuring that the low disk threshold is less than the high disk threshold, and the high disk threshold is less than the flood stage threshold. A similar comparison check is done for the max @@ -150,44 +156,6 @@ set. This caps the amount of free space required on dedicated frozen nodes. cluster. Defaults to `30s`. NOTE: Percentage values refer to used disk space, while byte values refer to -free disk space. This can be confusing, since it flips the meaning of high and +free disk space. This can be confusing, because it flips the meaning of high and low. For example, it makes sense to set the low watermark to 10gb and the high -watermark to 5gb, but not the other way around. - -An example of updating the low watermark to at least 100 gigabytes free, a high -watermark of at least 50 gigabytes free, and a flood stage watermark of 10 -gigabytes free, and updating the information about the cluster every minute: - -[source,console] --------------------------------------------------- -PUT _cluster/settings -{ - "persistent": { - "cluster.routing.allocation.disk.watermark.low": "100gb", - "cluster.routing.allocation.disk.watermark.high": "50gb", - "cluster.routing.allocation.disk.watermark.flood_stage": "10gb", - "cluster.info.update.interval": "1m" - } -} --------------------------------------------------- - -Concerning the max headroom settings for the watermarks, please note -that these apply only in the case that the watermark settings are percentages/ratios. -The aim of a max headroom value is to cap the required free disk space before hitting -the respective watermark. This is especially useful for servers with larger -disks, where a percentage/ratio watermark could translate to a big free disk space requirement, -and the max headroom can be used to cap the required free disk space amount. -As an example, let us take the default settings for the flood watermark. -It has a 95% default value, and the flood max headroom setting has a default value of 100GB. -This means that: - -* For a smaller disk, e.g., of 100GB, the flood watermark will hit at 95%, meaning at 5GB -of free space, since 5GB is smaller than the 100GB max headroom value. -* For a larger disk, e.g., of 100TB, the flood watermark will hit at 100GB of free space. -That is because the 95% flood watermark alone would require 5TB of free disk space, but -that is capped by the max headroom setting to 100GB. - -Finally, the max headroom settings have their default values only if their respective watermark -settings are not explicitly set (thus, they have their default percentage values). -If watermarks are explicitly set, then the max headroom settings do not have their default values, -and would need to be explicitly set if they are desired. +watermark to 5gb, but not the other way around. \ No newline at end of file diff --git a/docs/reference/modules/cluster/misc.asciidoc b/docs/reference/modules/cluster/misc.asciidoc index 75eaca88c66b1..b66ac1fdb0cca 100644 --- a/docs/reference/modules/cluster/misc.asciidoc +++ b/docs/reference/modules/cluster/misc.asciidoc @@ -1,6 +1,9 @@ [[misc-cluster-settings]] === Miscellaneous cluster settings +[[cluster-name]] +include::{es-ref-dir}/setup/important-settings/cluster-name.asciidoc[] + [discrete] [[cluster-read-only]] ==== Metadata diff --git a/docs/reference/modules/discovery/bootstrapping.asciidoc b/docs/reference/modules/discovery/bootstrapping.asciidoc index 5120c1d17e69b..a885f1633ea49 100644 --- a/docs/reference/modules/discovery/bootstrapping.asciidoc +++ b/docs/reference/modules/discovery/bootstrapping.asciidoc @@ -2,7 +2,7 @@ === Bootstrapping a cluster Starting an Elasticsearch cluster for the very first time requires the initial -set of <> to be explicitly defined on one or +set of <> to be explicitly defined on one or more of the master-eligible nodes in the cluster. This is known as _cluster bootstrapping_. This is only required the first time a cluster starts up. Freshly-started nodes that are joining a running cluster obtain this diff --git a/docs/reference/modules/discovery/publishing.asciidoc b/docs/reference/modules/discovery/publishing.asciidoc index af664585085c2..22fed9528e615 100644 --- a/docs/reference/modules/discovery/publishing.asciidoc +++ b/docs/reference/modules/discovery/publishing.asciidoc @@ -1,5 +1,23 @@ +[[cluster-state-overview]] +=== Cluster state + +The _cluster state_ is an internal data structure which keeps track of a +variety of information needed by every node, including: + +* The identity and attributes of the other nodes in the cluster + +* Cluster-wide settings + +* Index metadata, including the mapping and settings for each index + +* The location and status of every shard copy in the cluster + +The elected master node ensures that every node in the cluster has a copy of +the same cluster state. The <> lets you retrieve a +representation of this internal state for debugging or diagnostic purposes. + [[cluster-state-publishing]] -=== Publishing the cluster state +==== Publishing the cluster state The elected master node is the only node in a cluster that can make changes to the cluster state. The elected master node processes one batch of cluster state @@ -58,3 +76,16 @@ speed of the storage on each master-eligible node, as well as the reliability and latency of the network interconnections between all nodes in the cluster. You must therefore ensure that the storage and networking available to the nodes in your cluster are good enough to meet your performance goals. + +[[dangling-indices]] +==== Dangling indices + +When a node joins the cluster, if it finds any shards stored in its local +data directory that do not already exist in the cluster state, it will consider +those shards to belong to a "dangling" index. You can list, import or +delete dangling indices using the <>. + +NOTE: The API cannot offer any guarantees as to whether the imported data +truly represents the latest state of the data when the index was still part +of the cluster. \ No newline at end of file diff --git a/docs/reference/modules/discovery/voting.asciidoc b/docs/reference/modules/discovery/voting.asciidoc index 04cae9d02ab66..f4bd4756d8978 100644 --- a/docs/reference/modules/discovery/voting.asciidoc +++ b/docs/reference/modules/discovery/voting.asciidoc @@ -2,7 +2,7 @@ === Voting configurations Each {es} cluster has a _voting configuration_, which is the set of -<> whose responses are counted when making +<> whose responses are counted when making decisions such as electing a new master or committing a new cluster state. Decisions are made only after a majority (more than half) of the nodes in the voting configuration respond. diff --git a/docs/reference/modules/gateway.asciidoc b/docs/reference/modules/gateway.asciidoc index bf7e6de64f093..4716effb43083 100644 --- a/docs/reference/modules/gateway.asciidoc +++ b/docs/reference/modules/gateway.asciidoc @@ -4,7 +4,7 @@ The local gateway stores the cluster state and shard data across full cluster restarts. -The following _static_ settings, which must be set on every <>, +The following _static_ settings, which must be set on every <>, control how long a freshly elected master should wait before it tries to recover the <> and the cluster's data. @@ -36,17 +36,4 @@ These settings can be configured in `elasticsearch.yml` as follows: gateway.expected_data_nodes: 3 gateway.recover_after_time: 600s gateway.recover_after_data_nodes: 3 --------------------------------------------------- - -[[dangling-indices]] -==== Dangling indices - -When a node joins the cluster, if it finds any shards stored in its local -data directory that do not already exist in the cluster, it will consider -those shards to belong to a "dangling" index. You can list, import or -delete dangling indices using the <>. - -NOTE: The API cannot offer any guarantees as to whether the imported data -truly represents the latest state of the data when the index was still part -of the cluster. +-------------------------------------------------- \ No newline at end of file diff --git a/docs/reference/modules/indices/fielddata.asciidoc b/docs/reference/modules/indices/fielddata.asciidoc index 1383bf74d6d4c..688685c0a2247 100644 --- a/docs/reference/modules/indices/fielddata.asciidoc +++ b/docs/reference/modules/indices/fielddata.asciidoc @@ -5,10 +5,6 @@ The field data cache contains <> and <>. This behavior can be configured. @@ -20,16 +16,12 @@ at the cost of rebuilding the cache as needed. If the circuit breaker limit is reached, further requests that increase the cache size will be prevented. In this case you should manually <>. +TIP: You can monitor memory usage for field data as well as the field data circuit +breaker using +the <> or the <>. + `indices.fielddata.cache.size`:: (<>) The max size of the field data cache, eg `38%` of node heap space, or an absolute value, eg `12GB`. Defaults to unbounded. If you choose to set it, -it should be smaller than <> limit. - -[discrete] -[[fielddata-monitoring]] -==== Monitoring field data - -You can monitor memory usage for field data as well as the field data circuit -breaker using -the <> or the <>. +it should be smaller than <> limit. \ No newline at end of file diff --git a/docs/reference/modules/indices/request_cache.asciidoc b/docs/reference/modules/indices/request_cache.asciidoc index 4d4d349c685a1..f6ad65245836f 100644 --- a/docs/reference/modules/indices/request_cache.asciidoc +++ b/docs/reference/modules/indices/request_cache.asciidoc @@ -1,4 +1,4 @@ -[[shard-request-cache]] +[[shard-request-cache-settings]] === Shard request cache settings When a search request is run against an index or against many indices, each @@ -10,139 +10,16 @@ The shard-level request cache module caches the local results on each shard. This allows frequently used (and potentially heavy) search requests to return results almost instantly. The requests cache is a very good fit for the logging use case, where only the most recent index is being actively updated -- -results from older indices will be served directly from the cache. +results from older indices will be served directly from the cache. You can use shard request cache settings to control the size and expiration of the cache. -[IMPORTANT] -=================================== - -By default, the requests cache will only cache the results of search requests -where `size=0`, so it will not cache `hits`, -but it will cache `hits.total`, <>, and -<>. - -Most queries that use `now` (see <>) cannot be cached. - -Scripted queries that use the API calls which are non-deterministic, such as -`Math.random()` or `new Date()` are not cached. -=================================== - -[discrete] -==== Cache invalidation - -The cache is smart -- it keeps the same _near real-time_ promise as uncached -search. - -Cached results are invalidated automatically whenever the shard refreshes to -pick up changes to the documents or when you update the mapping. In other -words you will always get the same results from the cache as you would for an -uncached search request. - -The longer the refresh interval, the longer that cached entries will remain -valid even if there are changes to the documents. If the cache is full, the -least recently used cache keys will be evicted. - -The cache can be expired manually with the <>: - -[source,console] ------------------------- -POST /my-index-000001,my-index-000002/_cache/clear?request=true ------------------------- -// TEST[s/^/PUT my-index-000001\nPUT my-index-000002\n/] - -[discrete] -==== Enabling and disabling caching - -The cache is enabled by default, but can be disabled when creating a new -index as follows: - -[source,console] ------------------------------ -PUT /my-index-000001 -{ - "settings": { - "index.requests.cache.enable": false - } -} ------------------------------ - -It can also be enabled or disabled dynamically on an existing index with the -<> API: - -[source,console] ------------------------------ -PUT /my-index-000001/_settings -{ "index.requests.cache.enable": true } ------------------------------ -// TEST[continued] - - -[discrete] -==== Enabling and disabling caching per request - -The `request_cache` query-string parameter can be used to enable or disable -caching on a *per-request* basis. If set, it overrides the index-level setting: - -[source,console] ------------------------------ -GET /my-index-000001/_search?request_cache=true -{ - "size": 0, - "aggs": { - "popular_colors": { - "terms": { - "field": "colors" - } - } - } -} ------------------------------ -// TEST[continued] - -Requests where `size` is greater than 0 will not be cached even if the request cache is -enabled in the index settings. To cache these requests you will need to use the -query-string parameter detailed here. - -[discrete] -==== Cache key - -A hash of the whole JSON body is used as the cache key. This means that if the JSON -changes -- for instance if keys are output in a different order -- then the -cache key will not be recognised. - -TIP: Most JSON libraries support a _canonical_ mode which ensures that JSON -keys are always emitted in the same order. This canonical mode can be used in -the application to ensure that a request is always serialized in the same way. +To learn more about the shard request cache, see <>. [discrete] ==== Cache settings -The cache is managed at the node level, and has a default maximum size of `1%` -of the heap. This can be changed in the `config/elasticsearch.yml` file with: - -[source,yaml] --------------------------------- -indices.requests.cache.size: 2% --------------------------------- - -Also, you can use the +indices.requests.cache.expire+ setting to specify a TTL -for cached results, but there should be no reason to do so. Remember that -stale results are automatically invalidated when the index is refreshed. This -setting is provided for completeness' sake only. - -[discrete] -==== Monitoring cache usage - -The size of the cache (in bytes) and the number of evictions can be viewed -by index, with the <> API: - -[source,console] ------------------------- -GET /_stats/request_cache?human ------------------------- +`indices.requests.cache.size`:: +(<>) The maximum size of the cache, as a percentage of the heap. Default: `1%`. -or by node with the <> API: +`indices.requests.cache.expire`:: +(<>) The TTL for cached results. Stale results are automatically invalidated when the index is refreshed, so you shouldn't need to use this setting. -[source,console] ------------------------- -GET /_nodes/stats/indices/request_cache?human ------------------------- diff --git a/docs/reference/modules/network.asciidoc b/docs/reference/modules/network.asciidoc index 1e4c5a21d386c..2ea4dcb9b18f5 100644 --- a/docs/reference/modules/network.asciidoc +++ b/docs/reference/modules/network.asciidoc @@ -286,3 +286,22 @@ include::remote-cluster-network.asciidoc[] include::network/tracers.asciidoc[] include::network/threading.asciidoc[] + +[[readiness-tcp-port]] +==== TCP readiness port + +preview::[] + +If configured, a node can open a TCP port when the node is in a ready state. A node is deemed +ready when it has successfully joined a cluster. In a single node configuration, the node is +said to be ready, when it's able to accept requests. + +To enable the readiness TCP port, use the `readiness.port` setting. The readiness service will bind to +all host addresses. + +If the node leaves the cluster, or the <> is used to mark the node +for shutdown, the readiness port is immediately closed. + +A successful connection to the readiness TCP port signals that the {es} node is ready. When a client +connects to the readiness port, the server simply terminates the socket connection. No data is sent back +to the client. If a client cannot connect to the readiness port, the node is not ready. \ No newline at end of file diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc index 022e8b5d1e2fe..e8dd995623a1d 100644 --- a/docs/reference/modules/node.asciidoc +++ b/docs/reference/modules/node.asciidoc @@ -1,5 +1,5 @@ [[modules-node]] -=== Nodes +=== Node settings Any time that you start an instance of {es}, you are starting a _node_. A collection of connected nodes is called a <>. If you @@ -18,24 +18,33 @@ TIP: The performance of an {es} node is often limited by the performance of the Review our recommendations for optimizing your storage for <> and <>. +[[node-name-settings]] +==== Node name setting + +include::{es-ref-dir}/setup/important-settings/node-name.asciidoc[] + [[node-roles]] -==== Node roles +==== Node role settings You define a node's roles by setting `node.roles` in `elasticsearch.yml`. If you set `node.roles`, the node is only assigned the roles you specify. If you don't set `node.roles`, the node is assigned the following roles: -* `master` -* `data` +* [[master-node]]`master` +* [[data-node]]`data` * `data_content` * `data_hot` * `data_warm` * `data_cold` * `data_frozen` * `ingest` -* `ml` +* [[ml-node]]`ml` * `remote_cluster_client` -* `transform` +* [[transform-node]]`transform` + +The following additional roles are available: + +* `voting_only` [IMPORTANT] ==== @@ -65,386 +74,7 @@ As the cluster grows and in particular if you have large {ml} jobs or {ctransforms}, consider separating dedicated master-eligible nodes from dedicated data nodes, {ml} nodes, and {transform} nodes. -<>:: - -A node that has the `master` role, which makes it eligible to be -<>, which controls the cluster. - -<>:: - -A node that has one of several data roles. Data nodes hold data and perform data -related operations such as CRUD, search, and aggregations. A node with a generic `data` role can fill any of the specialized data node roles. - -<>:: - -A node that has the `ingest` role. Ingest nodes are able to apply an -<> to a document in order to transform and enrich the -document before indexing. With a heavy ingest load, it makes sense to use -dedicated ingest nodes and to not include the `ingest` role from nodes that have -the `master` or `data` roles. - -<>:: - -A node that has the `remote_cluster_client` role, which makes it eligible to act -as a remote client. - -<>:: - -A node that has the `ml` role. If you want to use {ml-features}, there must be -at least one {ml} node in your cluster. For more information, see -<> and {ml-docs}/index.html[Machine learning in the {stack}]. - -<>:: - -A node that has the `transform` role. If you want to use {transforms}, there -must be at least one {transform} node in your cluster. For more information, see -<> and <>. - -[NOTE] -[[coordinating-node]] -.Coordinating node -=============================================== - -Requests like search requests or bulk-indexing requests may involve data held -on different data nodes. A search request, for example, is executed in two -phases which are coordinated by the node which receives the client request -- -the _coordinating node_. - -In the _scatter_ phase, the coordinating node forwards the request to the data -nodes which hold the data. Each data node executes the request locally and -returns its results to the coordinating node. In the _gather_ phase, the -coordinating node reduces each data node's results into a single global -result set. - -Every node is implicitly a coordinating node. This means that a node that has -an explicit empty list of roles via `node.roles` will only act as a coordinating -node, which cannot be disabled. As a result, such a node needs to have enough -memory and CPU in order to deal with the gather phase. - -=============================================== - -[[master-node]] -==== Master-eligible node - -The master node is responsible for lightweight cluster-wide actions such as -creating or deleting an index, tracking which nodes are part of the cluster, -and deciding which shards to allocate to which nodes. It is important for -cluster health to have a stable master node. - -Any master-eligible node that is not a <> may -be elected to become the master node by the <>. - -IMPORTANT: Master nodes must have a `path.data` directory whose contents -persist across restarts, just like data nodes, because this is where the -cluster metadata is stored. The cluster metadata describes how to read the data -stored on the data nodes, so if it is lost then the data stored on the data -nodes cannot be read. - -[[dedicated-master-node]] -===== Dedicated master-eligible node - -It is important for the health of the cluster that the elected master node has -the resources it needs to fulfill its responsibilities. If the elected master -node is overloaded with other tasks then the cluster will not operate well. The -most reliable way to avoid overloading the master with other tasks is to -configure all the master-eligible nodes to be _dedicated master-eligible nodes_ -which only have the `master` role, allowing them to focus on managing the -cluster. Master-eligible nodes will still also behave as -<> that route requests from clients to -the other nodes in the cluster, but you should _not_ use dedicated master nodes -for this purpose. - -A small or lightly-loaded cluster may operate well if its master-eligible nodes -have other roles and responsibilities, but once your cluster comprises more -than a handful of nodes it usually makes sense to use dedicated master-eligible -nodes. - -To create a dedicated master-eligible node, set: - -[source,yaml] -------------------- -node.roles: [ master ] -------------------- - -[[voting-only-node]] -===== Voting-only master-eligible node - -A voting-only master-eligible node is a node that participates in -<> but which will not act as the cluster's -elected master node. In particular, a voting-only node can serve as a tiebreaker -in elections. - -It may seem confusing to use the term "master-eligible" to describe a -voting-only node since such a node is not actually eligible to become the master -at all. This terminology is an unfortunate consequence of history: -master-eligible nodes are those nodes that participate in elections and perform -certain tasks during cluster state publications, and voting-only nodes have the -same responsibilities even if they can never become the elected master. - -To configure a master-eligible node as a voting-only node, include `master` and -`voting_only` in the list of roles. For example to create a voting-only data -node: - -[source,yaml] -------------------- -node.roles: [ data, master, voting_only ] -------------------- - -IMPORTANT: Only nodes with the `master` role can be marked as having the -`voting_only` role. - -High availability (HA) clusters require at least three master-eligible nodes, at -least two of which are not voting-only nodes. Such a cluster will be able to -elect a master node even if one of the nodes fails. - -Voting-only master-eligible nodes may also fill other roles in your cluster. -For instance, a node may be both a data node and a voting-only master-eligible -node. A _dedicated_ voting-only master-eligible nodes is a voting-only -master-eligible node that fills no other roles in the cluster. To create a -dedicated voting-only master-eligible node, set: - -[source,yaml] -------------------- -node.roles: [ master, voting_only ] -------------------- - -Since dedicated voting-only nodes never act as the cluster's elected master, -they may require less heap and a less powerful CPU than the true master nodes. -However all master-eligible nodes, including voting-only nodes, are on the -critical path for <>. Cluster state updates are usually independent of -performance-critical workloads such as indexing or searches, but they are -involved in management activities such as index creation and rollover, mapping -updates, and recovery after a failure. The performance characteristics of these -activities are a function of the speed of the storage on each master-eligible -node, as well as the reliability and latency of the network interconnections -between the elected master node and the other nodes in the cluster. You must -therefore ensure that the storage and networking available to the nodes in your -cluster are good enough to meet your performance goals. - -[[data-node]] -==== Data nodes - -Data nodes hold the shards that contain the documents you have indexed. Data -nodes handle data related operations like CRUD, search, and aggregations. -These operations are I/O-, memory-, and CPU-intensive. It is important to -monitor these resources and to add more data nodes if they are overloaded. - -The main benefit of having dedicated data nodes is the separation of the master -and data roles. - -In a multi-tier deployment architecture, you use specialized data roles to -assign data nodes to specific tiers: `data_content`,`data_hot`, `data_warm`, -`data_cold`, or `data_frozen`. A node can belong to multiple tiers. - -If you want to include a node in all tiers, or if your cluster does not use multiple tiers, then you can use the generic `data` role. - -include::../how-to/shard-limits.asciidoc[] - -WARNING: If you assign a node to a specific tier using a specialized data role, then you shouldn't also assign it the generic `data` role. The generic `data` role takes precedence over specialized data roles. - -[[generic-data-node]] -===== Generic data node - -Generic data nodes are included in all content tiers. - -To create a dedicated generic data node, set: -[source,yaml] ----- -node.roles: [ data ] ----- - -[[data-content-node]] -===== Content data node - -Content data nodes are part of the content tier. -include::{es-ref-dir}/datatiers.asciidoc[tag=content-tier] - -To create a dedicated content node, set: -[source,yaml] ----- -node.roles: [ data_content ] ----- - -[[data-hot-node]] -===== Hot data node - -Hot data nodes are part of the hot tier. -include::{es-ref-dir}/datatiers.asciidoc[tag=hot-tier] - -To create a dedicated hot node, set: -[source,yaml] ----- -node.roles: [ data_hot ] ----- - -[[data-warm-node]] -===== Warm data node - -Warm data nodes are part of the warm tier. -include::{es-ref-dir}/datatiers.asciidoc[tag=warm-tier] - -To create a dedicated warm node, set: -[source,yaml] ----- -node.roles: [ data_warm ] ----- - -[[data-cold-node]] -===== Cold data node - -Cold data nodes are part of the cold tier. -include::{es-ref-dir}/datatiers.asciidoc[tag=cold-tier] - -To create a dedicated cold node, set: -[source,yaml] ----- -node.roles: [ data_cold ] ----- - -[[data-frozen-node]] -===== Frozen data node - -Frozen data nodes are part of the frozen tier. -include::{es-ref-dir}/datatiers.asciidoc[tag=frozen-tier] - -To create a dedicated frozen node, set: -[source,yaml] ----- -node.roles: [ data_frozen ] ----- - -[[node-ingest-node]] -==== Ingest node - -Ingest nodes can execute pre-processing pipelines, composed of one or more -ingest processors. Depending on the type of operations performed by the ingest -processors and the required resources, it may make sense to have dedicated -ingest nodes, that will only perform this specific task. - -To create a dedicated ingest node, set: - -[source,yaml] ----- -node.roles: [ ingest ] ----- - -[[coordinating-only-node]] -==== Coordinating only node - -If you take away the ability to be able to handle master duties, to hold data, -and pre-process documents, then you are left with a _coordinating_ node that -can only route requests, handle the search reduce phase, and distribute bulk -indexing. Essentially, coordinating only nodes behave as smart load balancers. - -Coordinating only nodes can benefit large clusters by offloading the -coordinating node role from data and master-eligible nodes. They join the -cluster and receive the full <>, like every other -node, and they use the cluster state to route requests directly to the -appropriate place(s). - -WARNING: Adding too many coordinating only nodes to a cluster can increase the -burden on the entire cluster because the elected master node must await -acknowledgement of cluster state updates from every node! The benefit of -coordinating only nodes should not be overstated -- data nodes can happily -serve the same purpose. - -To create a dedicated coordinating node, set: - -[source,yaml] ----- -node.roles: [ ] ----- - -[[remote-node]] -==== Remote-eligible node - -A remote-eligible node acts as a cross-cluster client and connects to -<>. Once connected, you can search -remote clusters using <>. You can also sync -data between clusters using <>. - -[source,yaml] ----- -node.roles: [ remote_cluster_client ] ----- - -[[ml-node]] -==== [xpack]#Machine learning node# - -{ml-cap} nodes run jobs and handle {ml} API requests. For more information, see -<>. - -To create a dedicated {ml} node, set: - -[source,yaml] ----- -node.roles: [ ml, remote_cluster_client] ----- - -The `remote_cluster_client` role is optional but strongly recommended. -Otherwise, {ccs} fails when used in {ml} jobs or {dfeeds}. If you use {ccs} in -your {anomaly-jobs}, the `remote_cluster_client` role is also required on all -master-eligible nodes. Otherwise, the {dfeed} cannot start. See <>. - -[[transform-node]] -==== [xpack]#{transform-cap} node# - -{transform-cap} nodes run {transforms} and handle {transform} API requests. For -more information, see <>. - -To create a dedicated {transform} node, set: - -[source,yaml] ----- -node.roles: [ transform, remote_cluster_client ] ----- - -The `remote_cluster_client` role is optional but strongly recommended. -Otherwise, {ccs} fails when used in {transforms}. See <>. - -[[change-node-role]] -==== Changing the role of a node - -Each data node maintains the following data on disk: - -* the shard data for every shard allocated to that node, -* the index metadata corresponding with every shard allocated to that node, and -* the cluster-wide metadata, such as settings and index templates. - -Similarly, each master-eligible node maintains the following data on disk: - -* the index metadata for every index in the cluster, and -* the cluster-wide metadata, such as settings and index templates. - -Each node checks the contents of its data path at startup. If it discovers -unexpected data then it will refuse to start. This is to avoid importing -unwanted <> which can lead -to a red cluster health. To be more precise, nodes without the `data` role will -refuse to start if they find any shard data on disk at startup, and nodes -without both the `master` and `data` roles will refuse to start if they have any -index metadata on disk at startup. - -It is possible to change the roles of a node by adjusting its -`elasticsearch.yml` file and restarting it. This is known as _repurposing_ a -node. In order to satisfy the checks for unexpected data described above, you -must perform some extra steps to prepare a node for repurposing when starting -the node without the `data` or `master` roles. - -* If you want to repurpose a data node by removing the `data` role then you - should first use an <> to safely - migrate all the shard data onto other nodes in the cluster. - -* If you want to repurpose a node to have neither the `data` nor `master` roles - then it is simplest to start a brand-new node with an empty data path and the - desired roles. You may find it safest to use an - <> to migrate the shard data elsewhere - in the cluster first. - -If it is not possible to follow these extra steps then you may be able to use -the <> tool to delete any -excess data that prevents a node from starting. +To learn more about the available node roles, see <>. [discrete] === Node data path settings @@ -495,6 +125,25 @@ modify the contents of the data directory. The data directory contains no executables so a virus scan will only find false positives. // end::modules-node-data-path-warning-tag[] +[[custom-node-attributes]] +==== Custom node attributes + +If needed, you can add custom attributes to a node. These attributes can be used to <>, or to group nodes together for <>. + +[TIP] +=============================================== +You can also set a node attribute using the `-E` command line argument when you start a node: + +[source,sh] +-------------------------------------------------------- +./bin/elasticsearch -Enode.attr.rack_id=rack_one +-------------------------------------------------------- +=============================================== + +`node.attr.`:: + (<>) + A custom attribute that you can assign to a node. For example, you might assign a `rack_id` attribute to each node to ensure that primary and replica shards are not allocated on the same rack. You can specify multiple attributes as a comma-separated list. + [discrete] [[other-node-settings]] === Other node settings @@ -504,4 +153,4 @@ including: * <> * <> -* <> +* <> \ No newline at end of file diff --git a/docs/reference/modules/remote-clusters.asciidoc b/docs/reference/modules/remote-clusters.asciidoc index ca1c507aa4ed9..87078c0f1956f 100644 --- a/docs/reference/modules/remote-clusters.asciidoc +++ b/docs/reference/modules/remote-clusters.asciidoc @@ -80,7 +80,7 @@ The _gateway nodes_ selection depends on the following criteria: + * *version*: Remote nodes must be compatible with the cluster they are registered to. -* *role*: By default, any non-<> node can act as a +* *role*: By default, any non-<> node can act as a gateway node. Dedicated master nodes are never selected as gateway nodes. * *attributes*: You can define the gateway nodes for a cluster by setting <> to `true`. diff --git a/docs/reference/modules/shard-ops.asciidoc b/docs/reference/modules/shard-ops.asciidoc index 66ceebcfa0319..93d6b6d3468f8 100644 --- a/docs/reference/modules/shard-ops.asciidoc +++ b/docs/reference/modules/shard-ops.asciidoc @@ -25,7 +25,7 @@ By default, the primary and replica shard copies for an index can be allocated t You can control how shard copies are allocated using the following settings: -- <>: Use these settings to control how shard copies are allocated and balanced across the entire cluster. For example, you might want to allocate nodes availability zones, or prevent certain nodes from being used so you can perform maintenance. +- <>: Use these settings to control how shard copies are allocated and balanced across the entire cluster. For example, you might want to <>, or prevent certain nodes from being used so you can perform maintenance. - <>: Use these settings to control how the shard copies for a specific index are allocated. For example, you might want to allocate an index to a node in a specific data tier, or to an node with specific attributes. @@ -80,4 +80,4 @@ When a shard copy is relocated, it is created as a new shard copy on the target You can control how and when shard copies are relocated. For example, you can adjust the rebalancing settings that control when shard copies are relocated to balance the cluster, or the high watermark for disk-based shard allocation that can trigger relocation. These settings are part of the <>. -Shard relocation operations also respect shard allocation and recovery settings. \ No newline at end of file +Shard relocation operations also respect shard allocation and recovery settings. \ No newline at end of file diff --git a/docs/reference/monitoring/index.asciidoc b/docs/reference/monitoring/index.asciidoc index 1b83f4c11ba54..82e1447ba8a1f 100644 --- a/docs/reference/monitoring/index.asciidoc +++ b/docs/reference/monitoring/index.asciidoc @@ -9,6 +9,7 @@ performance of your {es} cluster. * <> * <> +* <> * <> * <> * <> @@ -23,6 +24,8 @@ include::overview.asciidoc[] include::how-monitoring-works.asciidoc[] +include::{es-ref-dir}/setup/logging-config.asciidoc[] + include::production.asciidoc[] include::configuring-elastic-agent.asciidoc[] diff --git a/docs/reference/node-roles.asciidoc b/docs/reference/node-roles.asciidoc new file mode 100644 index 0000000000000..e8c1d9143a38e --- /dev/null +++ b/docs/reference/node-roles.asciidoc @@ -0,0 +1,437 @@ +[[node-roles-overview]] +== Node roles + +Any time that you start an instance of {es}, you are starting a _node_. A +collection of connected nodes is called a <>. If you +are running a single node of {es}, then you have a cluster of one node. All nodes know about all the other nodes in the cluster and can forward client +requests to the appropriate node. + +Each node performs one or more roles. Roles control the behavior of the node in the cluster. + +[discrete] +[[set-node-roles]] +=== Set node roles + +You define a node's roles by setting `node.roles` in <>. If you set `node.roles`, the node is only assigned the roles you specify. If you don't set `node.roles`, the node is assigned the following roles: + +* `master` +* `data` +* `data_content` +* `data_hot` +* `data_warm` +* `data_cold` +* `data_frozen` +* `ingest` +* `ml` +* `remote_cluster_client` +* `transform` + +[IMPORTANT] +==== +If you set `node.roles`, ensure you specify every node role your cluster needs. +Every cluster requires the following node roles: + +* `master` +* {blank} ++ +-- +`data_content` and `data_hot` + +OR + +`data` +-- + +Some {stack} features also require specific node roles: + +- {ccs-cap} and {ccr} require the `remote_cluster_client` role. +- {stack-monitor-app} and ingest pipelines require the `ingest` role. +- {fleet}, the {security-app}, and {transforms} require the `transform` role. + The `remote_cluster_client` role is also required to use {ccs} with these + features. +- {ml-cap} features, such as {anomaly-detect}, require the `ml` role. +==== + +As the cluster grows and in particular if you have large {ml} jobs or +{ctransforms}, consider separating dedicated master-eligible nodes from +dedicated data nodes, {ml} nodes, and {transform} nodes. + +[discrete] +[[change-node-role]] +=== Change the role of a node + +Each data node maintains the following data on disk: + +* the shard data for every shard allocated to that node, +* the index metadata corresponding with every shard allocated to that node, and +* the cluster-wide metadata, such as settings and index templates. + +Similarly, each master-eligible node maintains the following data on disk: + +* the index metadata for every index in the cluster, and +* the cluster-wide metadata, such as settings and index templates. + +Each node checks the contents of its data path at startup. If it discovers +unexpected data then it will refuse to start. This is to avoid importing +unwanted <> which can lead +to a red cluster health. To be more precise, nodes without the `data` role will +refuse to start if they find any shard data on disk at startup, and nodes +without both the `master` and `data` roles will refuse to start if they have any +index metadata on disk at startup. + +It is possible to change the roles of a node by adjusting its +`elasticsearch.yml` file and restarting it. This is known as _repurposing_ a +node. In order to satisfy the checks for unexpected data described above, you +must perform some extra steps to prepare a node for repurposing when starting +the node without the `data` or `master` roles. + +* If you want to repurpose a data node by removing the `data` role then you + should first use an <> to safely + migrate all the shard data onto other nodes in the cluster. + +* If you want to repurpose a node to have neither the `data` nor `master` roles + then it is simplest to start a brand-new node with an empty data path and the + desired roles. You may find it safest to use an + <> to migrate the shard data elsewhere + in the cluster first. + +If it is not possible to follow these extra steps then you may be able to use +the <> tool to delete any +excess data that prevents a node from starting. + +[discrete] +[[node-roles-list]] +=== Available node roles + +The following is a list of the roles that a node can perform in a cluster. A node can have one or more roles. + +* <> (`master`): A node that is eligible to be +<>, which controls the cluster. + +* <> (`data`, `data_content`, `data_hot`, `data_warm`, `data_cold`, `data_frozen`): A node that has one of several data roles. Data nodes hold data and perform data related operations such as CRUD, search, and aggregations. You might use multiple data roles in a cluster so you can implement <>. + +* <> (`ingest`): Ingest nodes are able to apply an <> to a document in order to transform and enrich the document before indexing. With a heavy ingest load, it makes sense to use dedicated ingest nodes and to not include the `ingest` role from nodes that have the `master` or `data` roles. + +* <> (`remote_cluster_client`): A node that is eligible to act as a remote client. + +* <> (`ml`): A node that can run {ml-features}. If you want to use {ml-features}, there must be at least one {ml} node in your cluster. For more information, see <> and {ml-docs}/index.html[Machine learning in the {stack}]. + +* <> (`transform`): A node that can perform {transforms}. If you want to use {transforms}, there must be at least one {transform} node in your cluster. For more information, see <> and <>. + +[NOTE] +[[coordinating-node]] +.Coordinating node +=============================================== + +Requests like search requests or bulk-indexing requests may involve data held +on different data nodes. A search request, for example, is executed in two +phases which are coordinated by the node which receives the client request -- +the _coordinating node_. + +In the _scatter_ phase, the coordinating node forwards the request to the data +nodes which hold the data. Each data node executes the request locally and +returns its results to the coordinating node. In the _gather_ phase, the +coordinating node reduces each data node's results into a single global +result set. + +Every node is implicitly a coordinating node. This means that a node that has +an explicit empty list of roles in the `node.roles` setting will only act as a coordinating +node, which cannot be disabled. As a result, such a node needs to have enough +memory and CPU in order to deal with the gather phase. + +=============================================== + +[discrete] + +[[master-node-role]] +==== Master-eligible node + +The master node is responsible for lightweight cluster-wide actions such as +creating or deleting an index, tracking which nodes are part of the cluster, +and deciding which shards to allocate to which nodes. It is important for +cluster health to have a stable master node. + +Any master-eligible node that is not a <> may +be elected to become the master node by the <>. + +IMPORTANT: Master nodes must have a `path.data` directory whose contents +persist across restarts, just like data nodes, because this is where the +cluster metadata is stored. The cluster metadata describes how to read the data +stored on the data nodes, so if it is lost then the data stored on the data +nodes cannot be read. + +[discrete] +[[dedicated-master-node]] +===== Dedicated master-eligible node + +It is important for the health of the cluster that the elected master node has +the resources it needs to fulfill its responsibilities. If the elected master +node is overloaded with other tasks then the cluster will not operate well. The +most reliable way to avoid overloading the master with other tasks is to +configure all the master-eligible nodes to be _dedicated master-eligible nodes_ +which only have the `master` role, allowing them to focus on managing the +cluster. Master-eligible nodes will still also behave as +<> that route requests from clients to +the other nodes in the cluster, but you should _not_ use dedicated master nodes +for this purpose. + +A small or lightly-loaded cluster may operate well if its master-eligible nodes +have other roles and responsibilities, but once your cluster comprises more +than a handful of nodes it usually makes sense to use dedicated master-eligible +nodes. + +To create a dedicated master-eligible node, set: + +[source,yaml] +------------------- +node.roles: [ master ] +------------------- + +[discrete] +[[voting-only-node]] +===== Voting-only master-eligible node + +A voting-only master-eligible node is a node that participates in +<> but which will not act as the cluster's +elected master node. In particular, a voting-only node can serve as a tiebreaker +in elections. + +It may seem confusing to use the term "master-eligible" to describe a +voting-only node since such a node is not actually eligible to become the master +at all. This terminology is an unfortunate consequence of history: +master-eligible nodes are those nodes that participate in elections and perform +certain tasks during cluster state publications, and voting-only nodes have the +same responsibilities even if they can never become the elected master. + +To configure a master-eligible node as a voting-only node, include `master` and +`voting_only` in the list of roles. For example to create a voting-only data +node: + +[source,yaml] +------------------- +node.roles: [ data, master, voting_only ] +------------------- + +IMPORTANT: Only nodes with the `master` role can be marked as having the +`voting_only` role. + +High availability (HA) clusters require at least three master-eligible nodes, at +least two of which are not voting-only nodes. Such a cluster will be able to +elect a master node even if one of the nodes fails. + +Voting-only master-eligible nodes may also fill other roles in your cluster. +For instance, a node may be both a data node and a voting-only master-eligible +node. A _dedicated_ voting-only master-eligible nodes is a voting-only +master-eligible node that fills no other roles in the cluster. To create a +dedicated voting-only master-eligible node, set: + +[source,yaml] +------------------- +node.roles: [ master, voting_only ] +------------------- + +Since dedicated voting-only nodes never act as the cluster's elected master, +they may require less heap and a less powerful CPU than the true master nodes. +However all master-eligible nodes, including voting-only nodes, are on the +critical path for <>. Cluster state updates are usually independent of +performance-critical workloads such as indexing or searches, but they are +involved in management activities such as index creation and rollover, mapping +updates, and recovery after a failure. The performance characteristics of these +activities are a function of the speed of the storage on each master-eligible +node, as well as the reliability and latency of the network interconnections +between the elected master node and the other nodes in the cluster. You must +therefore ensure that the storage and networking available to the nodes in your +cluster are good enough to meet your performance goals. + +[discrete] +[[data-node-role]] +==== Data nodes + +Data nodes hold the shards that contain the documents you have indexed. Data +nodes handle data related operations like CRUD, search, and aggregations. +These operations are I/O-, memory-, and CPU-intensive. It is important to +monitor these resources and to add more data nodes if they are overloaded. + +The main benefit of having dedicated data nodes is the separation of the master +and data roles. + +In a multi-tier deployment architecture, you use specialized data roles to +assign data nodes to specific tiers: `data_content`,`data_hot`, `data_warm`, +`data_cold`, or `data_frozen`. A node can belong to multiple tiers. + +If you want to include a node in all tiers, or if your cluster does not use multiple tiers, then you can use the generic `data` role. + +include::{es-ref-dir}/how-to/shard-limits.asciidoc[] + +WARNING: If you assign a node to a specific tier using a specialized data role, then you shouldn't also assign it the generic `data` role. The generic `data` role takes precedence over specialized data roles. + +[discrete] +[[generic-data-node]] +===== Generic data node + +Generic data nodes are included in all content tiers. A node with a generic `data` role can fill any of the specialized data node roles. + +To create a dedicated generic data node, set: +[source,yaml] +---- +node.roles: [ data ] +---- + +[discrete] +[[data-content-node]] +===== Content data node + +Content data nodes are part of the content tier. +include::{es-ref-dir}/datatiers.asciidoc[tag=content-tier] + +To create a dedicated content node, set: +[source,yaml] +---- +node.roles: [ data_content ] +---- + +[discrete] +[[data-hot-node]] +===== Hot data node + +Hot data nodes are part of the hot tier. +include::{es-ref-dir}/datatiers.asciidoc[tag=hot-tier] + +To create a dedicated hot node, set: +[source,yaml] +---- +node.roles: [ data_hot ] +---- + +[discrete] +[[data-warm-node]] +===== Warm data node + +Warm data nodes are part of the warm tier. +include::{es-ref-dir}/datatiers.asciidoc[tag=warm-tier] + +To create a dedicated warm node, set: +[source,yaml] +---- +node.roles: [ data_warm ] +---- + +[discrete] +[[data-cold-node]] +===== Cold data node + +Cold data nodes are part of the cold tier. +include::{es-ref-dir}/datatiers.asciidoc[tag=cold-tier] + +To create a dedicated cold node, set: +[source,yaml] +---- +node.roles: [ data_cold ] +---- + +[discrete] +[[data-frozen-node]] +===== Frozen data node + +Frozen data nodes are part of the frozen tier. +include::{es-ref-dir}/datatiers.asciidoc[tag=frozen-tier] + +To create a dedicated frozen node, set: +[source,yaml] +---- +node.roles: [ data_frozen ] +---- + +[discrete] +[[node-ingest-node]] +==== Ingest node + +Ingest nodes can execute pre-processing pipelines, composed of one or more +ingest processors. Depending on the type of operations performed by the ingest +processors and the required resources, it may make sense to have dedicated +ingest nodes, that will only perform this specific task. + +To create a dedicated ingest node, set: + +[source,yaml] +---- +node.roles: [ ingest ] +---- + +[discrete] +[[coordinating-only-node]] +==== Coordinating only node + +If you take away the ability to be able to handle master duties, to hold data, +and pre-process documents, then you are left with a _coordinating_ node that +can only route requests, handle the search reduce phase, and distribute bulk +indexing. Essentially, coordinating only nodes behave as smart load balancers. + +Coordinating only nodes can benefit large clusters by offloading the +coordinating node role from data and master-eligible nodes. They join the +cluster and receive the full <>, like every other +node, and they use the cluster state to route requests directly to the +appropriate place(s). + +WARNING: Adding too many coordinating only nodes to a cluster can increase the +burden on the entire cluster because the elected master node must await +acknowledgement of cluster state updates from every node! The benefit of +coordinating only nodes should not be overstated -- data nodes can happily +serve the same purpose. + +To create a dedicated coordinating node, set: + +[source,yaml] +---- +node.roles: [ ] +---- + +[discrete] +[[remote-node]] +==== Remote-eligible node + +A remote-eligible node acts as a cross-cluster client and connects to +<>. Once connected, you can search +remote clusters using <>. You can also sync +data between clusters using <>. + +[source,yaml] +---- +node.roles: [ remote_cluster_client ] +---- + +[discrete] +[[ml-node-role]] +==== [xpack]#Machine learning node# + +{ml-cap} nodes run jobs and handle {ml} API requests. For more information, see +<>. + +To create a dedicated {ml} node, set: + +[source,yaml] +---- +node.roles: [ ml, remote_cluster_client] +---- + +The `remote_cluster_client` role is optional but strongly recommended. +Otherwise, {ccs} fails when used in {ml} jobs or {dfeeds}. If you use {ccs} in +your {anomaly-jobs}, the `remote_cluster_client` role is also required on all +master-eligible nodes. Otherwise, the {dfeed} cannot start. See <>. + +[discrete] +[[transform-node-role]] +==== [xpack]#{transform-cap} node# + +{transform-cap} nodes run {transforms} and handle {transform} API requests. For +more information, see <>. + +To create a dedicated {transform} node, set: + +[source,yaml] +---- +node.roles: [ transform, remote_cluster_client ] +---- + +The `remote_cluster_client` role is optional but strongly recommended. +Otherwise, {ccs} fails when used in {transforms}. See <>. \ No newline at end of file diff --git a/docs/reference/path-settings-overview.asciidoc b/docs/reference/path-settings-overview.asciidoc new file mode 100644 index 0000000000000..0740b9769c9b2 --- /dev/null +++ b/docs/reference/path-settings-overview.asciidoc @@ -0,0 +1,112 @@ +[[path-settings-overview]] +=== Path settings + +include::{es-ref-dir}/setup/important-settings/path-settings.asciidoc[] + +[[multiple-data-paths]] +==== Multiple data paths +deprecated::[7.13.0] + +If needed, you can specify multiple paths in `path.data`. {es} stores the node's +data across all provided paths but keeps each shard's data on the same path. + +{es} does not balance shards across a node's data paths. High disk +usage in a single path can trigger a <> for the entire node. If triggered, {es} will not add shards to +the node, even if the node’s other paths have available disk space. If you need +additional disk space, we recommend you add a new node rather than additional +data paths. + +include::{es-ref-dir}/tab-widgets/multi-data-path-widget.asciidoc[] + +[[mdp-migrate]] +===== Migrate from multiple data paths + +Support for multiple data paths was deprecated in 7.13 and will be removed +in a future release. + +As an alternative to multiple data paths, you can create a filesystem which +spans multiple disks with a hardware virtualisation layer such as RAID, or a +software virtualisation layer such as Logical Volume Manager (LVM) on Linux or +Storage Spaces on Windows. If you wish to use multiple data paths on a single +machine then you must run one node for each data path. + +If you currently use multiple data paths in a +{ref}/high-availability-cluster-design.html[highly available cluster] then you +can migrate to a setup that uses a single path for each node without downtime +using a process similar to a +{ref}/restart-cluster.html#restart-cluster-rolling[rolling restart]: shut each +node down in turn and replace it with one or more nodes each configured to use +a single data path. In more detail, for each node that currently has multiple +data paths you should follow the following process. In principle you can +perform this migration during a rolling upgrade to 8.0, but we recommend +migrating to a single-data-path setup before starting to upgrade. + +1. Take a snapshot to protect your data in case of disaster. + +2. Optionally, migrate the data away from the target node by using an +{ref}/modules-cluster.html#cluster-shard-allocation-filtering[allocation filter]: ++ +[source,console] +-------------------------------------------------- +PUT _cluster/settings +{ + "persistent": { + "cluster.routing.allocation.exclude._name": "target-node-name" + } +} +-------------------------------------------------- ++ +You can use the {ref}/cat-allocation.html[cat allocation API] to track progress +of this data migration. If some shards do not migrate then the +{ref}/cluster-allocation-explain.html[cluster allocation explain API] will help +you to determine why. + +3. Follow the steps in the +{ref}/restart-cluster.html#restart-cluster-rolling[rolling restart process] +up to and including shutting the target node down. + +4. Ensure your cluster health is `yellow` or `green`, so that there is a copy +of every shard assigned to at least one of the other nodes in your cluster. + +5. If applicable, remove the allocation filter applied in the earlier step. ++ +[source,console] +-------------------------------------------------- +PUT _cluster/settings +{ + "persistent": { + "cluster.routing.allocation.exclude._name": null + } +} +-------------------------------------------------- + +6. Discard the data held by the stopped node by deleting the contents of its +data paths. + +7. Reconfigure your storage. For instance, combine your disks into a single +filesystem using LVM or Storage Spaces. Ensure that your reconfigured storage +has sufficient space for the data that it will hold. + +8. Reconfigure your node by adjusting the `path.data` setting in its +`elasticsearch.yml` file. If needed, install more nodes each with their own +`path.data` setting pointing at a separate data path. + +9. Start the new nodes and follow the rest of the +{ref}/restart-cluster.html#restart-cluster-rolling[rolling restart process] for +them. + +10. Ensure your cluster health is `green`, so that every shard has been +assigned. + +You can alternatively add some number of single-data-path nodes to your +cluster, migrate all your data over to these new nodes using +{ref}/modules-cluster.html#cluster-shard-allocation-filtering[allocation filters], +and then remove the old nodes from the cluster. This approach will temporarily +double the size of your cluster so it will only work if you have the capacity to +expand your cluster like this. + +If you currently use multiple data paths but your cluster is not highly +available then you can migrate to a non-deprecated configuration by taking +a snapshot, creating a new cluster with the desired configuration and restoring +the snapshot into it. diff --git a/docs/reference/quickstart/full-text-filtering-tutorial.asciidoc b/docs/reference/quickstart/full-text-filtering-tutorial.asciidoc index a024305588cae..b602ee5076434 100644 --- a/docs/reference/quickstart/full-text-filtering-tutorial.asciidoc +++ b/docs/reference/quickstart/full-text-filtering-tutorial.asciidoc @@ -4,7 +4,7 @@ Basics: Full-text search and filtering ++++ -This is a hands-on introduction to the basics of full-text search with {es}, also known as _lexical search_, using the <> and <>. +This is a hands-on introduction to the basics of <> with {es}, also known as _lexical search_, using the <> and <>. You'll also learn how to filter data, to narrow down search results based on exact criteria. In this scenario, we're implementing a search function for a cooking blog. @@ -632,6 +632,7 @@ This tutorial introduced the basics of full-text search and filtering in {es}. Building a real-world search experience requires understanding many more advanced concepts and techniques. Here are some resources once you're ready to dive deeper: +* <>: Learn about the core components of full-text search in {es}. * <>: Understand all your options for searching and analyzing data in {es}. * <>: Understand how text is processed for full-text search. * <>: Learn about more advanced search techniques using the `_search` API, including semantic search. diff --git a/docs/reference/search/multi-search-template-api.asciidoc b/docs/reference/search/multi-search-template-api.asciidoc index 010320c6b05ed..2fdb412575eb4 100644 --- a/docs/reference/search/multi-search-template-api.asciidoc +++ b/docs/reference/search/multi-search-template-api.asciidoc @@ -85,7 +85,7 @@ cross-cluster search requests. Defaults to `true`. `max_concurrent_searches`:: (Optional, integer) Maximum number of concurrent searches the API can run. -Defaults to +max(1, (# of <> * +Defaults to +max(1, (# of <> * min(<>, 10)))+. `rest_total_hits_as_int`:: diff --git a/docs/reference/search/multi-search.asciidoc b/docs/reference/search/multi-search.asciidoc index ea2dd59779332..6adcc62e5ec4f 100644 --- a/docs/reference/search/multi-search.asciidoc +++ b/docs/reference/search/multi-search.asciidoc @@ -97,7 +97,7 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailabl `max_concurrent_searches`:: (Optional, integer) Maximum number of concurrent searches the multi search API can execute. Defaults -to +max(1, (# of <> * min(<>, 10)))+. +to +max(1, (# of <> * min(<>, 10)))+. `max_concurrent_shard_requests`:: + diff --git a/docs/reference/search/search-your-data/full-text-search.asciidoc b/docs/reference/search/search-your-data/full-text-search.asciidoc new file mode 100644 index 0000000000000..8641d0e45748a --- /dev/null +++ b/docs/reference/search/search-your-data/full-text-search.asciidoc @@ -0,0 +1,82 @@ +[[full-text-search]] +== Full-text search + +.Hands-on introduction to full-text search +[TIP] +==== +Would you prefer to jump straight into a hands-on tutorial? +Refer to our quick start <>. +==== + +Full-text search, also known as lexical search, is a technique for fast, efficient searching through text fields in documents. +Documents and search queries are transformed to enable returning https://www.elastic.co/what-is/search-relevance[relevant] results instead of simply exact term matches. +Fields of type <> are analyzed and indexed for full-text search. + +Built on decades of information retrieval research, full-text search delivers reliable results that scale predictably as your data grows. Because it runs efficiently on CPUs, {es}'s full-text search requires minimal computational resources compared to GPU-intensive vector operations. + +You can combine full-text search with <> to build modern hybrid search applications. While vector search may require additional GPU resources, the full-text component remains cost-effective by leveraging existing CPU infrastructure. + +[discrete] +[[full-text-search-how-it-works]] +=== How full-text search works + +The following diagram illustrates the components of full-text search. + +image::images/search/full-text-search-overview.svg[Components of full-text search from analysis to relevance scoring, align=center, width=500] + +At a high level, full-text search involves the following: + +* <>: Analysis consists of a pipeline of sequential transformations. Text is transformed into a format optimized for searching using techniques such as stemming, lowercasing, and stop word elimination. {es} contains a number of built-in <> and tokenizers, including options to analyze specific language text. You can also create custom analyzers. ++ +[TIP] +==== +Refer to <> to learn how to test an analyzer and inspect the tokens and metadata it generates. +==== +* *Inverted index creation*: After analysis is complete, {es} builds an inverted index from the resulting tokens. +An inverted index is a data structure that maps each token to the documents that contain it. +It's made up of two key components: +** *Dictionary*: A sorted list of all unique terms in the collection of documents in your index. +** *Posting list*: For each term, a list of document IDs where the term appears, along with optional metadata like term frequency and position. +* *Relevance scoring*: Results are ranked by how relevant they are to the given query. The relevance score of each document is represented by a positive floating-point number called the `_score`. The higher the `_score`, the more relevant the document. ++ +The default <> {es} uses for calculating relevance scores is https://en.wikipedia.org/wiki/Okapi_BM25[Okapi BM25], a variation of the https://en.wikipedia.org/wiki/Tf–idf[TF-IDF algorithm]. BM25 calculates relevance scores based on term frequency, document frequency, and document length. +Refer to this https://www.elastic.co/blog/practical-bm25-part-2-the-bm25-algorithm-and-its-variables[technical blog post] for a deep dive into BM25. +* *Full-text search query*: Query text is analyzed <>, and the resulting tokens are used to search the inverted index. ++ +Query DSL supports a number of <>. ++ +As of 8.17, {esql} also supports <> functions. + +[discrete] +[[full-text-search-getting-started]] +=== Getting started + +For a hands-on introduction to full-text search, refer to the <>. + +[discrete] +[[full-text-search-learn-more]] +=== Learn more + +Here are some resources to help you learn more about full-text search with {es}. + +*Core concepts* + +Learn about the core components of full-text search: + +* <> +* <> +** <> +** <> + +*{es} query languages* + +Learn how to build full-text search queries using {es}'s query languages: + +* <> +* <> + +*Advanced topics* + +For a technical deep dive into {es}'s BM25 implementation read this blog post: https://www.elastic.co/blog/practical-bm25-part-2-the-bm25-algorithm-and-its-variables[The BM25 Algorithm and its Variables]. + +To learn how to optimize the relevance of your search results, refer to <>. \ No newline at end of file diff --git a/docs/reference/search/search-your-data/search-your-data.asciidoc b/docs/reference/search/search-your-data/search-your-data.asciidoc index b38af1fffca25..0828462fd1850 100644 --- a/docs/reference/search/search-your-data/search-your-data.asciidoc +++ b/docs/reference/search/search-your-data/search-your-data.asciidoc @@ -18,7 +18,7 @@ Search for exact values:: Search for <> of numbers, dates, IPs, or strings. -Full-text search:: +<>:: Use <> to query <> and find documents that best match query terms. @@ -43,6 +43,7 @@ DSL, with a simplified user experience. Create search applications based on your results directly in the Kibana Search UI. include::search-api.asciidoc[] +include::full-text-search.asciidoc[] include::../../how-to/recipes.asciidoc[] // ☝️ search relevance recipes include::retrievers-overview.asciidoc[] diff --git a/docs/reference/security/authorization/built-in-roles.asciidoc b/docs/reference/security/authorization/built-in-roles.asciidoc index 13812b915dc5e..846ab3b6f73aa 100644 --- a/docs/reference/security/authorization/built-in-roles.asciidoc +++ b/docs/reference/security/authorization/built-in-roles.asciidoc @@ -33,18 +33,6 @@ suitable for writing beats output to {es}. -- -[[built-in-roles-data-frame-transforms-admin]] `data_frame_transforms_admin` :: -Grants `manage_data_frame_transforms` cluster privileges, which enable you to -manage {transforms}. This role also includes all -{kibana-ref}/kibana-privileges.html[Kibana privileges] for the {ml-features}. -deprecated:[7.5.0,"Replaced by <>"]. - -[[built-in-roles-data-frame-transforms-user]] `data_frame_transforms_user` :: -Grants `monitor_data_frame_transforms` cluster privileges, which enable you to -use {transforms}. This role also includes all -{kibana-ref}/kibana-privileges.html[Kibana privileges] for the {ml-features}. -deprecated:[7.5.0,"Replaced by <>"]. - [[built-in-roles-editor]] `editor` :: Grants full access to all features in {kib} (including Solutions) and read-only access to data indices. diff --git a/docs/reference/settings/ml-settings.asciidoc b/docs/reference/settings/ml-settings.asciidoc index 1077a63b00249..08163b1391f2d 100644 --- a/docs/reference/settings/ml-settings.asciidoc +++ b/docs/reference/settings/ml-settings.asciidoc @@ -19,6 +19,8 @@ at all. // end::ml-settings-description-tag[] +TIP: To control memory usage used by {ml} jobs, you can use the <>. + [discrete] [[general-ml-settings]] ==== General machine learning settings @@ -67,7 +69,7 @@ limitations as described <>. The inference cache exists in the JVM heap on each ingest node. The cache affords faster processing times for the `inference` processor. The value can be a static byte sized value (such as `2gb`) or a percentage of total allocated -heap. Defaults to `40%`. See also <>. +heap. Defaults to `40%`. See also <>. [[xpack-interference-model-ttl]] // tag::interference-model-ttl-tag[] @@ -249,11 +251,4 @@ nodes in your cluster, you shouldn't use this setting. + If this setting is `true` it also affects the default value for `xpack.ml.max_model_memory_limit`. In this case `xpack.ml.max_model_memory_limit` -defaults to the largest size that could be assigned in the current cluster. - -[discrete] -[[model-inference-circuit-breaker]] -==== {ml-cap} circuit breaker settings - -The relevant circuit breaker settings can be found in the <>. - +defaults to the largest size that could be assigned in the current cluster. \ No newline at end of file diff --git a/docs/reference/setup.asciidoc b/docs/reference/setup.asciidoc index 80828fdbfbb02..922f1bdba4d1f 100644 --- a/docs/reference/setup.asciidoc +++ b/docs/reference/setup.asciidoc @@ -27,6 +27,8 @@ the only resource-intensive application on the host or container. For example, you might run {metricbeat} alongside {es} for cluster statistics, but a resource-heavy {ls} deployment should be on its own host. +// alphabetized + include::run-elasticsearch-locally.asciidoc[] include::setup/install.asciidoc[] @@ -47,30 +49,28 @@ include::settings/ccr-settings.asciidoc[] include::modules/discovery/discovery-settings.asciidoc[] +include::settings/data-stream-lifecycle-settings.asciidoc[] + include::modules/indices/fielddata.asciidoc[] +include::modules/gateway.asciidoc[] + include::settings/health-diagnostic-settings.asciidoc[] include::settings/ilm-settings.asciidoc[] -include::settings/data-stream-lifecycle-settings.asciidoc[] - include::modules/indices/index_management.asciidoc[] include::modules/indices/recovery.asciidoc[] include::modules/indices/indexing_buffer.asciidoc[] -include::settings/license-settings.asciidoc[] - -include::modules/gateway.asciidoc[] +include::settings/inference-settings.asciidoc[] -include::setup/logging-config.asciidoc[] +include::settings/license-settings.asciidoc[] include::settings/ml-settings.asciidoc[] -include::settings/inference-settings.asciidoc[] - include::settings/monitoring-settings.asciidoc[] include::modules/node.asciidoc[] @@ -79,6 +79,8 @@ include::modules/network.asciidoc[] include::modules/indices/query_cache.asciidoc[] +include::{es-ref-dir}/path-settings-overview.asciidoc[] + include::modules/indices/search-settings.asciidoc[] include::settings/security-settings.asciidoc[] diff --git a/docs/reference/setup/add-nodes.asciidoc b/docs/reference/setup/add-nodes.asciidoc index ba749782c092f..941a3e6c40f79 100644 --- a/docs/reference/setup/add-nodes.asciidoc +++ b/docs/reference/setup/add-nodes.asciidoc @@ -48,7 +48,7 @@ For more information about discovery and shard allocation, refer to As nodes are added or removed Elasticsearch maintains an optimal level of fault tolerance by automatically updating the cluster's _voting configuration_, which -is the set of <> whose responses are counted +is the set of <> whose responses are counted when making decisions such as electing a new master or committing a new cluster state. diff --git a/docs/reference/setup/advanced-configuration.asciidoc b/docs/reference/setup/advanced-configuration.asciidoc index 2a7ccc56742de..73b210ea559b2 100644 --- a/docs/reference/setup/advanced-configuration.asciidoc +++ b/docs/reference/setup/advanced-configuration.asciidoc @@ -1,13 +1,7 @@ [[advanced-configuration]] -=== Advanced configuration - -Modifying advanced settings is generally not recommended and could negatively -impact performance and stability. Using the {es}-provided defaults -is recommended in most circumstances. +=== Set JVM options [[set-jvm-options]] -==== Set JVM options - If needed, you can override the default JVM options by adding custom options files (preferred) or setting the `ES_JAVA_OPTS` environment variable. @@ -21,10 +15,14 @@ Where you put the JVM options files depends on the type of installation: * Docker: Bind mount custom JVM options files into `/usr/share/elasticsearch/config/jvm.options.d/`. +CAUTION: Setting your own JVM options is generally not recommended and could negatively +impact performance and stability. Using the {es}-provided defaults +is recommended in most circumstances. + NOTE: Do not modify the root `jvm.options` file. Use files in `jvm.options.d/` instead. [[jvm-options-syntax]] -===== JVM options syntax +==== JVM options syntax A JVM options file contains a line-delimited list of JVM arguments. Arguments are preceded by a dash (`-`). @@ -66,7 +64,7 @@ and ignored. Lines that aren't commented out and aren't recognized as valid JVM arguments are rejected and {es} will fail to start. [[jvm-options-env]] -===== Use environment variables to set JVM options +==== Use environment variables to set JVM options In production, use JVM options files to override the default settings. In testing and development environments, @@ -155,23 +153,11 @@ options. We do not recommend using `ES_JAVA_OPTS` in production. NOTE: If you are running {es} as a Windows service, you can change the heap size using the service manager. See <>. -[[readiness-tcp-port]] -===== Enable the Elasticsearch TCP readiness port - -preview::[] - -If configured, a node can open a TCP port when the node is in a ready state. A node is deemed -ready when it has successfully joined a cluster. In a single node configuration, the node is -said to be ready, when it's able to accept requests. - -To enable the readiness TCP port, use the `readiness.port` setting. The readiness service will bind to -all host addresses. - -If the node leaves the cluster, or the <> is used to mark the node -for shutdown, the readiness port is immediately closed. - -A successful connection to the readiness TCP port signals that the {es} node is ready. When a client -connects to the readiness port, the server simply terminates the socket connection. No data is sent back -to the client. If a client cannot connect to the readiness port, the node is not ready. +[[heap-dump-path]] +include::important-settings/heap-dump-path.asciidoc[leveloffset=-1] +[[gc-logging]] +include::important-settings/gc-logging.asciidoc[leveloffset=-1] +[[error-file-path]] +include::important-settings/error-file.asciidoc[leveloffset=-1] \ No newline at end of file diff --git a/docs/reference/setup/important-settings.asciidoc b/docs/reference/setup/important-settings.asciidoc index 03c891af70743..26f9c79cb6693 100644 --- a/docs/reference/setup/important-settings.asciidoc +++ b/docs/reference/setup/important-settings.asciidoc @@ -19,10 +19,20 @@ of items which *must* be considered before using your cluster in production: Our {ess-trial}[{ecloud}] service configures these items automatically, making your cluster production-ready by default. +[[path-settings]] +[discrete] +==== Path settings + include::important-settings/path-settings.asciidoc[] +Elasticsearch offers a deprecated setting that allows you to specify multiple paths in `path.data`. +To learn about this setting, and how to migrate away from it, refer to <>. + include::important-settings/cluster-name.asciidoc[] +[[node-name]] +[discrete] +==== Node name setting include::important-settings/node-name.asciidoc[] include::important-settings/network-host.asciidoc[] diff --git a/docs/reference/setup/important-settings/cluster-name.asciidoc b/docs/reference/setup/important-settings/cluster-name.asciidoc index 3f1516f21de1e..6d489eee76cf6 100644 --- a/docs/reference/setup/important-settings/cluster-name.asciidoc +++ b/docs/reference/setup/important-settings/cluster-name.asciidoc @@ -1,4 +1,3 @@ -[[cluster-name]] [discrete] ==== Cluster name setting diff --git a/docs/reference/setup/important-settings/error-file.asciidoc b/docs/reference/setup/important-settings/error-file.asciidoc index ca95ded78d53f..2f654002d51f8 100644 --- a/docs/reference/setup/important-settings/error-file.asciidoc +++ b/docs/reference/setup/important-settings/error-file.asciidoc @@ -1,4 +1,3 @@ -[[error-file-path]] [discrete] ==== JVM fatal error log setting diff --git a/docs/reference/setup/important-settings/gc-logging.asciidoc b/docs/reference/setup/important-settings/gc-logging.asciidoc index 3534e1335c9fd..873c85d58d914 100644 --- a/docs/reference/setup/important-settings/gc-logging.asciidoc +++ b/docs/reference/setup/important-settings/gc-logging.asciidoc @@ -1,4 +1,3 @@ -[[gc-logging]] [discrete] ==== GC logging settings @@ -20,9 +19,8 @@ To see further options not contained in the original JEP, see https://docs.oracle.com/en/java/javase/13/docs/specs/man/java.html#enable-logging-with-the-jvm-unified-logging-framework[Enable Logging with the JVM Unified Logging Framework]. -[[gc-logging-examples]] [discrete] -==== Examples +===== Examples Change the default GC log output location to `/opt/my-app/gc.log` by creating `$ES_HOME/config/jvm.options.d/gc.options` with some sample diff --git a/docs/reference/setup/important-settings/heap-dump-path.asciidoc b/docs/reference/setup/important-settings/heap-dump-path.asciidoc index 8f01379842a90..8b06ae752f360 100644 --- a/docs/reference/setup/important-settings/heap-dump-path.asciidoc +++ b/docs/reference/setup/important-settings/heap-dump-path.asciidoc @@ -1,4 +1,3 @@ -[[heap-dump-path]] [discrete] ==== JVM heap dump path setting diff --git a/docs/reference/setup/important-settings/node-name.asciidoc b/docs/reference/setup/important-settings/node-name.asciidoc index eda3052d119c9..f1260844a0549 100644 --- a/docs/reference/setup/important-settings/node-name.asciidoc +++ b/docs/reference/setup/important-settings/node-name.asciidoc @@ -1,7 +1,3 @@ -[[node-name]] -[discrete] -==== Node name setting - {es} uses `node.name` as a human-readable identifier for a particular instance of {es}. This name is included in the response of many APIs. The node name defaults to the hostname of the machine when diff --git a/docs/reference/setup/important-settings/path-settings.asciidoc b/docs/reference/setup/important-settings/path-settings.asciidoc index a0a444ca5090a..002e08e2dc746 100644 --- a/docs/reference/setup/important-settings/path-settings.asciidoc +++ b/docs/reference/setup/important-settings/path-settings.asciidoc @@ -1,7 +1,3 @@ -[[path-settings]] -[discrete] -==== Path settings - {es} writes the data you index to indices and data streams to a `data` directory. {es} writes its own application logs, which contain information about cluster health and operations, to a `logs` directory. @@ -20,113 +16,4 @@ Supported `path.data` and `path.logs` values vary by platform: include::{es-ref-dir}/tab-widgets/customize-data-log-path-widget.asciidoc[] -include::{es-ref-dir}/modules/node.asciidoc[tag=modules-node-data-path-warning-tag] - -[discrete] -==== Multiple data paths -deprecated::[7.13.0] - -If needed, you can specify multiple paths in `path.data`. {es} stores the node's -data across all provided paths but keeps each shard's data on the same path. - -{es} does not balance shards across a node's data paths. High disk -usage in a single path can trigger a <> for the entire node. If triggered, {es} will not add shards to -the node, even if the node’s other paths have available disk space. If you need -additional disk space, we recommend you add a new node rather than additional -data paths. - -include::{es-ref-dir}/tab-widgets/multi-data-path-widget.asciidoc[] - -[discrete] -[[mdp-migrate]] -==== Migrate from multiple data paths - -Support for multiple data paths was deprecated in 7.13 and will be removed -in a future release. - -As an alternative to multiple data paths, you can create a filesystem which -spans multiple disks with a hardware virtualisation layer such as RAID, or a -software virtualisation layer such as Logical Volume Manager (LVM) on Linux or -Storage Spaces on Windows. If you wish to use multiple data paths on a single -machine then you must run one node for each data path. - -If you currently use multiple data paths in a -{ref}/high-availability-cluster-design.html[highly available cluster] then you -can migrate to a setup that uses a single path for each node without downtime -using a process similar to a -{ref}/restart-cluster.html#restart-cluster-rolling[rolling restart]: shut each -node down in turn and replace it with one or more nodes each configured to use -a single data path. In more detail, for each node that currently has multiple -data paths you should follow the following process. In principle you can -perform this migration during a rolling upgrade to 8.0, but we recommend -migrating to a single-data-path setup before starting to upgrade. - -1. Take a snapshot to protect your data in case of disaster. - -2. Optionally, migrate the data away from the target node by using an -{ref}/modules-cluster.html#cluster-shard-allocation-filtering[allocation filter]: -+ -[source,console] --------------------------------------------------- -PUT _cluster/settings -{ - "persistent": { - "cluster.routing.allocation.exclude._name": "target-node-name" - } -} --------------------------------------------------- -+ -You can use the {ref}/cat-allocation.html[cat allocation API] to track progress -of this data migration. If some shards do not migrate then the -{ref}/cluster-allocation-explain.html[cluster allocation explain API] will help -you to determine why. - -3. Follow the steps in the -{ref}/restart-cluster.html#restart-cluster-rolling[rolling restart process] -up to and including shutting the target node down. - -4. Ensure your cluster health is `yellow` or `green`, so that there is a copy -of every shard assigned to at least one of the other nodes in your cluster. - -5. If applicable, remove the allocation filter applied in the earlier step. -+ -[source,console] --------------------------------------------------- -PUT _cluster/settings -{ - "persistent": { - "cluster.routing.allocation.exclude._name": null - } -} --------------------------------------------------- - -6. Discard the data held by the stopped node by deleting the contents of its -data paths. - -7. Reconfigure your storage. For instance, combine your disks into a single -filesystem using LVM or Storage Spaces. Ensure that your reconfigured storage -has sufficient space for the data that it will hold. - -8. Reconfigure your node by adjusting the `path.data` setting in its -`elasticsearch.yml` file. If needed, install more nodes each with their own -`path.data` setting pointing at a separate data path. - -9. Start the new nodes and follow the rest of the -{ref}/restart-cluster.html#restart-cluster-rolling[rolling restart process] for -them. - -10. Ensure your cluster health is `green`, so that every shard has been -assigned. - -You can alternatively add some number of single-data-path nodes to your -cluster, migrate all your data over to these new nodes using -{ref}/modules-cluster.html#cluster-shard-allocation-filtering[allocation filters], -and then remove the old nodes from the cluster. This approach will temporarily -double the size of your cluster so it will only work if you have the capacity to -expand your cluster like this. - -If you currently use multiple data paths but your cluster is not highly -available then you can migrate to a non-deprecated configuration by taking -a snapshot, creating a new cluster with the desired configuration and restoring -the snapshot into it. +include::{es-ref-dir}/modules/node.asciidoc[tag=modules-node-data-path-warning-tag] \ No newline at end of file diff --git a/docs/reference/setup/logging-config.asciidoc b/docs/reference/setup/logging-config.asciidoc index e382bbdacb464..04e9ba3f0bef9 100644 --- a/docs/reference/setup/logging-config.asciidoc +++ b/docs/reference/setup/logging-config.asciidoc @@ -1,5 +1,5 @@ [[logging]] -=== Logging +== Elasticsearch application logging You can use {es}'s application logs to monitor your cluster and diagnose issues. If you run {es} as a service, the default location of the logs varies based on @@ -11,7 +11,7 @@ If you run {es} from the command line, {es} prints logs to the standard output (`stdout`). [discrete] -[[loggin-configuration]] +[[logging-configuration]] === Logging configuration IMPORTANT: Elastic strongly recommends using the Log4j 2 configuration that is shipped by default. @@ -304,6 +304,7 @@ The user ID is included in the `X-Opaque-ID` field in deprecation JSON logs. Deprecation logs can be indexed into `.logs-deprecation.elasticsearch-default` data stream `cluster.deprecation_indexing.enabled` setting is set to true. +[discrete] ==== Deprecation logs throttling :es-rate-limiting-filter-java-doc: {elasticsearch-javadoc}/org/elasticsearch/common/logging/RateLimitingFilter.html Deprecation logs are deduplicated based on a deprecated feature key diff --git a/docs/reference/shard-request-cache.asciidoc b/docs/reference/shard-request-cache.asciidoc new file mode 100644 index 0000000000000..ec79dfb531bdb --- /dev/null +++ b/docs/reference/shard-request-cache.asciidoc @@ -0,0 +1,134 @@ +[[shard-request-cache]] +=== The shard request cache + +When a search request is run against an index or against many indices, each +involved shard executes the search locally and returns its local results to +the _coordinating node_, which combines these shard-level results into a +``global'' result set. + +The shard-level request cache module caches the local results on each shard. +This allows frequently used (and potentially heavy) search requests to return +results almost instantly. The requests cache is a very good fit for the logging +use case, where only the most recent index is being actively updated -- +results from older indices will be served directly from the cache. + +You can control the size and expiration of the cache at the node level using the <>. + +[IMPORTANT] +=================================== + +By default, the requests cache will only cache the results of search requests +where `size=0`, so it will not cache `hits`, +but it will cache `hits.total`, <>, and +<>. + +Most queries that use `now` (see <>) cannot be cached. + +Scripted queries that use the API calls which are non-deterministic, such as +`Math.random()` or `new Date()` are not cached. +=================================== + +[discrete] +==== Cache invalidation + +The cache is smart -- it keeps the same _near real-time_ promise as uncached +search. + +Cached results are invalidated automatically whenever the shard refreshes to +pick up changes to the documents or when you update the mapping. In other +words you will always get the same results from the cache as you would for an +uncached search request. + +The longer the refresh interval, the longer that cached entries will remain +valid even if there are changes to the documents. If the cache is full, the +least recently used cache keys will be evicted. + +The cache can be expired manually with the <>: + +[source,console] +------------------------ +POST /my-index-000001,my-index-000002/_cache/clear?request=true +------------------------ +// TEST[s/^/PUT my-index-000001\nPUT my-index-000002\n/] + +[discrete] +==== Enabling and disabling caching + +The cache is enabled by default, but can be disabled when creating a new +index as follows: + +[source,console] +----------------------------- +PUT /my-index-000001 +{ + "settings": { + "index.requests.cache.enable": false + } +} +----------------------------- + +It can also be enabled or disabled dynamically on an existing index with the +<> API: + +[source,console] +----------------------------- +PUT /my-index-000001/_settings +{ "index.requests.cache.enable": true } +----------------------------- +// TEST[continued] + + +[discrete] +==== Enabling and disabling caching per request + +The `request_cache` query-string parameter can be used to enable or disable +caching on a *per-request* basis. If set, it overrides the index-level setting: + +[source,console] +----------------------------- +GET /my-index-000001/_search?request_cache=true +{ + "size": 0, + "aggs": { + "popular_colors": { + "terms": { + "field": "colors" + } + } + } +} +----------------------------- +// TEST[continued] + +Requests where `size` is greater than 0 will not be cached even if the request cache is +enabled in the index settings. To cache these requests you will need to use the +query-string parameter detailed here. + +[discrete] +==== Cache key + +A hash of the whole JSON body is used as the cache key. This means that if the JSON +changes -- for instance if keys are output in a different order -- then the +cache key will not be recognised. + +TIP: Most JSON libraries support a _canonical_ mode which ensures that JSON +keys are always emitted in the same order. This canonical mode can be used in +the application to ensure that a request is always serialized in the same way. + +[discrete] +==== Monitoring cache usage + +The size of the cache (in bytes) and the number of evictions can be viewed +by index, with the <> API: + +[source,console] +------------------------ +GET /_stats/request_cache?human +------------------------ + +or by node with the <> API: + +[source,console] +------------------------ +GET /_nodes/stats/indices/request_cache?human +------------------------ diff --git a/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc index 2f2c7fcd8ebd2..89cd0f96915b9 100644 --- a/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc @@ -75,7 +75,7 @@ POST /_snapshot/my_repository/my_snapshot/_restore // tag::restore-prereqs[] * You can only restore a snapshot to a running cluster with an elected -<>. The snapshot's repository must be +<>. The snapshot's repository must be <> and available to the cluster. * The snapshot and cluster versions must be compatible. See diff --git a/docs/reference/snapshot-restore/take-snapshot.asciidoc b/docs/reference/snapshot-restore/take-snapshot.asciidoc index 711fcfe4cc484..1ae2258c7da89 100644 --- a/docs/reference/snapshot-restore/take-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/take-snapshot.asciidoc @@ -46,7 +46,7 @@ taking snapshots at different time intervals. include::register-repository.asciidoc[tag=kib-snapshot-prereqs] * You can only take a snapshot from a running cluster with an elected -<>. +<>. * A snapshot repository must be <> and available to the cluster. diff --git a/docs/reference/transform/setup.asciidoc b/docs/reference/transform/setup.asciidoc index dab357546d93e..3171086e43d61 100644 --- a/docs/reference/transform/setup.asciidoc +++ b/docs/reference/transform/setup.asciidoc @@ -11,7 +11,7 @@ To use {transforms}, you must have: -* at least one <>, +* at least one <>, * management features visible in the {kib} space, and * security privileges that: + diff --git a/docs/reference/upgrade/disable-shard-alloc.asciidoc b/docs/reference/upgrade/disable-shard-alloc.asciidoc index f69a673095257..4672ea65446b4 100644 --- a/docs/reference/upgrade/disable-shard-alloc.asciidoc +++ b/docs/reference/upgrade/disable-shard-alloc.asciidoc @@ -5,7 +5,7 @@ starting to replicate the shards on that node to other nodes in the cluster, which can involve a lot of I/O. Since the node is shortly going to be restarted, this I/O is unnecessary. You can avoid racing the clock by <> of replicas before -shutting down <>: +shutting down <>: [source,console] -------------------------------------------------- diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java index 8becc1e50ffcc..67d006868b48d 100644 --- a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java +++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java @@ -26,6 +26,20 @@ public interface EntitlementChecker { void check$java_lang_Runtime$halt(Class callerClass, Runtime runtime, int status); + // ClassLoader ctor + void check$java_lang_ClassLoader$(Class callerClass); + + void check$java_lang_ClassLoader$(Class callerClass, ClassLoader parent); + + void check$java_lang_ClassLoader$(Class callerClass, String name, ClassLoader parent); + + // SecureClassLoader ctor + void check$java_security_SecureClassLoader$(Class callerClass); + + void check$java_security_SecureClassLoader$(Class callerClass, ClassLoader parent); + + void check$java_security_SecureClassLoader$(Class callerClass, String name, ClassLoader parent); + // URLClassLoader constructors void check$java_net_URLClassLoader$(Class callerClass, URL[] urls); diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java index 2abfb11964a93..257d130302580 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java @@ -16,6 +16,7 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.entitlement.initialization.EntitlementInitialization; +import org.elasticsearch.entitlement.runtime.api.NotEntitledException; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; @@ -23,14 +24,24 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Collection; -import java.util.Objects; import java.util.function.Function; +import static java.util.Objects.requireNonNull; + public class EntitlementBootstrap { - public record PluginData(Path pluginPath, boolean isModular, boolean isExternalPlugin) {} + public record BootstrapArgs(Collection pluginData, Function, String> pluginResolver) { + public BootstrapArgs { + requireNonNull(pluginData); + requireNonNull(pluginResolver); + } + } - public record BootstrapArgs(Collection pluginData, Function, String> pluginResolver) {} + public record PluginData(Path pluginPath, boolean isModular, boolean isExternalPlugin) { + public PluginData { + requireNonNull(pluginPath); + } + } private static BootstrapArgs bootstrapArgs; @@ -50,9 +61,10 @@ public static void bootstrap(Collection pluginData, Function + * + * This serves two purposes: + * + *
    + *
  1. + * a smoke test to make sure the entitlements system is not completely broken, and + *
    2. + *
  2. + * an early test of certain important operations so they don't fail later on at an awkward time. + *
    3. + *
+ * + * @throws IllegalStateException if the entitlements system can't prevent an unauthorized action of our choosing + */ + private static void selfTest() { + ensureCannotStartProcess(); + ensureCanCreateTempFile(); + } + + private static void ensureCannotStartProcess() { + try { + // The command doesn't matter; it doesn't even need to exist + new ProcessBuilder("").start(); + } catch (NotEntitledException e) { + logger.debug("Success: Entitlement protection correctly prevented process creation"); + return; + } catch (IOException e) { + throw new IllegalStateException("Failed entitlement protection self-test", e); + } + throw new IllegalStateException("Entitlement protection self-test was incorrectly permitted"); + } + + /** + * Originally {@code Security.selfTest}. + */ + @SuppressForbidden(reason = "accesses jvm default tempdir as a self-test") + private static void ensureCanCreateTempFile() { + try { + Path p = Files.createTempFile(null, null); + p.toFile().deleteOnExit(); + + // Make an effort to clean up the file immediately; also, deleteOnExit leaves the file if the JVM exits abnormally. + try { + Files.delete(p); + } catch (IOException ignored) { + // Can be caused by virus scanner + } + } catch (NotEntitledException e) { + throw new IllegalStateException("Entitlement protection self-test was incorrectly forbidden", e); + } catch (Exception e) { + throw new IllegalStateException("Unable to perform entitlement protection self-test", e); + } + logger.debug("Success: Entitlement protection correctly permitted temp file creation"); + } + private static final Logger logger = LogManager.getLogger(EntitlementBootstrap.class); } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java index aded5344024d3..ba5ccbafa70ae 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -20,6 +20,7 @@ import org.elasticsearch.entitlement.instrumentation.Transformer; import org.elasticsearch.entitlement.runtime.api.ElasticsearchEntitlementChecker; import org.elasticsearch.entitlement.runtime.policy.CreateClassLoaderEntitlement; +import org.elasticsearch.entitlement.runtime.policy.Entitlement; import org.elasticsearch.entitlement.runtime.policy.ExitVMEntitlement; import org.elasticsearch.entitlement.runtime.policy.Policy; import org.elasticsearch.entitlement.runtime.policy.PolicyManager; @@ -93,9 +94,17 @@ private static PolicyManager createPolicyManager() throws IOException { // TODO(ES-10031): Decide what goes in the elasticsearch default policy and extend it var serverPolicy = new Policy( "server", - List.of(new Scope("org.elasticsearch.server", List.of(new ExitVMEntitlement(), new CreateClassLoaderEntitlement()))) + List.of( + new Scope("org.elasticsearch.base", List.of(new CreateClassLoaderEntitlement())), + new Scope("org.elasticsearch.xcontent", List.of(new CreateClassLoaderEntitlement())), + new Scope("org.elasticsearch.server", List.of(new ExitVMEntitlement(), new CreateClassLoaderEntitlement())) + ) ); - return new PolicyManager(serverPolicy, pluginPolicies, EntitlementBootstrap.bootstrapArgs().pluginResolver(), ENTITLEMENTS_MODULE); + // agents run without a module, so this is a special hack for the apm agent + // this should be removed once https://github.com/elastic/elasticsearch/issues/109335 is completed + List agentEntitlements = List.of(new CreateClassLoaderEntitlement()); + var resolver = EntitlementBootstrap.bootstrapArgs().pluginResolver(); + return new PolicyManager(serverPolicy, agentEntitlements, pluginPolicies, resolver, ENTITLEMENTS_MODULE); } private static Map createPluginPolicies(Collection pluginData) throws IOException { @@ -119,13 +128,13 @@ private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, Strin final Policy policy = parsePolicyIfExists(pluginName, policyFile, isExternalPlugin); // TODO: should this check actually be part of the parser? - for (Scope scope : policy.scopes) { - if (moduleNames.contains(scope.name) == false) { + for (Scope scope : policy.scopes()) { + if (moduleNames.contains(scope.moduleName()) == false) { throw new IllegalStateException( Strings.format( "Invalid module name in policy: plugin [%s] does not have module [%s]; available modules [%s]; policy file [%s]", pluginName, - scope.name, + scope.moduleName(), String.join(", ", moduleNames), policyFile ) diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java index 27bf9ea553d87..450786ee57d86 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java @@ -27,6 +27,7 @@ * The trampoline module loads this object via SPI. */ public class ElasticsearchEntitlementChecker implements EntitlementChecker { + private final PolicyManager policyManager; public ElasticsearchEntitlementChecker(PolicyManager policyManager) { @@ -43,6 +44,36 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) { policyManager.checkExitVM(callerClass); } + @Override + public void check$java_lang_ClassLoader$(Class callerClass) { + policyManager.checkCreateClassLoader(callerClass); + } + + @Override + public void check$java_lang_ClassLoader$(Class callerClass, ClassLoader parent) { + policyManager.checkCreateClassLoader(callerClass); + } + + @Override + public void check$java_lang_ClassLoader$(Class callerClass, String name, ClassLoader parent) { + policyManager.checkCreateClassLoader(callerClass); + } + + @Override + public void check$java_security_SecureClassLoader$(Class callerClass) { + policyManager.checkCreateClassLoader(callerClass); + } + + @Override + public void check$java_security_SecureClassLoader$(Class callerClass, ClassLoader parent) { + policyManager.checkCreateClassLoader(callerClass); + } + + @Override + public void check$java_security_SecureClassLoader$(Class callerClass, String name, ClassLoader parent) { + policyManager.checkCreateClassLoader(callerClass); + } + @Override public void check$java_net_URLClassLoader$(Class callerClass, URL[] urls) { policyManager.checkCreateClassLoader(callerClass); diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java index 138515be9ffcb..55e4b66595642 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java @@ -9,7 +9,7 @@ package org.elasticsearch.entitlement.runtime.policy; -public class CreateClassLoaderEntitlement implements Entitlement { +public record CreateClassLoaderEntitlement() implements Entitlement { @ExternalEntitlement - public CreateClassLoaderEntitlement() {} + public CreateClassLoaderEntitlement {} } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExitVMEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExitVMEntitlement.java index c4a8fc6833581..e5c836ea22b20 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExitVMEntitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExitVMEntitlement.java @@ -12,4 +12,4 @@ /** * Internal policy type (not-parseable -- not available to plugins). */ -public class ExitVMEntitlement implements Entitlement {} +public record ExitVMEntitlement() implements Entitlement {} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Policy.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Policy.java index e8bd7a3fff357..3546472f485fb 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Policy.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Policy.java @@ -9,38 +9,15 @@ package org.elasticsearch.entitlement.runtime.policy; -import java.util.Collections; import java.util.List; import java.util.Objects; /** * A holder for scoped entitlements. */ -public class Policy { - - public final String name; - public final List scopes; - +public record Policy(String name, List scopes) { public Policy(String name, List scopes) { this.name = Objects.requireNonNull(name); - this.scopes = Collections.unmodifiableList(Objects.requireNonNull(scopes)); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Policy policy = (Policy) o; - return Objects.equals(name, policy.name) && Objects.equals(scopes, policy.scopes); - } - - @Override - public int hashCode() { - return Objects.hash(name, scopes); - } - - @Override - public String toString() { - return "Policy{" + "name='" + name + '\'' + ", scopes=" + scopes + '}'; + this.scopes = List.copyOf(scopes); } } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java index 330c7e59c60c7..188ce1d747db6 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java @@ -17,34 +17,31 @@ import java.lang.StackWalker.StackFrame; import java.lang.module.ModuleFinder; import java.lang.module.ModuleReference; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.IdentityHashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import static java.lang.StackWalker.Option.RETAIN_CLASS_REFERENCE; import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.groupingBy; public class PolicyManager { private static final Logger logger = LogManager.getLogger(PolicyManager.class); - static class ModuleEntitlements { - public static final ModuleEntitlements NONE = new ModuleEntitlements(List.of()); - private final IdentityHashMap, List> entitlementsByType; + record ModuleEntitlements(Map, List> entitlementsByType) { + public static final ModuleEntitlements NONE = new ModuleEntitlements(Map.of()); - ModuleEntitlements(List entitlements) { - this.entitlementsByType = entitlements.stream() - .collect(Collectors.toMap(Entitlement::getClass, e -> new ArrayList<>(List.of(e)), (a, b) -> { - a.addAll(b); - return a; - }, IdentityHashMap::new)); + ModuleEntitlements { + entitlementsByType = Map.copyOf(entitlementsByType); + } + + public static ModuleEntitlements from(List entitlements) { + return new ModuleEntitlements(entitlements.stream().collect(groupingBy(Entitlement::getClass))); } public boolean hasEntitlement(Class entitlementClass) { @@ -56,9 +53,10 @@ public Stream getEntitlements(Class entitlementCla } } - final Map moduleEntitlementsMap = new HashMap<>(); + final Map moduleEntitlementsMap = new ConcurrentHashMap<>(); protected final Map> serverEntitlements; + protected final List agentEntitlements; protected final Map>> pluginsEntitlements; private final Function, String> pluginResolver; @@ -85,12 +83,14 @@ private static Set findSystemModules() { private final Module entitlementsModule; public PolicyManager( - Policy defaultPolicy, + Policy serverPolicy, + List agentEntitlements, Map pluginPolicies, Function, String> pluginResolver, Module entitlementsModule ) { - this.serverEntitlements = buildScopeEntitlementsMap(requireNonNull(defaultPolicy)); + this.serverEntitlements = buildScopeEntitlementsMap(requireNonNull(serverPolicy)); + this.agentEntitlements = agentEntitlements; this.pluginsEntitlements = requireNonNull(pluginPolicies).entrySet() .stream() .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e -> buildScopeEntitlementsMap(e.getValue()))); @@ -99,7 +99,7 @@ public PolicyManager( } private static Map> buildScopeEntitlementsMap(Policy policy) { - return policy.scopes.stream().collect(Collectors.toUnmodifiableMap(scope -> scope.name, scope -> scope.entitlements)); + return policy.scopes().stream().collect(Collectors.toUnmodifiableMap(scope -> scope.moduleName(), scope -> scope.entitlements())); } public void checkStartProcess(Class callerClass) { @@ -107,7 +107,7 @@ public void checkStartProcess(Class callerClass) { } private void neverEntitled(Class callerClass, String operationDescription) { - var requestingModule = requestingModule(callerClass); + var requestingModule = requestingClass(callerClass); if (isTriviallyAllowed(requestingModule)) { return; } @@ -139,18 +139,18 @@ public void checkSetGlobalHttpsConnectionProperties(Class callerClass) { } private void checkEntitlementPresent(Class callerClass, Class entitlementClass) { - var requestingModule = requestingModule(callerClass); - if (isTriviallyAllowed(requestingModule)) { + var requestingClass = requestingClass(callerClass); + if (isTriviallyAllowed(requestingClass)) { return; } - ModuleEntitlements entitlements = getEntitlementsOrThrow(callerClass, requestingModule); + ModuleEntitlements entitlements = getEntitlements(requestingClass); if (entitlements.hasEntitlement(entitlementClass)) { logger.debug( () -> Strings.format( - "Entitled: caller [%s], module [%s], type [%s]", - callerClass, - requestingModule.getName(), + "Entitled: class [%s], module [%s], entitlement [%s]", + requestingClass, + requestingClass.getModule().getName(), entitlementClass.getSimpleName() ) ); @@ -158,30 +158,26 @@ private void checkEntitlementPresent(Class callerClass, Class callerClass, Module requestingModule) { - ModuleEntitlements cachedEntitlement = moduleEntitlementsMap.get(requestingModule); - if (cachedEntitlement != null) { - if (cachedEntitlement == ModuleEntitlements.NONE) { - throw new NotEntitledException(buildModuleNoPolicyMessage(callerClass, requestingModule) + "[CACHED]"); - } - return cachedEntitlement; - } + ModuleEntitlements getEntitlements(Class requestingClass) { + return moduleEntitlementsMap.computeIfAbsent(requestingClass.getModule(), m -> computeEntitlements(requestingClass)); + } + private ModuleEntitlements computeEntitlements(Class requestingClass) { + Module requestingModule = requestingClass.getModule(); if (isServerModule(requestingModule)) { - var scopeName = requestingModule.getName(); - return getModuleEntitlementsOrThrow(callerClass, requestingModule, serverEntitlements, scopeName); + return getModuleScopeEntitlements(requestingClass, serverEntitlements, requestingModule.getName()); } // plugins - var pluginName = pluginResolver.apply(callerClass); + var pluginName = pluginResolver.apply(requestingClass); if (pluginName != null) { var pluginEntitlements = pluginsEntitlements.get(pluginName); if (pluginEntitlements != null) { @@ -191,34 +187,30 @@ ModuleEntitlements getEntitlementsOrThrow(Class callerClass, Module requestin } else { scopeName = requestingModule.getName(); } - return getModuleEntitlementsOrThrow(callerClass, requestingModule, pluginEntitlements, scopeName); + return getModuleScopeEntitlements(requestingClass, pluginEntitlements, scopeName); } } - moduleEntitlementsMap.put(requestingModule, ModuleEntitlements.NONE); - throw new NotEntitledException(buildModuleNoPolicyMessage(callerClass, requestingModule)); - } + if (requestingModule.isNamed() == false) { + // agents are the only thing running non-modular + return ModuleEntitlements.from(agentEntitlements); + } - private static String buildModuleNoPolicyMessage(Class callerClass, Module requestingModule) { - return Strings.format("Missing entitlement policy: caller [%s], module [%s]", callerClass, requestingModule.getName()); + logger.warn("No applicable entitlement policy for class [{}]", requestingClass.getName()); + return ModuleEntitlements.NONE; } - private ModuleEntitlements getModuleEntitlementsOrThrow( + private ModuleEntitlements getModuleScopeEntitlements( Class callerClass, - Module module, Map> scopeEntitlements, String moduleName ) { var entitlements = scopeEntitlements.get(moduleName); if (entitlements == null) { - // Module without entitlements - remember we don't have any - moduleEntitlementsMap.put(module, ModuleEntitlements.NONE); - throw new NotEntitledException(buildModuleNoPolicyMessage(callerClass, module)); + logger.warn("No applicable entitlement policy for module [{}], class [{}]", moduleName, callerClass); + return ModuleEntitlements.NONE; } - // We have a policy for this module - var classEntitlements = new ModuleEntitlements(entitlements); - moduleEntitlementsMap.put(module, classEntitlements); - return classEntitlements; + return ModuleEntitlements.from(entitlements); } private static boolean isServerModule(Module requestingModule) { @@ -226,25 +218,22 @@ private static boolean isServerModule(Module requestingModule) { } /** - * Walks the stack to determine which module's entitlements should be checked. + * Walks the stack to determine which class should be checked for entitlements. * - * @param callerClass when non-null will be used if its module is suitable; + * @param callerClass when non-null will be returned; * this is a fast-path check that can avoid the stack walk * in cases where the caller class is available. - * @return the requesting module, or {@code null} if the entire call stack + * @return the requesting class, or {@code null} if the entire call stack * comes from the entitlement library itself. */ - Module requestingModule(Class callerClass) { + Class requestingClass(Class callerClass) { if (callerClass != null) { - var callerModule = callerClass.getModule(); - if (callerModule != null && entitlementsModule.equals(callerModule) == false) { - // fast path - return callerModule; - } + // fast path + return callerClass; } - Optional module = StackWalker.getInstance(RETAIN_CLASS_REFERENCE) - .walk(frames -> findRequestingModule(frames.map(StackFrame::getDeclaringClass))); - return module.orElse(null); + Optional> result = StackWalker.getInstance(RETAIN_CLASS_REFERENCE) + .walk(frames -> findRequestingClass(frames.map(StackFrame::getDeclaringClass))); + return result.orElse(null); } /** @@ -253,33 +242,25 @@ Module requestingModule(Class callerClass) { * * @throws NullPointerException if the requesting module is {@code null} */ - Optional findRequestingModule(Stream> classes) { - return classes.map(Objects::requireNonNull) - .map(PolicyManager::moduleOf) - .filter(m -> m != entitlementsModule) // Ignore the entitlements library itself entirely - .skip(1) // Skip the sensitive method itself + Optional> findRequestingClass(Stream> classes) { + return classes.filter(c -> c.getModule() != entitlementsModule) // Ignore the entitlements library + .skip(1) // Skip the sensitive caller method .findFirst(); } - private static Module moduleOf(Class c) { - var result = c.getModule(); - if (result == null) { - throw new NullPointerException("Entitlements system does not support non-modular class [" + c.getName() + "]"); - } else { - return result; - } - } - - private static boolean isTriviallyAllowed(Module requestingModule) { + /** + * @return true if permission is granted regardless of the entitlement + */ + private static boolean isTriviallyAllowed(Class requestingClass) { if (logger.isTraceEnabled()) { logger.trace("Stack trace for upcoming trivially-allowed check", new Exception()); } - if (requestingModule == null) { + if (requestingClass == null) { logger.debug("Entitlement trivially allowed: no caller frames outside the entitlement library"); return true; } - if (systemModules.contains(requestingModule)) { - logger.debug("Entitlement trivially allowed from system module [{}]", requestingModule.getName()); + if (systemModules.contains(requestingClass.getModule())) { + logger.debug("Entitlement trivially allowed from system module [{}]", requestingClass.getModule().getName()); return true; } logger.trace("Entitlement not trivially allowed"); diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java index 0fe63eb8da1b7..55e257797d603 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java @@ -9,38 +9,17 @@ package org.elasticsearch.entitlement.runtime.policy; -import java.util.Collections; import java.util.List; import java.util.Objects; /** * A holder for entitlements within a single scope. */ -public class Scope { +public record Scope(String moduleName, List entitlements) { - public final String name; - public final List entitlements; - - public Scope(String name, List entitlements) { - this.name = Objects.requireNonNull(name); - this.entitlements = Collections.unmodifiableList(Objects.requireNonNull(entitlements)); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Scope scope = (Scope) o; - return Objects.equals(name, scope.name) && Objects.equals(entitlements, scope.entitlements); + public Scope(String moduleName, List entitlements) { + this.moduleName = Objects.requireNonNull(moduleName); + this.entitlements = List.copyOf(entitlements); } - @Override - public int hashCode() { - return Objects.hash(name, entitlements); - } - - @Override - public String toString() { - return "Scope{" + "name='" + name + '\'' + ", entitlements=" + entitlements + '}'; - } } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/SetHttpsConnectionPropertiesEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/SetHttpsConnectionPropertiesEntitlement.java index 6f165f27b31ff..bb2f65def9e18 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/SetHttpsConnectionPropertiesEntitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/SetHttpsConnectionPropertiesEntitlement.java @@ -12,7 +12,7 @@ /** * An Entitlement to allow setting properties to a single Https connection after this has been created */ -public class SetHttpsConnectionPropertiesEntitlement implements Entitlement { +public record SetHttpsConnectionPropertiesEntitlement() implements Entitlement { @ExternalEntitlement(esModulesOnly = false) - public SetHttpsConnectionPropertiesEntitlement() {} + public SetHttpsConnectionPropertiesEntitlement {} } diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java index 31e3e62f56bf5..d22c2f598e344 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java @@ -9,7 +9,7 @@ package org.elasticsearch.entitlement.runtime.policy; -import org.elasticsearch.entitlement.runtime.api.NotEntitledException; +import org.elasticsearch.entitlement.runtime.policy.PolicyManager.ModuleEntitlements; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.compiler.InMemoryJavaCompiler; import org.elasticsearch.test.jar.JarUtils; @@ -31,8 +31,6 @@ import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.endsWith; -import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.sameInstance; @@ -58,6 +56,7 @@ public static void beforeClass() { public void testGetEntitlementsThrowsOnMissingPluginUnnamedModule() { var policyManager = new PolicyManager( createEmptyTestServerPolicy(), + List.of(), Map.of("plugin1", createPluginPolicy("plugin.module")), c -> "plugin1", NO_ENTITLEMENTS_MODULE @@ -67,60 +66,44 @@ public void testGetEntitlementsThrowsOnMissingPluginUnnamedModule() { var callerClass = this.getClass(); var requestingModule = callerClass.getModule(); - var ex = assertThrows( - "No policy for the unnamed module", - NotEntitledException.class, - () -> policyManager.getEntitlementsOrThrow(callerClass, requestingModule) - ); + assertEquals("No policy for the unnamed module", ModuleEntitlements.NONE, policyManager.getEntitlements(callerClass)); - assertEquals( - "Missing entitlement policy: caller [class org.elasticsearch.entitlement.runtime.policy.PolicyManagerTests], module [null]", - ex.getMessage() - ); - assertThat(policyManager.moduleEntitlementsMap, hasEntry(requestingModule, PolicyManager.ModuleEntitlements.NONE)); + assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap); } public void testGetEntitlementsThrowsOnMissingPolicyForPlugin() { - var policyManager = new PolicyManager(createEmptyTestServerPolicy(), Map.of(), c -> "plugin1", NO_ENTITLEMENTS_MODULE); + var policyManager = new PolicyManager(createEmptyTestServerPolicy(), List.of(), Map.of(), c -> "plugin1", NO_ENTITLEMENTS_MODULE); // Any class from the current module (unnamed) will do var callerClass = this.getClass(); var requestingModule = callerClass.getModule(); - var ex = assertThrows( - "No policy for this plugin", - NotEntitledException.class, - () -> policyManager.getEntitlementsOrThrow(callerClass, requestingModule) - ); + assertEquals("No policy for this plugin", ModuleEntitlements.NONE, policyManager.getEntitlements(callerClass)); - assertEquals( - "Missing entitlement policy: caller [class org.elasticsearch.entitlement.runtime.policy.PolicyManagerTests], module [null]", - ex.getMessage() - ); - assertThat(policyManager.moduleEntitlementsMap, hasEntry(requestingModule, PolicyManager.ModuleEntitlements.NONE)); + assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap); } public void testGetEntitlementsFailureIsCached() { - var policyManager = new PolicyManager(createEmptyTestServerPolicy(), Map.of(), c -> "plugin1", NO_ENTITLEMENTS_MODULE); + var policyManager = new PolicyManager(createEmptyTestServerPolicy(), List.of(), Map.of(), c -> "plugin1", NO_ENTITLEMENTS_MODULE); // Any class from the current module (unnamed) will do var callerClass = this.getClass(); var requestingModule = callerClass.getModule(); - assertThrows(NotEntitledException.class, () -> policyManager.getEntitlementsOrThrow(callerClass, requestingModule)); - assertThat(policyManager.moduleEntitlementsMap, hasEntry(requestingModule, PolicyManager.ModuleEntitlements.NONE)); + assertEquals(ModuleEntitlements.NONE, policyManager.getEntitlements(callerClass)); + assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap); // A second time - var ex = assertThrows(NotEntitledException.class, () -> policyManager.getEntitlementsOrThrow(callerClass, requestingModule)); + assertEquals(ModuleEntitlements.NONE, policyManager.getEntitlements(callerClass)); - assertThat(ex.getMessage(), endsWith("[CACHED]")); // Nothing new in the map - assertThat(policyManager.moduleEntitlementsMap, aMapWithSize(1)); + assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap); } public void testGetEntitlementsReturnsEntitlementsForPluginUnnamedModule() { var policyManager = new PolicyManager( createEmptyTestServerPolicy(), + List.of(), Map.ofEntries(entry("plugin2", createPluginPolicy(ALL_UNNAMED))), c -> "plugin2", NO_ENTITLEMENTS_MODULE @@ -128,14 +111,13 @@ public void testGetEntitlementsReturnsEntitlementsForPluginUnnamedModule() { // Any class from the current module (unnamed) will do var callerClass = this.getClass(); - var requestingModule = callerClass.getModule(); - var entitlements = policyManager.getEntitlementsOrThrow(callerClass, requestingModule); + var entitlements = policyManager.getEntitlements(callerClass); assertThat(entitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true)); } public void testGetEntitlementsThrowsOnMissingPolicyForServer() throws ClassNotFoundException { - var policyManager = new PolicyManager(createTestServerPolicy("example"), Map.of(), c -> null, NO_ENTITLEMENTS_MODULE); + var policyManager = new PolicyManager(createTestServerPolicy("example"), List.of(), Map.of(), c -> null, NO_ENTITLEMENTS_MODULE); // Tests do not run modular, so we cannot use a server class. // But we know that in production code the server module and its classes are in the boot layer. @@ -144,21 +126,19 @@ public void testGetEntitlementsThrowsOnMissingPolicyForServer() throws ClassNotF var mockServerClass = ModuleLayer.boot().findLoader("jdk.httpserver").loadClass("com.sun.net.httpserver.HttpServer"); var requestingModule = mockServerClass.getModule(); - var ex = assertThrows( - "No policy for this module in server", - NotEntitledException.class, - () -> policyManager.getEntitlementsOrThrow(mockServerClass, requestingModule) - ); + assertEquals("No policy for this module in server", ModuleEntitlements.NONE, policyManager.getEntitlements(mockServerClass)); - assertEquals( - "Missing entitlement policy: caller [class com.sun.net.httpserver.HttpServer], module [jdk.httpserver]", - ex.getMessage() - ); - assertThat(policyManager.moduleEntitlementsMap, hasEntry(requestingModule, PolicyManager.ModuleEntitlements.NONE)); + assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap); } public void testGetEntitlementsReturnsEntitlementsForServerModule() throws ClassNotFoundException { - var policyManager = new PolicyManager(createTestServerPolicy("jdk.httpserver"), Map.of(), c -> null, NO_ENTITLEMENTS_MODULE); + var policyManager = new PolicyManager( + createTestServerPolicy("jdk.httpserver"), + List.of(), + Map.of(), + c -> null, + NO_ENTITLEMENTS_MODULE + ); // Tests do not run modular, so we cannot use a server class. // But we know that in production code the server module and its classes are in the boot layer. @@ -167,7 +147,7 @@ public void testGetEntitlementsReturnsEntitlementsForServerModule() throws Class var mockServerClass = ModuleLayer.boot().findLoader("jdk.httpserver").loadClass("com.sun.net.httpserver.HttpServer"); var requestingModule = mockServerClass.getModule(); - var entitlements = policyManager.getEntitlementsOrThrow(mockServerClass, requestingModule); + var entitlements = policyManager.getEntitlements(mockServerClass); assertThat(entitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true)); assertThat(entitlements.hasEntitlement(ExitVMEntitlement.class), is(true)); } @@ -179,6 +159,7 @@ public void testGetEntitlementsReturnsEntitlementsForPluginModule() throws IOExc var policyManager = new PolicyManager( createEmptyTestServerPolicy(), + List.of(), Map.of("mock-plugin", createPluginPolicy("org.example.plugin")), c -> "mock-plugin", NO_ENTITLEMENTS_MODULE @@ -188,7 +169,7 @@ public void testGetEntitlementsReturnsEntitlementsForPluginModule() throws IOExc var mockPluginClass = layer.findLoader("org.example.plugin").loadClass("q.B"); var requestingModule = mockPluginClass.getModule(); - var entitlements = policyManager.getEntitlementsOrThrow(mockPluginClass, requestingModule); + var entitlements = policyManager.getEntitlements(mockPluginClass); assertThat(entitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true)); assertThat( entitlements.getEntitlements(FileEntitlement.class).toList(), @@ -199,6 +180,7 @@ public void testGetEntitlementsReturnsEntitlementsForPluginModule() throws IOExc public void testGetEntitlementsResultIsCached() { var policyManager = new PolicyManager( createEmptyTestServerPolicy(), + List.of(), Map.ofEntries(entry("plugin2", createPluginPolicy(ALL_UNNAMED))), c -> "plugin2", NO_ENTITLEMENTS_MODULE @@ -206,22 +188,21 @@ public void testGetEntitlementsResultIsCached() { // Any class from the current module (unnamed) will do var callerClass = this.getClass(); - var requestingModule = callerClass.getModule(); - var entitlements = policyManager.getEntitlementsOrThrow(callerClass, requestingModule); + var entitlements = policyManager.getEntitlements(callerClass); assertThat(entitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true)); assertThat(policyManager.moduleEntitlementsMap, aMapWithSize(1)); var cachedResult = policyManager.moduleEntitlementsMap.values().stream().findFirst().get(); - var entitlementsAgain = policyManager.getEntitlementsOrThrow(callerClass, requestingModule); + var entitlementsAgain = policyManager.getEntitlements(callerClass); // Nothing new in the map assertThat(policyManager.moduleEntitlementsMap, aMapWithSize(1)); assertThat(entitlementsAgain, sameInstance(cachedResult)); } - public void testRequestingModuleFastPath() throws IOException, ClassNotFoundException { + public void testRequestingClassFastPath() throws IOException, ClassNotFoundException { var callerClass = makeClassInItsOwnModule(); - assertEquals(callerClass.getModule(), policyManagerWithEntitlementsModule(NO_ENTITLEMENTS_MODULE).requestingModule(callerClass)); + assertEquals(callerClass, policyManagerWithEntitlementsModule(NO_ENTITLEMENTS_MODULE).requestingClass(callerClass)); } public void testRequestingModuleWithStackWalk() throws IOException, ClassNotFoundException { @@ -232,24 +213,21 @@ public void testRequestingModuleWithStackWalk() throws IOException, ClassNotFoun var policyManager = policyManagerWithEntitlementsModule(entitlementsClass.getModule()); - var requestingModule = requestingClass.getModule(); - assertEquals( "Skip entitlement library and the instrumented method", - requestingModule, - policyManager.findRequestingModule(Stream.of(entitlementsClass, instrumentedClass, requestingClass, ignorableClass)) - .orElse(null) + requestingClass, + policyManager.findRequestingClass(Stream.of(entitlementsClass, instrumentedClass, requestingClass, ignorableClass)).orElse(null) ); assertEquals( "Skip multiple library frames", - requestingModule, - policyManager.findRequestingModule(Stream.of(entitlementsClass, entitlementsClass, instrumentedClass, requestingClass)) + requestingClass, + policyManager.findRequestingClass(Stream.of(entitlementsClass, entitlementsClass, instrumentedClass, requestingClass)) .orElse(null) ); assertThrows( "Non-modular caller frames are not supported", NullPointerException.class, - () -> policyManager.findRequestingModule(Stream.of(entitlementsClass, null)) + () -> policyManager.findRequestingClass(Stream.of(entitlementsClass, null)) ); } @@ -261,7 +239,7 @@ private static Class makeClassInItsOwnModule() throws IOException, ClassNotFo } private static PolicyManager policyManagerWithEntitlementsModule(Module entitlementsModule) { - return new PolicyManager(createEmptyTestServerPolicy(), Map.of(), c -> "test", entitlementsModule); + return new PolicyManager(createEmptyTestServerPolicy(), List.of(), Map.of(), c -> "test", entitlementsModule); } private static Policy createEmptyTestServerPolicy() { diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java index bee8767fcd900..4d17fc92e1578 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java @@ -16,11 +16,7 @@ import java.nio.charset.StandardCharsets; import java.util.List; -import static org.elasticsearch.test.LambdaMatchers.transformedMatch; -import static org.hamcrest.Matchers.both; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; public class PolicyParserTests extends ESTestCase { @@ -39,21 +35,21 @@ public void testGetEntitlementTypeName() { public void testPolicyBuilder() throws IOException { Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", false) .parsePolicy(); - Policy builtPolicy = new Policy( + Policy expected = new Policy( "test-policy.yaml", List.of(new Scope("entitlement-module-name", List.of(new FileEntitlement("test/path/to/file", List.of("read", "write"))))) ); - assertEquals(parsedPolicy, builtPolicy); + assertEquals(expected, parsedPolicy); } public void testPolicyBuilderOnExternalPlugin() throws IOException { Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", true) .parsePolicy(); - Policy builtPolicy = new Policy( + Policy expected = new Policy( "test-policy.yaml", List.of(new Scope("entitlement-module-name", List.of(new FileEntitlement("test/path/to/file", List.of("read", "write"))))) ); - assertEquals(parsedPolicy, builtPolicy); + assertEquals(expected, parsedPolicy); } public void testParseCreateClassloader() throws IOException { @@ -61,18 +57,11 @@ public void testParseCreateClassloader() throws IOException { entitlement-module-name: - create_class_loader """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", false).parsePolicy(); - Policy builtPolicy = new Policy( + Policy expected = new Policy( "test-policy.yaml", List.of(new Scope("entitlement-module-name", List.of(new CreateClassLoaderEntitlement()))) ); - assertThat( - parsedPolicy.scopes, - contains( - both(transformedMatch((Scope scope) -> scope.name, equalTo("entitlement-module-name"))).and( - transformedMatch(scope -> scope.entitlements, contains(instanceOf(CreateClassLoaderEntitlement.class))) - ) - ) - ); + assertEquals(expected, parsedPolicy); } public void testParseSetHttpsConnectionProperties() throws IOException { @@ -80,17 +69,10 @@ public void testParseSetHttpsConnectionProperties() throws IOException { entitlement-module-name: - set_https_connection_properties """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", true).parsePolicy(); - Policy builtPolicy = new Policy( + Policy expected = new Policy( "test-policy.yaml", - List.of(new Scope("entitlement-module-name", List.of(new CreateClassLoaderEntitlement()))) - ); - assertThat( - parsedPolicy.scopes, - contains( - both(transformedMatch((Scope scope) -> scope.name, equalTo("entitlement-module-name"))).and( - transformedMatch(scope -> scope.entitlements, contains(instanceOf(SetHttpsConnectionPropertiesEntitlement.class))) - ) - ) + List.of(new Scope("entitlement-module-name", List.of(new SetHttpsConnectionPropertiesEntitlement()))) ); + assertEquals(expected, parsedPolicy); } } diff --git a/modules/apm/src/main/plugin-metadata/entitlement-policy.yaml b/modules/apm/src/main/plugin-metadata/entitlement-policy.yaml index 30b2bd1978d1b..9c10bafca42f9 100644 --- a/modules/apm/src/main/plugin-metadata/entitlement-policy.yaml +++ b/modules/apm/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,2 +1,4 @@ +org.elasticsearch.telemetry.apm: + - create_class_loader elastic.apm.agent: - set_https_connection_properties diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java index 2c0d6d47e0233..3d08be1f24a42 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java @@ -241,7 +241,7 @@ public void setup() throws Exception { new MetadataFieldMapper[] { dtfm }, Collections.emptyMap() ); - MappingLookup mappingLookup = MappingLookup.fromMappers(mapping, List.of(dtfm, dateFieldMapper), List.of(), null); + MappingLookup mappingLookup = MappingLookup.fromMappers(mapping, List.of(dtfm, dateFieldMapper), List.of()); indicesService = DataStreamTestHelper.mockIndicesServices(mappingLookup); } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java index 1970883e91b3e..68b3ce279a89d 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Objects; import static org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationMetadata.DATABASE; @@ -91,6 +92,11 @@ protected Response(StreamInput in) throws IOException { this.databases = in.readCollectionAsList(DatabaseConfigurationMetadata::new); } + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeCollection(databases); + } + @Override protected List readNodesFrom(StreamInput in) throws IOException { return in.readCollectionAsList(NodeResponse::new); @@ -122,6 +128,63 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); return builder; } + + /* + * This implementation of equals exists solely for testing the serialization of this object. + */ + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return Objects.equals(databases, response.databases) + && Objects.equals(getClusterName(), response.getClusterName()) + && Objects.equals(equalsHashCodeFailures(), response.equalsHashCodeFailures()) + && Objects.equals(getNodes(), response.getNodes()) + && Objects.equals(equalsHashCodeNodesMap(), response.equalsHashCodeNodesMap()); + } + + /* + * This implementation of hashCode exists solely for testing the serialization of this object. + */ + @Override + public int hashCode() { + return Objects.hash(databases, getClusterName(), equalsHashCodeFailures(), getNodes(), equalsHashCodeNodesMap()); + } + + /* + * FailedNodeException does not implement equals or hashCode, making it difficult to test the serialization of this class. This + * helper method wraps the failures() list with a class that does implement equals and hashCode. + */ + private List equalsHashCodeFailures() { + return failures().stream().map(EqualsHashCodeFailedNodeException::new).toList(); + } + + private record EqualsHashCodeFailedNodeException(FailedNodeException failedNodeException) { + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (o == null || getClass() != o.getClass()) return false; + EqualsHashCodeFailedNodeException other = (EqualsHashCodeFailedNodeException) o; + return Objects.equals(failedNodeException.nodeId(), other.failedNodeException.nodeId()) + && Objects.equals(failedNodeException.getMessage(), other.failedNodeException.getMessage()); + } + + @Override + public int hashCode() { + return Objects.hash(failedNodeException.nodeId(), failedNodeException.getMessage()); + } + } + + /* + * The getNodesMap method changes the value of the nodesMap, causing failures when testing the concurrent serialization and + * deserialization of this class. Since this is a response object, we do not actually care about concurrency since it will not + * happen in practice. So this helper method synchronizes access to getNodesMap, which can be used from equals and hashCode for + * tests. + */ + private synchronized Map equalsHashCodeNodesMap() { + return getNodesMap(); + } } public static class NodeRequest extends TransportRequest { @@ -186,6 +249,7 @@ public List getDatabases() { @Override public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); out.writeCollection(databases); } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationActionNodeResponseTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationActionNodeResponseTests.java new file mode 100644 index 0000000000000..12fb08a5a1abf --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationActionNodeResponseTests.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; +import java.util.List; + +import static java.util.Collections.emptySet; + +public class GetDatabaseConfigurationActionNodeResponseTests extends AbstractWireSerializingTestCase< + GetDatabaseConfigurationAction.NodeResponse> { + @Override + protected Writeable.Reader instanceReader() { + return GetDatabaseConfigurationAction.NodeResponse::new; + } + + @Override + protected GetDatabaseConfigurationAction.NodeResponse createTestInstance() { + return getRandomDatabaseConfigurationActionNodeResponse(); + } + + static GetDatabaseConfigurationAction.NodeResponse getRandomDatabaseConfigurationActionNodeResponse() { + return new GetDatabaseConfigurationAction.NodeResponse(randomDiscoveryNode(), getRandomDatabaseConfigurationMetadata()); + } + + private static DiscoveryNode randomDiscoveryNode() { + return DiscoveryNodeUtils.builder(randomAlphaOfLength(6)).roles(emptySet()).build(); + } + + static List getRandomDatabaseConfigurationMetadata() { + return randomList( + 0, + 20, + () -> new DatabaseConfigurationMetadata( + new DatabaseConfiguration( + randomAlphaOfLength(20), + randomAlphaOfLength(20), + randomFrom( + List.of( + new DatabaseConfiguration.Local(randomAlphaOfLength(10)), + new DatabaseConfiguration.Web(), + new DatabaseConfiguration.Ipinfo(), + new DatabaseConfiguration.Maxmind(randomAlphaOfLength(10)) + ) + ) + ), + randomNonNegativeLong(), + randomNonNegativeLong() + ) + ); + } + + @Override + protected GetDatabaseConfigurationAction.NodeResponse mutateInstance(GetDatabaseConfigurationAction.NodeResponse instance) + throws IOException { + return null; + } + + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry( + List.of( + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Maxmind.NAME, + DatabaseConfiguration.Maxmind::new + ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Ipinfo.NAME, + DatabaseConfiguration.Ipinfo::new + ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Local.NAME, + DatabaseConfiguration.Local::new + ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Web.NAME, + DatabaseConfiguration.Web::new + ) + ) + ); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationActionResponseTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationActionResponseTests.java new file mode 100644 index 0000000000000..1b48a409d7876 --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationActionResponseTests.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; +import java.util.List; + +public class GetDatabaseConfigurationActionResponseTests extends AbstractWireSerializingTestCase { + @Override + protected Writeable.Reader instanceReader() { + return GetDatabaseConfigurationAction.Response::new; + } + + @Override + protected GetDatabaseConfigurationAction.Response createTestInstance() { + return new GetDatabaseConfigurationAction.Response( + GetDatabaseConfigurationActionNodeResponseTests.getRandomDatabaseConfigurationMetadata(), + getTestClusterName(), + getTestNodeResponses(), + getTestFailedNodeExceptions() + ); + } + + @Override + protected GetDatabaseConfigurationAction.Response mutateInstance(GetDatabaseConfigurationAction.Response instance) throws IOException { + return null; + } + + private ClusterName getTestClusterName() { + return new ClusterName(randomAlphaOfLength(30)); + } + + private List getTestNodeResponses() { + return randomList(0, 20, GetDatabaseConfigurationActionNodeResponseTests::getRandomDatabaseConfigurationActionNodeResponse); + } + + private List getTestFailedNodeExceptions() { + return randomList( + 0, + 5, + () -> new FailedNodeException( + randomAlphaOfLength(10), + randomAlphaOfLength(20), + new ElasticsearchException(randomAlphaOfLength(10)) + ) + ); + } + + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry( + List.of( + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Maxmind.NAME, + DatabaseConfiguration.Maxmind::new + ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Ipinfo.NAME, + DatabaseConfiguration.Ipinfo::new + ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Local.NAME, + DatabaseConfiguration.Local::new + ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Web.NAME, + DatabaseConfiguration.Web::new + ) + ) + ); + } +} diff --git a/modules/lang-expression/src/main/plugin-metadata/entitlement-policy.yaml b/modules/lang-expression/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..b05e6e3a7bf7c --- /dev/null +++ b/modules/lang-expression/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,2 @@ +org.elasticsearch.script.expression: + - create_class_loader diff --git a/modules/lang-painless/src/main/plugin-metadata/entitlement-policy.yaml b/modules/lang-painless/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..d7e4ad872fc32 --- /dev/null +++ b/modules/lang-painless/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,2 @@ +org.elasticsearch.painless: + - create_class_loader diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt index a5118db4876cb..e76db7cfb1d26 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt @@ -50,7 +50,5 @@ static_import { double cosineSimilarity(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$CosineSimilarity double dotProduct(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$DotProduct double hamming(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$Hamming - double maxSimDotProduct(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.RankVectorsScoreScriptUtils$MaxSimDotProduct - double maxSimInvHamming(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.RankVectorsScoreScriptUtils$MaxSimInvHamming } diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java index edcf03580da09..3ed492881afa9 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java @@ -145,13 +145,11 @@ public void compareAndExchangeRegister( BytesReference updated, ActionListener listener ) { - if (skipCas(listener)) return; ActionListener.completeWith(listener, () -> blobStore.compareAndExchangeRegister(buildKey(key), path, key, expected, updated)); } @Override public void getRegister(OperationPurpose purpose, String key, ActionListener listener) { - if (skipCas(listener)) return; ActionListener.completeWith(listener, () -> blobStore.getRegister(buildKey(key), path, key)); } } diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java index a53ec71f66376..5700fa6de63fa 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java @@ -59,9 +59,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import static fixture.gcs.GoogleCloudStorageHttpHandler.getContentRangeEnd; -import static fixture.gcs.GoogleCloudStorageHttpHandler.getContentRangeLimit; -import static fixture.gcs.GoogleCloudStorageHttpHandler.getContentRangeStart; import static fixture.gcs.GoogleCloudStorageHttpHandler.parseMultipartRequestBody; import static fixture.gcs.TestUtils.createServiceAccount; import static java.nio.charset.StandardCharsets.UTF_8; @@ -369,14 +366,14 @@ public void testWriteLargeBlob() throws IOException { assertThat(Math.toIntExact(requestBody.length()), anyOf(equalTo(defaultChunkSize), equalTo(lastChunkSize))); - final int rangeStart = getContentRangeStart(range); - final int rangeEnd = getContentRangeEnd(range); + final HttpHeaderParser.ContentRange contentRange = HttpHeaderParser.parseContentRangeHeader(range); + final int rangeStart = Math.toIntExact(contentRange.start()); + final int rangeEnd = Math.toIntExact(contentRange.end()); assertThat(rangeEnd + 1 - rangeStart, equalTo(Math.toIntExact(requestBody.length()))); assertThat(new BytesArray(data, rangeStart, rangeEnd - rangeStart + 1), is(requestBody)); bytesReceived.updateAndGet(existing -> Math.max(existing, rangeEnd)); - final Integer limit = getContentRangeLimit(range); - if (limit != null) { + if (contentRange.size() != null) { exchange.sendResponseHeaders(RestStatus.OK.getStatus(), -1); return; } else { diff --git a/muted-tests.yml b/muted-tests.yml index fc93134be9c99..e36ccb56614a9 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -213,8 +213,6 @@ tests: - class: org.elasticsearch.cluster.service.MasterServiceTests method: testThreadContext issue: https://github.com/elastic/elasticsearch/issues/118914 -- class: org.elasticsearch.repositories.blobstore.testkit.analyze.SecureHdfsRepositoryAnalysisRestIT - issue: https://github.com/elastic/elasticsearch/issues/118970 - class: org.elasticsearch.aggregations.bucket.SearchCancellationIT method: testCancellationDuringTimeSeriesAggregation issue: https://github.com/elastic/elasticsearch/issues/118992 @@ -226,37 +224,56 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/116777 - class: org.elasticsearch.xpack.security.authc.ldap.ActiveDirectoryRunAsIT issue: https://github.com/elastic/elasticsearch/issues/115727 -- class: org.elasticsearch.xpack.security.authc.kerberos.KerberosAuthenticationIT - issue: https://github.com/elastic/elasticsearch/issues/118414 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/search/search-your-data/retrievers-examples/line_98} issue: https://github.com/elastic/elasticsearch/issues/119155 - class: org.elasticsearch.xpack.esql.action.EsqlNodeFailureIT method: testFailureLoadingFields issue: https://github.com/elastic/elasticsearch/issues/118000 -- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT - method: test {yaml=indices.create/20_synthetic_source/create index with use_synthetic_source} - issue: https://github.com/elastic/elasticsearch/issues/119191 - class: org.elasticsearch.index.mapper.AbstractShapeGeometryFieldMapperTests method: testCartesianBoundsBlockLoader issue: https://github.com/elastic/elasticsearch/issues/119201 - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header} issue: https://github.com/elastic/elasticsearch/issues/119332 -- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT - method: test {lookup-join.MvJoinKeyOnTheDataNode ASYNC} - issue: https://github.com/elastic/elasticsearch/issues/119179 -- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT - issue: https://github.com/elastic/elasticsearch/issues/119191 -- class: org.elasticsearch.xpack.logsdb.qa.LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT - method: testEsqlTermsAggregationByMethod - issue: https://github.com/elastic/elasticsearch/issues/119355 -- class: org.elasticsearch.xpack.logsdb.qa.LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT - method: testMatchAllQuery - issue: https://github.com/elastic/elasticsearch/issues/119432 -- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT - method: test {lookup-join.MvJoinKeyOnTheDataNode SYNC} - issue: https://github.com/elastic/elasticsearch/issues/119446 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=transform/transforms_start_stop/Test start/stop/start transform} + issue: https://github.com/elastic/elasticsearch/issues/119508 +- class: org.elasticsearch.smoketest.MlWithSecurityIT + method: test {yaml=ml/sparse_vector_search/Test sparse_vector search with query vector and pruning config} + issue: https://github.com/elastic/elasticsearch/issues/119548 +- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT + method: testSearchableSnapshotUpgrade {p0=[9.0.0, 8.18.0, 8.18.0]} + issue: https://github.com/elastic/elasticsearch/issues/119549 +- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT + method: testMountSearchableSnapshot {p0=[9.0.0, 8.18.0, 8.18.0]} + issue: https://github.com/elastic/elasticsearch/issues/119550 +- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT + method: testMountSearchableSnapshot {p0=[9.0.0, 9.0.0, 8.18.0]} + issue: https://github.com/elastic/elasticsearch/issues/119551 +- class: org.elasticsearch.index.engine.LuceneSyntheticSourceChangesSnapshotTests + method: testSkipNonRootOfNestedDocuments + issue: https://github.com/elastic/elasticsearch/issues/119553 +- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT + method: testSearchableSnapshotUpgrade {p0=[9.0.0, 9.0.0, 8.18.0]} + issue: https://github.com/elastic/elasticsearch/issues/119560 +- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT + method: testMountSearchableSnapshot {p0=[9.0.0, 9.0.0, 9.0.0]} + issue: https://github.com/elastic/elasticsearch/issues/119561 +- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT + method: testSearchableSnapshotUpgrade {p0=[9.0.0, 9.0.0, 9.0.0]} + issue: https://github.com/elastic/elasticsearch/issues/119562 +- class: org.elasticsearch.xpack.ml.integration.ForecastIT + method: testOverflowToDisk + issue: https://github.com/elastic/elasticsearch/issues/117740 +- class: org.elasticsearch.xpack.security.authc.ldap.MultiGroupMappingIT + issue: https://github.com/elastic/elasticsearch/issues/119599 +- class: org.elasticsearch.lucene.FullClusterRestartSearchableSnapshotIndexCompatibilityIT + method: testSearchableSnapshotUpgrade {p0=8.18.0} + issue: https://github.com/elastic/elasticsearch/issues/119631 +- class: org.elasticsearch.lucene.FullClusterRestartSearchableSnapshotIndexCompatibilityIT + method: testSearchableSnapshotUpgrade {p0=9.0.0} + issue: https://github.com/elastic/elasticsearch/issues/119632 # Examples: # diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractIndexCompatibilityTestCase.java similarity index 58% rename from qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java rename to qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractIndexCompatibilityTestCase.java index 1865da06e20c5..8c9a42dc926e9 100644 --- a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractIndexCompatibilityTestCase.java @@ -9,29 +9,33 @@ package org.elasticsearch.lucene; -import com.carrotsearch.randomizedtesting.TestMethodAndParams; -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; - +import org.apache.http.entity.ContentType; +import org.apache.http.entity.InputStreamEntity; import org.elasticsearch.client.Request; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Strings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.cluster.util.Version; import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xcontent.XContentType; +import org.junit.After; import org.junit.Before; import org.junit.ClassRule; import org.junit.rules.RuleChain; import org.junit.rules.TemporaryFolder; import org.junit.rules.TestRule; -import java.util.Comparator; +import java.io.IOException; +import java.util.HashMap; import java.util.Locale; +import java.util.Map; import java.util.stream.IntStream; -import java.util.stream.Stream; import static org.elasticsearch.test.cluster.util.Version.CURRENT; import static org.elasticsearch.test.cluster.util.Version.fromString; @@ -41,24 +45,21 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -/** - * Test suite for Lucene indices backward compatibility with N-2 versions. The test suite creates a cluster in N-2 version, then upgrades it - * to N-1 version and finally upgrades it to the current version. Test methods are executed after each upgrade. - */ -@TestCaseOrdering(AbstractLuceneIndexCompatibilityTestCase.TestCaseOrdering.class) -public abstract class AbstractLuceneIndexCompatibilityTestCase extends ESRestTestCase { +public abstract class AbstractIndexCompatibilityTestCase extends ESRestTestCase { protected static final Version VERSION_MINUS_2 = fromString(System.getProperty("tests.minimum.index.compatible")); protected static final Version VERSION_MINUS_1 = fromString(System.getProperty("tests.minimum.wire.compatible")); protected static final Version VERSION_CURRENT = CURRENT; - protected static TemporaryFolder REPOSITORY_PATH = new TemporaryFolder(); + protected static final int NODES = 3; + + private static TemporaryFolder REPOSITORY_PATH = new TemporaryFolder(); protected static LocalClusterConfigProvider clusterConfig = c -> {}; private static ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) .version(VERSION_MINUS_2) - .nodes(2) + .nodes(NODES) .setting("path.repo", () -> REPOSITORY_PATH.getRoot().getPath()) .setting("xpack.security.enabled", "false") .setting("xpack.ml.enabled", "false") @@ -71,15 +72,44 @@ public abstract class AbstractLuceneIndexCompatibilityTestCase extends ESRestTes private static boolean upgradeFailed = false; - private final Version clusterVersion; + @Before + public final void maybeUpgradeBeforeTest() throws Exception { + // We want to use this test suite for the V9 upgrade, but we are not fully committed to necessarily having N-2 support + // in V10, so we add a check here to ensure we'll revisit this decision once V10 exists. + assertThat("Explicit check that N-2 version is Elasticsearch 7", VERSION_MINUS_2.getMajor(), equalTo(7)); - public AbstractLuceneIndexCompatibilityTestCase(@Name("cluster") Version clusterVersion) { - this.clusterVersion = clusterVersion; + if (upgradeFailed == false) { + try { + maybeUpgrade(); + } catch (Exception e) { + upgradeFailed = true; + throw e; + } + } + + // Skip remaining tests if upgrade failed + assumeFalse("Cluster upgrade failed", upgradeFailed); } - @ParametersFactory - public static Iterable parameters() { - return Stream.of(VERSION_MINUS_2, VERSION_MINUS_1, CURRENT).map(v -> new Object[] { v }).toList(); + protected abstract void maybeUpgrade() throws Exception; + + @After + public final void deleteSnapshotBlobCache() throws IOException { + // TODO ES-10475: The .snapshot-blob-cache created in legacy version can block upgrades, we should probably delete it automatically + try { + var request = new Request("DELETE", "/.snapshot-blob-cache"); + request.setOptions( + expectWarnings( + "this request accesses system indices: [.snapshot-blob-cache], but in a future major version, " + + "direct access to system indices will be prevented by default" + ) + ); + adminClient().performRequest(request); + } catch (IOException e) { + if (isNotFoundResponseException(e) == false) { + throw e; + } + } } @Override @@ -92,26 +122,8 @@ protected boolean preserveClusterUponCompletion() { return true; } - @Before - public void maybeUpgrade() throws Exception { - // We want to use this test suite for the V9 upgrade, but we are not fully committed to necessarily having N-2 support - // in V10, so we add a check here to ensure we'll revisit this decision once V10 exists. - assertThat("Explicit check that N-2 version is Elasticsearch 7", VERSION_MINUS_2.getMajor(), equalTo(7)); - - var currentVersion = clusterVersion(); - if (currentVersion.before(clusterVersion)) { - try { - cluster.upgradeToVersion(clusterVersion); - closeClients(); - initClient(); - } catch (Exception e) { - upgradeFailed = true; - throw e; - } - } - - // Skip remaining tests if upgrade failed - assumeFalse("Cluster upgrade failed", upgradeFailed); + protected ElasticsearchCluster cluster() { + return cluster; } protected String suffix(String name) { @@ -124,12 +136,18 @@ protected Settings repositorySettings() { .build(); } - protected static Version clusterVersion() throws Exception { - var response = assertOK(client().performRequest(new Request("GET", "/"))); - var responseBody = createFromResponse(response); - var version = Version.fromString(responseBody.evaluate("version.number").toString()); - assertThat("Failed to retrieve cluster version", version, notNullValue()); - return version; + protected static Map nodesVersions() throws Exception { + var nodesInfos = getNodesInfo(adminClient()); + assertThat(nodesInfos.size(), equalTo(NODES)); + var versions = new HashMap(); + for (var nodeInfos : nodesInfos.values()) { + versions.put((String) nodeInfos.get("name"), Version.fromString((String) nodeInfos.get("version"))); + } + return versions; + } + + protected static boolean isFullyUpgradedTo(Version version) throws Exception { + return nodesVersions().values().stream().allMatch(v -> v.equals(version)); } protected static Version indexVersion(String indexName) throws Exception { @@ -142,9 +160,9 @@ protected static void indexDocs(String indexName, int numDocs) throws Exception var request = new Request("POST", "/_bulk"); var docs = new StringBuilder(); IntStream.range(0, numDocs).forEach(n -> docs.append(Strings.format(""" - {"index":{"_id":"%s","_index":"%s"}} - {"test":"test"} - """, n, indexName))); + {"index":{"_index":"%s"}} + {"field_0":"%s","field_1":%d,"field_2":"%s"} + """, indexName, Integer.toString(n), n, randomFrom(Locale.getAvailableLocales()).getDisplayName()))); request.setJsonEntity(docs.toString()); var response = assertOK(client().performRequest(request)); assertThat(entityAsMap(response).get("errors"), allOf(notNullValue(), is(false))); @@ -182,15 +200,42 @@ protected static void restoreIndex(String repository, String snapshot, String in assertThat(responseBody.evaluate("snapshot.shards.successful"), equalTo(0)); } - /** - * Execute the test suite with the parameters provided by the {@link #parameters()} in version order. - */ - public static class TestCaseOrdering implements Comparator { - @Override - public int compare(TestMethodAndParams o1, TestMethodAndParams o2) { - var version1 = (Version) o1.getInstanceArguments().get(0); - var version2 = (Version) o2.getInstanceArguments().get(0); - return version1.compareTo(version2); + protected static void updateRandomIndexSettings(String indexName) throws IOException { + final var settings = Settings.builder(); + int updates = randomIntBetween(1, 3); + for (int i = 0; i < updates; i++) { + switch (i) { + case 0 -> settings.putList(IndexSettings.DEFAULT_FIELD_SETTING.getKey(), "field_" + randomInt(2)); + case 1 -> settings.put(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), randomIntBetween(1, 100)); + case 2 -> settings.put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), randomLongBetween(0L, 1000L)); + case 3 -> settings.put(IndexSettings.MAX_SLICES_PER_SCROLL.getKey(), randomIntBetween(1, 1024)); + default -> throw new IllegalStateException(); + } } + updateIndexSettings(indexName, settings); + } + + protected static void updateRandomMappings(String indexName) throws IOException { + final var runtime = new HashMap<>(); + runtime.put("field_" + randomInt(2), Map.of("type", "keyword")); + final var properties = new HashMap<>(); + properties.put(randomIdentifier(), Map.of("type", "long")); + var body = XContentTestUtils.convertToXContent(Map.of("runtime", runtime, "properties", properties), XContentType.JSON); + var request = new Request("PUT", indexName + "/_mappings"); + request.setEntity( + new InputStreamEntity( + body.streamInput(), + body.length(), + + ContentType.create(XContentType.JSON.mediaTypeWithoutParameters()) + ) + ); + assertOK(client().performRequest(request)); + } + + protected static boolean isIndexClosed(String indexName) throws Exception { + var responseBody = createFromResponse(client().performRequest(new Request("GET", "_cluster/state/metadata/" + indexName))); + var state = responseBody.evaluate("metadata.indices." + indexName + ".state"); + return IndexMetadata.State.fromString((String) state) == IndexMetadata.State.CLOSE; } } diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartIndexCompatibilityTestCase.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartIndexCompatibilityTestCase.java new file mode 100644 index 0000000000000..9ca7132493ae6 --- /dev/null +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartIndexCompatibilityTestCase.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.lucene; + +import com.carrotsearch.randomizedtesting.TestMethodAndParams; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; + +import org.elasticsearch.test.cluster.util.Version; + +import java.util.Comparator; +import java.util.stream.Stream; + +import static org.elasticsearch.test.cluster.util.Version.CURRENT; +import static org.hamcrest.Matchers.equalTo; + +/** + * Test suite for Lucene indices backward compatibility with N-2 versions after full cluster restart upgrades. The test suite creates a + * cluster in N-2 version, then upgrades it to N-1 version and finally upgrades it to the current version. Test methods are executed after + * each upgrade. + */ +@TestCaseOrdering(FullClusterRestartIndexCompatibilityTestCase.TestCaseOrdering.class) +public abstract class FullClusterRestartIndexCompatibilityTestCase extends AbstractIndexCompatibilityTestCase { + + private final Version clusterVersion; + + public FullClusterRestartIndexCompatibilityTestCase(@Name("cluster") Version clusterVersion) { + this.clusterVersion = clusterVersion; + } + + @ParametersFactory + public static Iterable parameters() { + return Stream.of(VERSION_MINUS_2, VERSION_MINUS_1, CURRENT).map(v -> new Object[] { v }).toList(); + } + + @Override + protected void maybeUpgrade() throws Exception { + if (nodesVersions().values().stream().anyMatch(version -> version.before(clusterVersion))) { + cluster().upgradeToVersion(clusterVersion); + closeClients(); + initClient(); + } + assertThat(isFullyUpgradedTo(clusterVersion), equalTo(true)); + } + + /** + * Execute the test suite with the parameters provided by the {@link #parameters()} in version order. + */ + public static class TestCaseOrdering implements Comparator { + @Override + public int compare(TestMethodAndParams o1, TestMethodAndParams o2) { + var version1 = (Version) o1.getInstanceArguments().get(0); + var version2 = (Version) o2.getInstanceArguments().get(0); + return version1.compareTo(version2); + } + } +} diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartLuceneIndexCompatibilityIT.java similarity index 92% rename from qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java rename to qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartLuceneIndexCompatibilityIT.java index 655e30f069f18..15d41cc981cea 100644 --- a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartLuceneIndexCompatibilityIT.java @@ -23,13 +23,13 @@ import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.equalTo; -public class LuceneCompatibilityIT extends AbstractLuceneIndexCompatibilityTestCase { +public class FullClusterRestartLuceneIndexCompatibilityIT extends FullClusterRestartIndexCompatibilityTestCase { static { clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial"); } - public LuceneCompatibilityIT(Version version) { + public FullClusterRestartLuceneIndexCompatibilityIT(Version version) { super(version); } @@ -42,7 +42,7 @@ public void testRestoreIndex() throws Exception { final String index = suffix("index"); final int numDocs = 1234; - if (VERSION_MINUS_2.equals(clusterVersion())) { + if (isFullyUpgradedTo(VERSION_MINUS_2)) { logger.debug("--> registering repository [{}]", repository); registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings()); @@ -65,7 +65,7 @@ public void testRestoreIndex() throws Exception { return; } - if (VERSION_MINUS_1.equals(clusterVersion())) { + if (isFullyUpgradedTo(VERSION_MINUS_1)) { ensureGreen(index); assertThat(indexVersion(index), equalTo(VERSION_MINUS_2)); @@ -76,7 +76,7 @@ public void testRestoreIndex() throws Exception { return; } - if (VERSION_CURRENT.equals(clusterVersion())) { + if (isFullyUpgradedTo(VERSION_CURRENT)) { var restoredIndex = suffix("index-restored"); logger.debug("--> restoring index [{}] as [{}]", index, restoredIndex); diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/SearchableSnapshotCompatibilityIT.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartSearchableSnapshotIndexCompatibilityIT.java similarity index 71% rename from qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/SearchableSnapshotCompatibilityIT.java rename to qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartSearchableSnapshotIndexCompatibilityIT.java index d5db17f257b0c..a7dc5e41fd327 100644 --- a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/SearchableSnapshotCompatibilityIT.java +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartSearchableSnapshotIndexCompatibilityIT.java @@ -17,7 +17,7 @@ import static org.hamcrest.Matchers.equalTo; -public class SearchableSnapshotCompatibilityIT extends AbstractLuceneIndexCompatibilityTestCase { +public class FullClusterRestartSearchableSnapshotIndexCompatibilityIT extends FullClusterRestartIndexCompatibilityTestCase { static { clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial") @@ -25,7 +25,7 @@ public class SearchableSnapshotCompatibilityIT extends AbstractLuceneIndexCompat .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB"); } - public SearchableSnapshotCompatibilityIT(Version version) { + public FullClusterRestartSearchableSnapshotIndexCompatibilityIT(Version version) { super(version); } @@ -38,7 +38,7 @@ public void testSearchableSnapshot() throws Exception { final String index = suffix("index"); final int numDocs = 1234; - if (VERSION_MINUS_2.equals(clusterVersion())) { + if (isFullyUpgradedTo(VERSION_MINUS_2)) { logger.debug("--> registering repository [{}]", repository); registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings()); @@ -61,7 +61,7 @@ public void testSearchableSnapshot() throws Exception { return; } - if (VERSION_MINUS_1.equals(clusterVersion())) { + if (isFullyUpgradedTo(VERSION_MINUS_1)) { ensureGreen(index); assertThat(indexVersion(index), equalTo(VERSION_MINUS_2)); @@ -72,19 +72,38 @@ public void testSearchableSnapshot() throws Exception { return; } - if (VERSION_CURRENT.equals(clusterVersion())) { + if (isFullyUpgradedTo(VERSION_CURRENT)) { var mountedIndex = suffix("index-mounted"); logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex); mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex); - ensureGreen(mountedIndex); assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2)); assertDocCount(client(), mountedIndex, numDocs); + updateRandomIndexSettings(mountedIndex); + updateRandomMappings(mountedIndex); + logger.debug("--> adding replica to test peer-recovery"); updateIndexSettings(mountedIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)); ensureGreen(mountedIndex); + + logger.debug("--> closing index [{}]", mountedIndex); + closeIndex(mountedIndex); + ensureGreen(mountedIndex); + + logger.debug("--> adding replica to test peer-recovery for closed shards"); + updateIndexSettings(mountedIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2)); + ensureGreen(mountedIndex); + + logger.debug("--> re-opening index [{}]", mountedIndex); + openIndex(mountedIndex); + ensureGreen(mountedIndex); + + assertDocCount(client(), mountedIndex, numDocs); + + logger.debug("--> deleting index [{}]", mountedIndex); + deleteIndex(mountedIndex); } } @@ -98,7 +117,7 @@ public void testSearchableSnapshotUpgrade() throws Exception { final String index = suffix("index"); final int numDocs = 4321; - if (VERSION_MINUS_2.equals(clusterVersion())) { + if (isFullyUpgradedTo(VERSION_MINUS_2)) { logger.debug("--> registering repository [{}]", repository); registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings()); @@ -124,25 +143,47 @@ public void testSearchableSnapshotUpgrade() throws Exception { return; } - if (VERSION_MINUS_1.equals(clusterVersion())) { + if (isFullyUpgradedTo(VERSION_MINUS_1)) { logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex); mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex); ensureGreen(mountedIndex); + updateRandomIndexSettings(mountedIndex); + updateRandomMappings(mountedIndex); + assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2)); assertDocCount(client(), mountedIndex, numDocs); + + logger.debug("--> adding replica to test replica upgrade"); + updateIndexSettings(mountedIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)); + ensureGreen(mountedIndex); + + if (randomBoolean()) { + logger.debug("--> random closing of index [{}] before upgrade", mountedIndex); + closeIndex(mountedIndex); + ensureGreen(mountedIndex); + } return; } - if (VERSION_CURRENT.equals(clusterVersion())) { + if (isFullyUpgradedTo(VERSION_CURRENT)) { ensureGreen(mountedIndex); + if (isIndexClosed(mountedIndex)) { + logger.debug("--> re-opening index [{}] after upgrade", mountedIndex); + openIndex(mountedIndex); + ensureGreen(mountedIndex); + } + assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2)); assertDocCount(client(), mountedIndex, numDocs); + updateRandomIndexSettings(mountedIndex); + updateRandomMappings(mountedIndex); + logger.debug("--> adding replica to test peer-recovery"); - updateIndexSettings(mountedIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)); + updateIndexSettings(mountedIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2)); ensureGreen(mountedIndex); } } diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeIndexCompatibilityTestCase.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeIndexCompatibilityTestCase.java new file mode 100644 index 0000000000000..03b6a9292e355 --- /dev/null +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeIndexCompatibilityTestCase.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.lucene; + +import com.carrotsearch.randomizedtesting.TestMethodAndParams; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; + +import org.elasticsearch.test.cluster.util.Version; + +import java.util.Comparator; +import java.util.List; +import java.util.stream.Stream; + +import static org.elasticsearch.test.cluster.util.Version.CURRENT; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.notNullValue; + +/** + * Test suite for Lucene indices backward compatibility with N-2 versions during rolling upgrades. The test suite creates a cluster in N-2 + * version, then upgrades each node sequentially to N-1 version and finally upgrades each node sequentially to the current version. Test + * methods are executed after each node upgrade. + */ +@TestCaseOrdering(RollingUpgradeIndexCompatibilityTestCase.TestCaseOrdering.class) +public abstract class RollingUpgradeIndexCompatibilityTestCase extends AbstractIndexCompatibilityTestCase { + + private final List nodesVersions; + + public RollingUpgradeIndexCompatibilityTestCase(@Name("cluster") List nodesVersions) { + this.nodesVersions = nodesVersions; + } + + @ParametersFactory + public static Iterable parameters() { + return Stream.of( + // Begin on N-2 + List.of(VERSION_MINUS_2, VERSION_MINUS_2, VERSION_MINUS_2), + // Rolling upgrade to VERSION_MINUS_1 + List.of(VERSION_MINUS_1, VERSION_MINUS_2, VERSION_MINUS_2), + List.of(VERSION_MINUS_1, VERSION_MINUS_1, VERSION_MINUS_2), + List.of(VERSION_MINUS_1, VERSION_MINUS_1, VERSION_MINUS_1), + // Rolling upgrade to CURRENT + List.of(CURRENT, VERSION_MINUS_1, VERSION_MINUS_1), + List.of(CURRENT, CURRENT, VERSION_MINUS_1), + List.of(CURRENT, CURRENT, CURRENT) + ).map(nodesVersion -> new Object[] { nodesVersion }).toList(); + } + + @Override + protected void maybeUpgrade() throws Exception { + assertThat(nodesVersions, hasSize(NODES)); + + for (int i = 0; i < NODES; i++) { + var nodeName = cluster().getName(i); + + var expectedNodeVersion = nodesVersions.get(i); + assertThat(expectedNodeVersion, notNullValue()); + + var currentNodeVersion = nodesVersions().get(nodeName); + assertThat(currentNodeVersion, notNullValue()); + assertThat(currentNodeVersion.onOrBefore(expectedNodeVersion), equalTo(true)); + + if (currentNodeVersion.equals(expectedNodeVersion) == false) { + closeClients(); + cluster().upgradeNodeToVersion(i, expectedNodeVersion); + initClient(); + } + + currentNodeVersion = nodesVersions().get(nodeName); + assertThat(currentNodeVersion, equalTo(expectedNodeVersion)); + } + } + + /** + * Execute the test suite with the parameters provided by the {@link #parameters()} in nodes versions order. + */ + public static class TestCaseOrdering implements Comparator { + @Override + public int compare(TestMethodAndParams o1, TestMethodAndParams o2) { + List nodesVersions1 = asInstanceOf(List.class, o1.getInstanceArguments().get(0)); + assertThat(nodesVersions1, hasSize(NODES)); + List nodesVersions2 = asInstanceOf(List.class, o2.getInstanceArguments().get(0)); + assertThat(nodesVersions2, hasSize(NODES)); + for (int i = 0; i < NODES; i++) { + var nodeVersion1 = asInstanceOf(Version.class, nodesVersions1.get(i)); + var nodeVersion2 = asInstanceOf(Version.class, nodesVersions2.get(i)); + var result = nodeVersion1.compareTo(nodeVersion2); + if (result != 0) { + return result; + } + } + return 0; + } + } +} diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeSearchableSnapshotIndexCompatibilityIT.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeSearchableSnapshotIndexCompatibilityIT.java new file mode 100644 index 0000000000000..1117d36024bf0 --- /dev/null +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeSearchableSnapshotIndexCompatibilityIT.java @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.lucene; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.repositories.fs.FsRepository; +import org.elasticsearch.test.cluster.util.Version; + +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; + +public class RollingUpgradeSearchableSnapshotIndexCompatibilityIT extends RollingUpgradeIndexCompatibilityTestCase { + + static { + clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial") + .setting("xpack.searchable.snapshot.shared_cache.size", "16MB") + .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB"); + } + + public RollingUpgradeSearchableSnapshotIndexCompatibilityIT(List nodesVersion) { + super(nodesVersion); + } + + /** + * Creates an index and a snapshot on N-2, then mounts the snapshot during rolling upgrades. + */ + public void testMountSearchableSnapshot() throws Exception { + final String repository = suffix("repository"); + final String snapshot = suffix("snapshot"); + final String index = suffix("index-rolling-upgrade"); + final var mountedIndex = suffix("index-rolling-upgrade-mounted"); + final int numDocs = 3145; + + if (isFullyUpgradedTo(VERSION_MINUS_2)) { + logger.debug("--> registering repository [{}]", repository); + registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings()); + + logger.debug("--> creating index [{}]", index); + createIndex( + client(), + index, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) + .build() + ); + + logger.debug("--> indexing [{}] docs in [{}]", numDocs, index); + indexDocs(index, numDocs); + + logger.debug("--> creating snapshot [{}]", snapshot); + createSnapshot(client(), repository, snapshot, true); + + logger.debug("--> deleting index [{}]", index); + deleteIndex(index); + return; + } + + boolean success = false; + try { + logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex); + mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex); + ensureGreen(mountedIndex); + + updateRandomIndexSettings(mountedIndex); + updateRandomMappings(mountedIndex); + + assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2)); + assertDocCount(client(), mountedIndex, numDocs); + + logger.debug("--> closing mounted index [{}]", mountedIndex); + closeIndex(mountedIndex); + ensureGreen(mountedIndex); + + logger.debug("--> re-opening index [{}]", mountedIndex); + openIndex(mountedIndex); + ensureGreen(mountedIndex); + + logger.debug("--> deleting mounted index [{}]", mountedIndex); + deleteIndex(mountedIndex); + + success = true; + } finally { + if (success == false) { + try { + client().performRequest(new Request("DELETE", "/" + mountedIndex)); + } catch (ResponseException e) { + logger.warn("Failed to delete mounted index [" + mountedIndex + ']', e); + } + } + } + } + + /** + * Creates an index and a snapshot on N-2, mounts the snapshot and ensures it remains searchable during rolling upgrades. + */ + public void testSearchableSnapshotUpgrade() throws Exception { + final String mountedIndex = suffix("index-rolling-upgraded-mounted"); + final String repository = suffix("repository"); + final String snapshot = suffix("snapshot"); + final String index = suffix("index-rolling-upgraded"); + final int numDocs = 2143; + + if (isFullyUpgradedTo(VERSION_MINUS_2)) { + logger.debug("--> registering repository [{}]", repository); + registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings()); + + logger.debug("--> creating index [{}]", index); + createIndex( + client(), + index, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) + .build() + ); + + logger.debug("--> indexing [{}] docs in [{}]", numDocs, index); + indexDocs(index, numDocs); + + logger.debug("--> creating snapshot [{}]", snapshot); + createSnapshot(client(), repository, snapshot, true); + + logger.debug("--> deleting index [{}]", index); + deleteIndex(index); + + logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex); + mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex); + } + + ensureGreen(mountedIndex); + + if (isIndexClosed(mountedIndex)) { + logger.debug("--> re-opening index [{}] after upgrade", mountedIndex); + openIndex(mountedIndex); + ensureGreen(mountedIndex); + } + + updateRandomIndexSettings(mountedIndex); + updateRandomMappings(mountedIndex); + + assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2)); + assertDocCount(client(), mountedIndex, numDocs); + + if (randomBoolean()) { + logger.debug("--> random closing of index [{}] before upgrade", mountedIndex); + closeIndex(mountedIndex); + ensureGreen(mountedIndex); + } + } +} diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java index 0cd2823080b9b..808aec92fb35d 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java @@ -206,13 +206,32 @@ public static void waitForElasticsearchToStart() { ps output: %s - stdout(): + Stdout: %s Stderr: + %s + + Thread dump: %s\ - """, psOutput, dockerLogs.stdout(), dockerLogs.stderr())); + """, psOutput, dockerLogs.stdout(), dockerLogs.stderr(), getThreadDump())); + } + } + + /** + * @return output of jstack for currently running Java process + */ + private static String getThreadDump() { + try { + String pid = dockerShell.run("/usr/share/elasticsearch/jdk/bin/jps | grep -v 'Jps' | awk '{print $1}'").stdout(); + if (pid.isEmpty() == false) { + return dockerShell.run("/usr/share/elasticsearch/jdk/bin/jstack " + Integer.parseInt(pid)).stdout(); + } + } catch (Exception e) { + logger.error("Failed to get thread dump", e); } + + return ""; } /** diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java index 5dc47993072a8..554ae871ea9fa 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java @@ -29,6 +29,9 @@ */ public class DockerRun { + // Use less secure entropy source to avoid hanging when generating certificates + private static final String DEFAULT_JAVA_OPTS = "-Djava.security.egd=file:/dev/urandom"; + private Distribution distribution; private final Map envVars = new HashMap<>(); private final Map volumes = new HashMap<>(); @@ -112,6 +115,11 @@ String build() { // Limit container memory cmd.add("--memory " + memory); + // Add default java opts + for (String envVar : List.of("CLI_JAVA_OPTS", "ES_JAVA_OPTS")) { + this.envVars.put(envVar, this.envVars.getOrDefault(envVar, "") + " " + DEFAULT_JAVA_OPTS); + } + this.envVars.forEach((key, value) -> cmd.add("--env " + key + "=\"" + value + "\"")); // Map ports in the container to the host, so that we can send requests diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 147b04ccd6722..c627335af6263 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -59,25 +59,5 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.replaceValueInMatch("profile.shards.0.dfs.knn.0.query.0.description", "DocAndScoreQuery[0,...][0.009673266,...],0.009673266", "dfs knn vector profiling with vector_operations_count") task.skipTest("cat.aliases/10_basic/Deprecated local parameter", "CAT APIs not covered by compatibility policy") task.skipTest("cat.shards/10_basic/Help", "sync_id is removed in 9.0") - task.skipTest("tsdb/20_mapping/exact match object type", "skip until pr/116687 gets backported") - task.skipTest("tsdb/25_id_generation/delete over _bulk", "skip until pr/116687 gets backported") - task.skipTest("tsdb/80_index_resize/split", "skip until pr/116687 gets backported") - task.skipTest("tsdb/90_unsupported_operations/noop update", "skip until pr/116687 gets backported") - task.skipTest("tsdb/90_unsupported_operations/regular update", "skip until pr/116687 gets backported") - task.skipTest("tsdb/90_unsupported_operations/search with routing", "skip until pr/116687 gets backported") - task.skipTest("tsdb/90_unsupported_operations/index with routing over _bulk", "skip until pr/116687 gets backported") - task.skipTest("tsdb/90_unsupported_operations/update over _bulk", "skip until pr/116687 gets backported") - task.skipTest("tsdb/90_unsupported_operations/index with routing", "skip until pr/116687 gets backported") task.skipTest("search/500_date_range/from, to, include_lower, include_upper deprecated", "deprecated parameters are removed in 9.0") - task.skipTest("tsdb/20_mapping/stored source is supported", "no longer serialize source_mode") - task.skipTest("tsdb/20_mapping/Synthetic source", "no longer serialize source_mode") - task.skipTest("logsdb/10_settings/create logs index", "no longer serialize source_mode") - task.skipTest("logsdb/20_source_mapping/stored _source mode is supported", "no longer serialize source_mode") - task.skipTest("logsdb/20_source_mapping/include/exclude is supported with stored _source", "no longer serialize source_mode") - task.skipTest("logsdb/20_source_mapping/synthetic _source is default", "no longer serialize source_mode") - task.skipTest("search/520_fetch_fields/fetch _seq_no via fields", "error code is changed from 5xx to 400 in 9.0") - task.skipTest("search.vectors/41_knn_search_bbq_hnsw/Test knn search", "Scoring has changed in latest versions") - task.skipTest("search.vectors/42_knn_search_bbq_flat/Test knn search", "Scoring has changed in latest versions") - task.skipTest("synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set", "Can't work until auto-expand replicas is 0-1 for synonyms index") - task.skipTest("search/90_search_after/_shard_doc sort", "restriction has been lifted in latest versions") }) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.async_query_delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.async_query_delete.json new file mode 100644 index 0000000000000..a6339559afd72 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.async_query_delete.json @@ -0,0 +1,27 @@ +{ + "esql.async_query_delete": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/esql-async-query-delete-api.html", + "description": "Delete an async query request given its ID." + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_query/async/{id}", + "methods": ["DELETE"], + "parts": { + "id": { + "type": "string", + "description": "The async query ID" + } + } + } + ] + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle_stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle_stats.json new file mode 100644 index 0000000000000..8c9e947903402 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle_stats.json @@ -0,0 +1,21 @@ +{ + "indices.get_data_lifecycle_stats": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams-get-lifecycle-stats.html", + "description": "Get data stream lifecycle statistics." + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_lifecycle/stats", + "methods": ["GET"] + } + ] + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.update.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.update.json new file mode 100644 index 0000000000000..6c458ce080aa7 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.update.json @@ -0,0 +1,45 @@ +{ + "inference.update": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/update-inference-api.html", + "description": "Update inference" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], + "content_type": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_inference/{inference_id}/_update", + "methods": ["POST"], + "parts": { + "inference_id": { + "type": "string", + "description": "The inference Id" + } + } + }, + { + "path": "/_inference/{task_type}/{inference_id}/_update", + "methods": ["POST"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" + }, + "inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference endpoint's task and service settings" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/security.delegate_pki.json b/rest-api-spec/src/main/resources/rest-api-spec/api/security.delegate_pki.json new file mode 100644 index 0000000000000..752ea35028b4f --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/security.delegate_pki.json @@ -0,0 +1,26 @@ +{ + "security.delegate_pki": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/security-api-delegate-pki-authentication.html", + "description": "Delegate PKI authentication." + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_security/delegate_pki", + "methods": ["POST"] + } + ] + }, + "params": {}, + "body": { + "description":"The X509Certificate chain.", + "required":true + } + } +} diff --git a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java index 084e212a913b2..675092bffe8d5 100644 --- a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java +++ b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java @@ -14,7 +14,6 @@ import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; import org.apache.lucene.tests.util.TimeUnits; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.FeatureFlag; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; @@ -43,15 +42,9 @@ public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate super(testCandidate); } - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // remove restCompat check @ParametersFactory public static Iterable parameters() throws Exception { - String restCompatProperty = System.getProperty("tests.restCompat"); - if ("true".equals(restCompatProperty)) { - return createParametersWithLegacyNodeSelectorSupport(); - } else { - return createParameters(); - } + return createParameters(); } @Override diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java index 77c4f8a26f478..43bca39c02ce5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java @@ -16,8 +16,8 @@ import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.util.CollectionUtils; @@ -365,11 +365,10 @@ private void iterateAssertCount(final int numberOfShards, final int iterations, ); } - ClusterService clusterService = clusterService(); - final ClusterState state = clusterService.state(); + final ClusterState state = clusterService().state(); for (int shard = 0; shard < numberOfShards; shard++) { for (String id : ids) { - ShardId docShard = clusterService.operationRouting().shardId(state, "test", id, null); + ShardId docShard = OperationRouting.shardId(state, "test", id, null); if (docShard.id() == shard) { final IndexShardRoutingTable indexShardRoutingTable = state.routingTable().shardRoutingTable("test", shard); for (int copy = 0; copy < indexShardRoutingTable.size(); copy++) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java index 9c1daccd2cc9e..ab79fd7ba1813 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java @@ -40,18 +40,12 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.test.AbstractMultiClustersTestCase; import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.test.SkipUnavailableRule; +import org.elasticsearch.test.SkipUnavailableRule.NotSkipped; import org.elasticsearch.usage.UsageService; import org.junit.Assert; import org.junit.Rule; -import org.junit.rules.TestRule; -import org.junit.runner.Description; -import org.junit.runners.model.Statement; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; -import java.util.Arrays; + import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -59,8 +53,6 @@ import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.stream.Collectors; import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.ASYNC_FEATURE; import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.MRT_FEATURE; @@ -498,7 +490,7 @@ public void testRemoteOnlyTimesOut() throws Exception { assertThat(perCluster.get(REMOTE2), equalTo(null)); } - @SkipOverride(aliases = { REMOTE1 }) + @NotSkipped(aliases = { REMOTE1 }) public void testRemoteTimesOutFailure() throws Exception { Map testClusterInfo = setupClusters(); String remoteIndex = (String) testClusterInfo.get("remote.index"); @@ -528,7 +520,7 @@ public void testRemoteTimesOutFailure() throws Exception { /** * Search when all the remotes failed and not skipped */ - @SkipOverride(aliases = { REMOTE1, REMOTE2 }) + @NotSkipped(aliases = { REMOTE1, REMOTE2 }) public void testFailedAllRemotesSearch() throws Exception { Map testClusterInfo = setupClusters(); String localIndex = (String) testClusterInfo.get("local.index"); @@ -577,7 +569,7 @@ public void testRemoteHasNoIndex() throws Exception { /** * Test that we're still counting remote search even if remote cluster has no such index */ - @SkipOverride(aliases = { REMOTE1 }) + @NotSkipped(aliases = { REMOTE1 }) public void testRemoteHasNoIndexFailure() throws Exception { SearchRequest searchRequest = makeSearchRequest(REMOTE1 + ":no_such_index"); CCSTelemetrySnapshot telemetry = getTelemetryFromFailedSearch(searchRequest); @@ -695,40 +687,4 @@ private void indexDocs(Client client, String index, ActionListener listene bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).execute(listener.safeMap(r -> null)); } - /** - * Annotation to mark specific cluster in a test as not to be skipped when unavailable - */ - @Retention(RetentionPolicy.RUNTIME) - @Target(ElementType.METHOD) - @interface SkipOverride { - String[] aliases(); - } - - /** - * Test rule to process skip annotations - */ - static class SkipUnavailableRule implements TestRule { - private final Map skipMap; - - SkipUnavailableRule(String... clusterAliases) { - this.skipMap = Arrays.stream(clusterAliases).collect(Collectors.toMap(Function.identity(), alias -> true)); - } - - public Map getMap() { - return skipMap; - } - - @Override - public Statement apply(Statement base, Description description) { - // Check for annotation named "SkipOverride" and set the overrides accordingly - var aliases = description.getAnnotation(SkipOverride.class); - if (aliases != null) { - for (String alias : aliases.aliases()) { - skipMap.put(alias, false); - } - } - return base; - } - - } } diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index bff1fa4015acd..1cee9ca7bd495 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -148,6 +148,7 @@ static TransportVersion def(int id) { public static final TransportVersion SIMULATE_IGNORED_FIELDS = def(8_813_00_0); public static final TransportVersion TRANSFORMS_UPGRADE_MODE = def(8_814_00_0); public static final TransportVersion NODE_SHUTDOWN_EPHEMERAL_ID_ADDED = def(8_815_00_0); + public static final TransportVersion ESQL_CCS_TELEMETRY_STATS = def(8_816_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java index 3bbaa80ec200e..8500302e4f755 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java @@ -41,7 +41,6 @@ *
*/ public final class CCSTelemetrySnapshot implements Writeable, ToXContentFragment { - public static final String CCS_TELEMETRY_FIELD_NAME = "_search"; private long totalCount; private long successCount; private final Map failureReasons; @@ -66,6 +65,9 @@ public final class CCSTelemetrySnapshot implements Writeable, ToXContentFragment private final Map clientCounts; private final Map byRemoteCluster; + // Whether we should use per-MRT (minimize roundtrips) metrics. + // ES|QL does not have "minimize_roundtrips" option, so we don't collect those metrics for ES|QL usage. + private boolean useMRT = true; /** * Creates a new stats instance with the provided info. @@ -191,6 +193,11 @@ public Map getByRemoteCluster() { return Collections.unmodifiableMap(byRemoteCluster); } + public CCSTelemetrySnapshot setUseMRT(boolean useMRT) { + this.useMRT = useMRT; + return this; + } + public static class PerClusterCCSTelemetry implements Writeable, ToXContentFragment { private long count; private long skippedCount; @@ -270,6 +277,11 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(count, skippedCount, took); } + + @Override + public String toString() { + return Strings.toString(this, true, true); + } } /** @@ -291,8 +303,10 @@ public void add(CCSTelemetrySnapshot stats) { stats.featureCounts.forEach((k, v) -> featureCounts.merge(k, v, Long::sum)); stats.clientCounts.forEach((k, v) -> clientCounts.merge(k, v, Long::sum)); took.add(stats.took); - tookMrtTrue.add(stats.tookMrtTrue); - tookMrtFalse.add(stats.tookMrtFalse); + if (useMRT) { + tookMrtTrue.add(stats.tookMrtTrue); + tookMrtFalse.add(stats.tookMrtFalse); + } remotesPerSearchMax = Math.max(remotesPerSearchMax, stats.remotesPerSearchMax); if (totalCount > 0 && oldCount > 0) { // Weighted average @@ -328,30 +342,28 @@ private static void publishLatency(XContentBuilder builder, String name, LongMet @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(CCS_TELEMETRY_FIELD_NAME); - { - builder.field("total", totalCount); - builder.field("success", successCount); - builder.field("skipped", skippedRemotes); - publishLatency(builder, "took", took); + builder.field("total", totalCount); + builder.field("success", successCount); + builder.field("skipped", skippedRemotes); + publishLatency(builder, "took", took); + if (useMRT) { publishLatency(builder, "took_mrt_true", tookMrtTrue); publishLatency(builder, "took_mrt_false", tookMrtFalse); - builder.field("remotes_per_search_max", remotesPerSearchMax); - builder.field("remotes_per_search_avg", remotesPerSearchAvg); - builder.field("failure_reasons", failureReasons); - builder.field("features", featureCounts); - builder.field("clients", clientCounts); - builder.startObject("clusters"); - { - for (var entry : byRemoteCluster.entrySet()) { - String remoteName = entry.getKey(); - if (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(remoteName)) { - remoteName = SearchResponse.LOCAL_CLUSTER_NAME_REPRESENTATION; - } - builder.field(remoteName, entry.getValue()); + } + builder.field("remotes_per_search_max", remotesPerSearchMax); + builder.field("remotes_per_search_avg", remotesPerSearchAvg); + builder.field("failure_reasons", failureReasons); + builder.field("features", featureCounts); + builder.field("clients", clientCounts); + builder.startObject("clusters"); + { + for (var entry : byRemoteCluster.entrySet()) { + String remoteName = entry.getKey(); + if (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(remoteName)) { + remoteName = SearchResponse.LOCAL_CLUSTER_NAME_REPRESENTATION; } + builder.field(remoteName, entry.getValue()); } - builder.endObject(); } builder.endObject(); return builder; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java index 9e58d6d8febef..29a7dcb5d07d8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java @@ -10,6 +10,7 @@ package org.elasticsearch.action.admin.cluster.stats; import org.elasticsearch.ElasticsearchSecurityException; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ShardOperationFailedException; @@ -20,6 +21,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.query.SearchTimeoutException; +import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskCancelledException; import java.util.Arrays; @@ -84,6 +86,15 @@ public Builder setClient(String client) { return this; } + public Builder setClientFromTask(Task task) { + String client = task.getHeader(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER); + if (client != null) { + return setClient(client); + } else { + return this; + } + } + public Builder skippedRemote(String remote) { this.skippedRemotes.add(remote); return this; @@ -133,6 +144,10 @@ public static Result getFailureType(Exception e) { if (ExceptionsHelper.unwrapCorruption(e) != null) { return Result.CORRUPTION; } + ElasticsearchStatusException se = (ElasticsearchStatusException) ExceptionsHelper.unwrap(e, ElasticsearchStatusException.class); + if (se != null && se.getDetailedMessage().contains("license")) { + return Result.LICENSE; + } // This is kind of last resort check - if we still don't know the reason but all shard failures are remote, // we assume it's remote's fault somehow. if (e instanceof SearchPhaseExecutionException spe) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java index 6c8178282d3c3..3f04eceed7eb5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java @@ -47,6 +47,7 @@ public enum Result { TIMEOUT("timeout"), CORRUPTION("corruption"), SECURITY("security"), + LICENSE("license"), // May be helpful if there's a lot of other reasons, and it may be hard to calculate the unknowns for some clients. UNKNOWN("other"); @@ -106,8 +107,14 @@ public String getName() { private final Map clientCounts; private final Map byRemoteCluster; + // Should we calculate separate metrics per MRT? + private final boolean useMRT; public CCSUsageTelemetry() { + this(true); + } + + public CCSUsageTelemetry(boolean useMRT) { this.byRemoteCluster = new ConcurrentHashMap<>(); totalCount = new LongAdder(); successCount = new LongAdder(); @@ -119,6 +126,7 @@ public CCSUsageTelemetry() { skippedRemotes = new LongAdder(); featureCounts = new ConcurrentHashMap<>(); clientCounts = new ConcurrentHashMap<>(); + this.useMRT = useMRT; } public void updateUsage(CCSUsage ccsUsage) { @@ -134,10 +142,12 @@ private void doUpdate(CCSUsage ccsUsage) { if (isSuccess(ccsUsage)) { successCount.increment(); took.record(searchTook); - if (isMRT(ccsUsage)) { - tookMrtTrue.record(searchTook); - } else { - tookMrtFalse.record(searchTook); + if (useMRT) { + if (isMRT(ccsUsage)) { + tookMrtTrue.record(searchTook); + } else { + tookMrtFalse.record(searchTook); + } } ccsUsage.getPerClusterUsage().forEach((r, u) -> byRemoteCluster.computeIfAbsent(r, PerClusterCCSTelemetry::new).update(u)); } else { @@ -243,6 +253,6 @@ public CCSTelemetrySnapshot getCCSTelemetrySnapshot() { Collections.unmodifiableMap(Maps.transformValues(featureCounts, LongAdder::longValue)), Collections.unmodifiableMap(Maps.transformValues(clientCounts, LongAdder::longValue)), Collections.unmodifiableMap(Maps.transformValues(byRemoteCluster, PerClusterCCSTelemetry::getSnapshot)) - ); + ).setUseMRT(useMRT); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java index abeb73e5d8c3e..48b4e967742cd 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java @@ -31,7 +31,8 @@ public class ClusterStatsNodeResponse extends BaseNodeResponse { private final ClusterHealthStatus clusterStatus; private final SearchUsageStats searchUsageStats; private final RepositoryUsageStats repositoryUsageStats; - private final CCSTelemetrySnapshot ccsMetrics; + private final CCSTelemetrySnapshot searchCcsMetrics; + private final CCSTelemetrySnapshot esqlCcsMetrics; public ClusterStatsNodeResponse(StreamInput in) throws IOException { super(in); @@ -46,10 +47,15 @@ public ClusterStatsNodeResponse(StreamInput in) throws IOException { } if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { repositoryUsageStats = RepositoryUsageStats.readFrom(in); - ccsMetrics = new CCSTelemetrySnapshot(in); + searchCcsMetrics = new CCSTelemetrySnapshot(in); } else { repositoryUsageStats = RepositoryUsageStats.EMPTY; - ccsMetrics = new CCSTelemetrySnapshot(); + searchCcsMetrics = new CCSTelemetrySnapshot(); + } + if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_TELEMETRY_STATS)) { + esqlCcsMetrics = new CCSTelemetrySnapshot(in); + } else { + esqlCcsMetrics = new CCSTelemetrySnapshot(); } } @@ -61,7 +67,8 @@ public ClusterStatsNodeResponse( ShardStats[] shardsStats, SearchUsageStats searchUsageStats, RepositoryUsageStats repositoryUsageStats, - CCSTelemetrySnapshot ccsTelemetrySnapshot + CCSTelemetrySnapshot ccsTelemetrySnapshot, + CCSTelemetrySnapshot esqlTelemetrySnapshot ) { super(node); this.nodeInfo = nodeInfo; @@ -70,7 +77,8 @@ public ClusterStatsNodeResponse( this.clusterStatus = clusterStatus; this.searchUsageStats = Objects.requireNonNull(searchUsageStats); this.repositoryUsageStats = Objects.requireNonNull(repositoryUsageStats); - this.ccsMetrics = ccsTelemetrySnapshot; + this.searchCcsMetrics = ccsTelemetrySnapshot; + this.esqlCcsMetrics = esqlTelemetrySnapshot; } public NodeInfo nodeInfo() { @@ -101,8 +109,12 @@ public RepositoryUsageStats repositoryUsageStats() { return repositoryUsageStats; } - public CCSTelemetrySnapshot getCcsMetrics() { - return ccsMetrics; + public CCSTelemetrySnapshot getSearchCcsMetrics() { + return searchCcsMetrics; + } + + public CCSTelemetrySnapshot getEsqlCcsMetrics() { + return esqlCcsMetrics; } @Override @@ -117,8 +129,11 @@ public void writeTo(StreamOutput out) throws IOException { } if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { repositoryUsageStats.writeTo(out); - ccsMetrics.writeTo(out); + searchCcsMetrics.writeTo(out); } // else just drop these stats, ok for bwc + if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_TELEMETRY_STATS)) { + esqlCcsMetrics.writeTo(out); + } } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java index 5f7c45c5807a5..ed8ca2f94a78b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java @@ -36,10 +36,14 @@ public class ClusterStatsResponse extends BaseNodesResponse remoteClustersStats; + public static final String CCS_TELEMETRY_FIELD_NAME = "_search"; + public static final String ESQL_TELEMETRY_FIELD_NAME = "_esql"; + public ClusterStatsResponse( long timestamp, String clusterUUID, @@ -58,6 +62,7 @@ public ClusterStatsResponse( nodesStats = new ClusterStatsNodes(nodes); indicesStats = new ClusterStatsIndices(nodes, mappingStats, analysisStats, versionStats); ccsMetrics = new CCSTelemetrySnapshot(); + esqlMetrics = new CCSTelemetrySnapshot().setUseMRT(false); ClusterHealthStatus status = null; for (ClusterStatsNodeResponse response : nodes) { // only the master node populates the status @@ -66,7 +71,10 @@ public ClusterStatsResponse( break; } } - nodes.forEach(node -> ccsMetrics.add(node.getCcsMetrics())); + nodes.forEach(node -> { + ccsMetrics.add(node.getSearchCcsMetrics()); + esqlMetrics.add(node.getEsqlCcsMetrics()); + }); this.status = status; this.clusterSnapshotStats = clusterSnapshotStats; @@ -147,9 +155,18 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (remoteClustersStats != null) { builder.field("clusters", remoteClustersStats); } + builder.startObject(CCS_TELEMETRY_FIELD_NAME); ccsMetrics.toXContent(builder, params); builder.endObject(); + if (esqlMetrics.getTotalCount() > 0) { + builder.startObject(ESQL_TELEMETRY_FIELD_NAME); + esqlMetrics.toXContent(builder, params); + builder.endObject(); + } + + builder.endObject(); + return builder; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 2c20daa5d7afb..6f69def7aa4e0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -103,6 +103,7 @@ public class TransportClusterStatsAction extends TransportNodesAction< private final RepositoriesService repositoriesService; private final SearchUsageHolder searchUsageHolder; private final CCSUsageTelemetry ccsUsageHolder; + private final CCSUsageTelemetry esqlUsageHolder; private final Executor clusterStateStatsExecutor; private final MetadataStatsCache mappingStatsCache; @@ -135,6 +136,7 @@ public TransportClusterStatsAction( this.repositoriesService = repositoriesService; this.searchUsageHolder = usageService.getSearchUsageHolder(); this.ccsUsageHolder = usageService.getCcsUsageHolder(); + this.esqlUsageHolder = usageService.getEsqlUsageHolder(); this.clusterStateStatsExecutor = threadPool.executor(ThreadPool.Names.MANAGEMENT); this.mappingStatsCache = new MetadataStatsCache<>(threadPool.getThreadContext(), MappingStats::of); this.analysisStatsCache = new MetadataStatsCache<>(threadPool.getThreadContext(), AnalysisStats::of); @@ -293,6 +295,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq final RepositoryUsageStats repositoryUsageStats = repositoriesService.getUsageStats(); final CCSTelemetrySnapshot ccsTelemetry = ccsUsageHolder.getCCSTelemetrySnapshot(); + final CCSTelemetrySnapshot esqlTelemetry = esqlUsageHolder.getCCSTelemetrySnapshot(); return new ClusterStatsNodeResponse( nodeInfo.getNode(), @@ -302,7 +305,8 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq shardsStats.toArray(new ShardStats[shardsStats.size()]), searchUsageStats, repositoryUsageStats, - ccsTelemetry + ccsTelemetry, + esqlTelemetry ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java index c30a2a44274a7..e3e737595cac6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java @@ -141,7 +141,7 @@ protected void doExecuteForked(Task task, ResolveClusterActionRequest request, A RemoteClusterClient remoteClusterClient = remoteClusterService.getRemoteClusterClient( clusterAlias, searchCoordinationExecutor, - RemoteClusterService.DisconnectedStrategy.RECONNECT_IF_DISCONNECTED + RemoteClusterService.DisconnectedStrategy.FAIL_IF_DISCONNECTED ); var remoteRequest = new ResolveClusterActionRequest(originalIndices.indices(), request.indicesOptions()); // allow cancellation requests to propagate to remote clusters diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/RequestDispatcher.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/RequestDispatcher.java index 802f5d196569a..fce925d868532 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/RequestDispatcher.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/RequestDispatcher.java @@ -95,7 +95,7 @@ final class RequestDispatcher { for (String index : indices) { final GroupShardsIterator shardIts; try { - shardIts = clusterService.operationRouting().searchShards(clusterState, new String[] { index }, null, null, null, null); + shardIts = clusterService.operationRouting().searchShards(clusterState, new String[] { index }, null, null); } catch (Exception e) { onIndexFailure.accept(index, e); continue; diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java index a2c7c8664e81a..29b926598ac32 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java @@ -30,7 +30,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.PlainShardIterator; import org.elasticsearch.cluster.routing.ShardIterator; -import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.TimeValue; @@ -109,7 +108,7 @@ protected ShardIterator shards(ClusterState state, InternalRequest request) { if (iterator == null) { return null; } - return new PlainShardIterator(iterator.shardId(), iterator.getShardRoutings().stream().filter(ShardRouting::isSearchable).toList()); + return PlainShardIterator.allSearchableShards(iterator); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java index 01dc705d7146b..e5087a790a292 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.EsExecutors; @@ -81,7 +82,7 @@ protected void doExecute(Task task, final MultiGetRequest request, final ActionL lastResolvedIndex = Tuple.tuple(item.index(), concreteSingleIndex); } item.routing(clusterState.metadata().resolveIndexRouting(item.routing(), item.index())); - shardId = clusterService.operationRouting().shardId(clusterState, concreteSingleIndex, item.id(), item.routing()); + shardId = OperationRouting.shardId(clusterState, concreteSingleIndex, item.id(), item.routing()); } catch (RoutingMissingException e) { responses.set(i, newItemFailure(e.getIndex().getName(), e.getId(), e)); continue; diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java index 0fa770df8e4ef..d9a04acc0466e 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java @@ -30,7 +30,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.PlainShardIterator; import org.elasticsearch.cluster.routing.ShardIterator; -import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.TimeValue; @@ -113,7 +112,7 @@ protected ShardIterator shards(ClusterState state, InternalRequest request) { if (iterator == null) { return null; } - return new PlainShardIterator(iterator.shardId(), iterator.getShardRoutings().stream().filter(ShardRouting::isSearchable).toList()); + return PlainShardIterator.allSearchableShards(iterator); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index ae27406bf396d..70a7f4c8cad0c 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -388,10 +388,7 @@ void executeRequest( if (original.pointInTimeBuilder() != null) { tl.setFeature(CCSUsageTelemetry.PIT_FEATURE); } - String client = task.getHeader(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER); - if (client != null) { - tl.setClient(client); - } + tl.setClient(task); // Check if any of the index patterns are wildcard patterns var localIndices = resolvedIndices.getLocalIndices(); if (localIndices != null && Arrays.stream(localIndices.indices()).anyMatch(Regex::isSimpleMatchPattern)) { @@ -508,6 +505,7 @@ void executeRequest( } } }); + final SearchSourceBuilder source = original.source(); if (shouldOpenPIT(source)) { // disabling shard reordering for request @@ -1883,7 +1881,7 @@ private interface TelemetryListener { void setFeature(String feature); - void setClient(String client); + void setClient(Task task); } private class SearchResponseActionListener extends DelegatingActionListener @@ -1917,8 +1915,8 @@ public void setFeature(String feature) { } @Override - public void setClient(String client) { - usageBuilder.setClient(client); + public void setClient(Task task) { + usageBuilder.setClientFromTask(task); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java index c8b53e0185606..b3230fae9834a 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.EsExecutors; @@ -72,8 +73,12 @@ protected void doExecute(Task task, final MultiTermVectorsRequest request, final clusterState.metadata().resolveIndexRouting(termVectorsRequest.routing(), termVectorsRequest.index()) ); String concreteSingleIndex = indexNameExpressionResolver.concreteSingleIndex(clusterState, termVectorsRequest).getName(); - shardId = clusterService.operationRouting() - .shardId(clusterState, concreteSingleIndex, termVectorsRequest.id(), termVectorsRequest.routing()); + shardId = OperationRouting.shardId( + clusterState, + concreteSingleIndex, + termVectorsRequest.id(), + termVectorsRequest.routing() + ); } catch (RoutingMissingException e) { responses.set( i, diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java index 7567577c78452..02479a9f8d143 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java @@ -62,16 +62,27 @@ public TransportTermVectorsAction( @Override protected ShardIterator shards(ClusterState state, InternalRequest request) { + final var operationRouting = clusterService.operationRouting(); if (request.request().doc() != null && request.request().routing() == null) { // artificial document without routing specified, ignore its "id" and use either random shard or according to preference - GroupShardsIterator groupShardsIter = clusterService.operationRouting() - .searchShards(state, new String[] { request.concreteIndex() }, null, request.request().preference()); + GroupShardsIterator groupShardsIter = operationRouting.searchShards( + state, + new String[] { request.concreteIndex() }, + null, + request.request().preference() + ); return groupShardsIter.iterator().next(); } - ShardIterator shards = clusterService.operationRouting() - .getShards(state, request.concreteIndex(), request.request().id(), request.request().routing(), request.request().preference()); - return clusterService.operationRouting().useOnlyPromotableShardsForStateless(shards); + return operationRouting.useOnlyPromotableShardsForStateless( + operationRouting.getShards( + state, + request.concreteIndex(), + request.request().id(), + request.request().routing(), + request.request().preference() + ) + ); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index c74275991c899..056292177646b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -41,7 +41,6 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.gateway.MetadataStateFormat; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; @@ -944,22 +943,12 @@ public IndexMetadata withIncrementedPrimaryTerm(int shardId) { /** * @param timestampRange new @timestamp range * @param eventIngestedRange new 'event.ingested' range - * @param minClusterTransportVersion minimum transport version used between nodes of this cluster * @return copy of this instance with updated timestamp range */ - public IndexMetadata withTimestampRanges( - IndexLongFieldRange timestampRange, - IndexLongFieldRange eventIngestedRange, - TransportVersion minClusterTransportVersion - ) { + public IndexMetadata withTimestampRanges(IndexLongFieldRange timestampRange, IndexLongFieldRange eventIngestedRange) { if (timestampRange.equals(this.timestampRange) && eventIngestedRange.equals(this.eventIngestedRange)) { return this; } - @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_FOUNDATIONS) // remove this check when 8.15 is no longer communicable - IndexLongFieldRange allowedEventIngestedRange = eventIngestedRange; - if (minClusterTransportVersion.before(TransportVersions.V_8_15_0)) { - allowedEventIngestedRange = IndexLongFieldRange.UNKNOWN; - } return new IndexMetadata( this.index, this.version, @@ -990,7 +979,7 @@ public IndexMetadata withTimestampRanges( this.isSystem, this.isHidden, timestampRange, - allowedEventIngestedRange, + eventIngestedRange, this.priority, this.creationDate, this.ignoreDiskWatermarks, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 71de9ac88a360..a9617058acd60 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -12,7 +12,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.CollectionUtil; -import org.elasticsearch.TransportVersion; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.Diffable; @@ -521,7 +520,7 @@ public Metadata withLastCommittedValues( /** * Creates a copy of this instance updated with the given {@link IndexMetadata} that must only contain changes to primary terms * and in-sync allocation ids relative to the existing entries. This method is only used by - * {@link org.elasticsearch.cluster.routing.allocation.IndexMetadataUpdater#applyChanges(Metadata, RoutingTable, TransportVersion)}. + * {@link org.elasticsearch.cluster.routing.allocation.IndexMetadataUpdater#applyChanges(Metadata, RoutingTable)}. * @param updates map of index name to {@link IndexMetadata}. * @return updated metadata instance */ diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java index 5e2dbf1c5df5d..49da00eae8a5a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java @@ -27,7 +27,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -125,32 +124,12 @@ public GroupShardsIterator searchShards( nodeCounts ); if (iterator != null) { - final List shardsThatCanHandleSearches; - if (isStateless) { - shardsThatCanHandleSearches = statelessShardsThatHandleSearches(clusterState, iterator); - } else { - shardsThatCanHandleSearches = statefulShardsThatHandleSearches(iterator); - } - set.add(new PlainShardIterator(iterator.shardId(), shardsThatCanHandleSearches)); + set.add(PlainShardIterator.allSearchableShards(iterator)); } } return GroupShardsIterator.sortAndCreate(new ArrayList<>(set)); } - private static List statefulShardsThatHandleSearches(ShardIterator iterator) { - final List shardsThatCanHandleSearches = new ArrayList<>(iterator.size()); - for (ShardRouting shardRouting : iterator) { - if (shardRouting.isSearchable()) { - shardsThatCanHandleSearches.add(shardRouting); - } - } - return shardsThatCanHandleSearches; - } - - private static List statelessShardsThatHandleSearches(ClusterState clusterState, ShardIterator iterator) { - return iterator.getShardRoutings().stream().filter(ShardRouting::isSearchable).toList(); - } - public static ShardIterator getShards(ClusterState clusterState, ShardId shardId) { final IndexShardRoutingTable shard = clusterState.routingTable().shardRoutingTable(shardId); return shard.activeInitializingShardsRandomIt(); @@ -297,7 +276,7 @@ private static IndexMetadata indexMetadata(ClusterState clusterState, String ind return indexMetadata; } - public ShardId shardId(ClusterState clusterState, String index, String id, @Nullable String routing) { + public static ShardId shardId(ClusterState clusterState, String index, String id, @Nullable String routing) { IndexMetadata indexMetadata = indexMetadata(clusterState, index); return new ShardId(indexMetadata.getIndex(), IndexRouting.fromIndexMetadata(indexMetadata).getShard(id, routing)); } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/PlainShardIterator.java b/server/src/main/java/org/elasticsearch/cluster/routing/PlainShardIterator.java index b49f37c7440dd..a429cc040de3c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/PlainShardIterator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/PlainShardIterator.java @@ -11,6 +11,7 @@ import org.elasticsearch.index.shard.ShardId; +import java.util.ArrayList; import java.util.List; /** @@ -21,6 +22,20 @@ public class PlainShardIterator extends PlainShardsIterator implements ShardIter private final ShardId shardId; + public static PlainShardIterator allSearchableShards(ShardIterator shardIterator) { + return new PlainShardIterator(shardIterator.shardId(), shardsThatCanHandleSearches(shardIterator)); + } + + private static List shardsThatCanHandleSearches(ShardIterator iterator) { + final List shardsThatCanHandleSearches = new ArrayList<>(iterator.size()); + for (ShardRouting shardRouting : iterator) { + if (shardRouting.isSearchable()) { + shardsThatCanHandleSearches.add(shardRouting); + } + } + return shardsThatCanHandleSearches; + } + /** * Creates a {@link PlainShardIterator} instance that iterates over a subset of the given shards * this the a given shardId. diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java index 1980cb17417d4..ec4bd4b2d077a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java @@ -10,7 +10,6 @@ package org.elasticsearch.cluster.routing.allocation; import org.apache.logging.log4j.Logger; -import org.elasticsearch.TransportVersion; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; @@ -106,10 +105,9 @@ public void relocationCompleted(ShardRouting removedRelocationSource) { * * @param oldMetadata {@link Metadata} object from before the routing nodes was changed. * @param newRoutingTable {@link RoutingTable} object after routing changes were applied. - * @param minClusterTransportVersion minimum TransportVersion used between nodes of this cluster * @return adapted {@link Metadata}, potentially the original one if no change was needed. */ - public Metadata applyChanges(Metadata oldMetadata, RoutingTable newRoutingTable, TransportVersion minClusterTransportVersion) { + public Metadata applyChanges(Metadata oldMetadata, RoutingTable newRoutingTable) { Map>> changesGroupedByIndex = shardChanges.entrySet() .stream() .collect(Collectors.groupingBy(e -> e.getKey().getIndex())); @@ -122,14 +120,7 @@ public Metadata applyChanges(Metadata oldMetadata, RoutingTable newRoutingTable, for (Map.Entry shardEntry : indexChanges.getValue()) { ShardId shardId = shardEntry.getKey(); Updates updates = shardEntry.getValue(); - updatedIndexMetadata = updateInSyncAllocations( - newRoutingTable, - oldIndexMetadata, - updatedIndexMetadata, - shardId, - updates, - minClusterTransportVersion - ); + updatedIndexMetadata = updateInSyncAllocations(newRoutingTable, oldIndexMetadata, updatedIndexMetadata, shardId, updates); updatedIndexMetadata = updates.increaseTerm ? updatedIndexMetadata.withIncrementedPrimaryTerm(shardId.id()) : updatedIndexMetadata; @@ -150,8 +141,7 @@ private static IndexMetadata updateInSyncAllocations( IndexMetadata oldIndexMetadata, IndexMetadata updatedIndexMetadata, ShardId shardId, - Updates updates, - TransportVersion minClusterTransportVersion + Updates updates ) { assert Sets.haveEmptyIntersection(updates.addedAllocationIds, updates.removedAllocationIds) : "allocation ids cannot be both added and removed in the same allocation round, added ids: " @@ -183,8 +173,7 @@ private static IndexMetadata updateInSyncAllocations( allocationId = RecoverySource.ExistingStoreRecoverySource.FORCED_ALLOCATION_ID; updatedIndexMetadata = updatedIndexMetadata.withTimestampRanges( updatedIndexMetadata.getTimestampRange().removeShard(shardId.id(), oldIndexMetadata.getNumberOfShards()), - updatedIndexMetadata.getEventIngestedRange().removeShard(shardId.id(), oldIndexMetadata.getNumberOfShards()), - minClusterTransportVersion + updatedIndexMetadata.getEventIngestedRange().removeShard(shardId.id(), oldIndexMetadata.getNumberOfShards()) ); } else { assert recoverySource instanceof RecoverySource.SnapshotRecoverySource diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java index e0ada293e5508..1f6832649edf9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java @@ -340,7 +340,7 @@ public RoutingChangesObserver changes() { * Returns updated {@link Metadata} based on the changes that were made to the routing nodes */ public Metadata updateMetadataWithRoutingChanges(RoutingTable newRoutingTable) { - Metadata metadata = indexMetadataUpdater.applyChanges(metadata(), newRoutingTable, clusterState.getMinTransportVersion()); + Metadata metadata = indexMetadataUpdater.applyChanges(metadata(), newRoutingTable); return resizeSourceIndexUpdater.applyChanges(metadata, newRoutingTable); } diff --git a/server/src/main/java/org/elasticsearch/common/blobstore/support/AbstractBlobContainer.java b/server/src/main/java/org/elasticsearch/common/blobstore/support/AbstractBlobContainer.java index 635196dd7a52e..f62e4710c81d8 100644 --- a/server/src/main/java/org/elasticsearch/common/blobstore/support/AbstractBlobContainer.java +++ b/server/src/main/java/org/elasticsearch/common/blobstore/support/AbstractBlobContainer.java @@ -9,7 +9,6 @@ package org.elasticsearch.common.blobstore.support; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; @@ -24,17 +23,6 @@ protected AbstractBlobContainer(BlobPath path) { this.path = path; } - /** - * Temporary check that permits disabling CAS operations at runtime; TODO remove this when no longer needed - */ - protected static boolean skipCas(ActionListener listener) { - if ("true".equals(System.getProperty("test.repository_test_kit.skip_cas"))) { - listener.onFailure(new UnsupportedOperationException()); - return true; - } - return false; - } - @Override public BlobPath path() { return this.path; diff --git a/server/src/main/java/org/elasticsearch/features/FeatureService.java b/server/src/main/java/org/elasticsearch/features/FeatureService.java index c04fbae05ee2c..da71b8f0ec2f7 100644 --- a/server/src/main/java/org/elasticsearch/features/FeatureService.java +++ b/server/src/main/java/org/elasticsearch/features/FeatureService.java @@ -29,7 +29,7 @@ public class FeatureService { /** * A feature indicating that node features are supported. */ - public static final NodeFeature FEATURES_SUPPORTED = new NodeFeature("features_supported"); + public static final NodeFeature FEATURES_SUPPORTED = new NodeFeature("features_supported", true); public static final NodeFeature TEST_FEATURES_ENABLED = new NodeFeature("test_features_enabled"); private static final Logger logger = LogManager.getLogger(FeatureService.class); diff --git a/server/src/main/java/org/elasticsearch/features/NodeFeature.java b/server/src/main/java/org/elasticsearch/features/NodeFeature.java index 961b386d62802..ad270540274b9 100644 --- a/server/src/main/java/org/elasticsearch/features/NodeFeature.java +++ b/server/src/main/java/org/elasticsearch/features/NodeFeature.java @@ -17,7 +17,7 @@ * @param id The feature id. Must be unique in the node. * @param assumedAfterNextCompatibilityBoundary * {@code true} if this feature is removed at the next compatibility boundary (ie next major version), - * and so should be assumed to be true for all nodes after that boundary. + * and so should be assumed to be met by all nodes after that boundary, even if they don't publish it. */ public record NodeFeature(String id, boolean assumedAfterNextCompatibilityBoundary) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index 2168ad1df5d2f..7d63a0432cdbc 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -203,8 +203,9 @@ public IndexModule( this.engineFactory = Objects.requireNonNull(engineFactory); // Need to have a mutable arraylist for plugins to add listeners to it this.searchOperationListeners = new ArrayList<>(searchOperationListeners); - this.searchOperationListeners.add(new SearchSlowLog(indexSettings, slowLogFieldProvider)); - this.indexOperationListeners.add(new IndexingSlowLog(indexSettings, slowLogFieldProvider)); + SlowLogFields slowLogFields = slowLogFieldProvider.create(indexSettings); + this.searchOperationListeners.add(new SearchSlowLog(indexSettings, slowLogFields)); + this.indexOperationListeners.add(new IndexingSlowLog(indexSettings, slowLogFields)); this.directoryFactories = Collections.unmodifiableMap(directoryFactories); this.allowExpensiveQueries = allowExpensiveQueries; this.expressionResolver = expressionResolver; diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 7a5cd97e5a3a3..8d6404e0530e5 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -133,6 +133,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion V8_DEPRECATE_SOURCE_MODE_MAPPER = def(8_521_00_0, Version.LUCENE_9_12_0); public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BACKPORT = def(8_522_00_0, Version.LUCENE_9_12_0); public static final IndexVersion UPGRADE_TO_LUCENE_9_12_1 = def(8_523_00_0, parseUnchecked("9.12.1")); + public static final IndexVersion INFERENCE_METADATA_FIELDS_BACKPORT = def(8_524_00_0, parseUnchecked("9.12.1")); public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_00_0, Version.LUCENE_10_0_0); public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT = def(9_001_00_0, Version.LUCENE_10_0_0); public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID = def(9_002_00_0, Version.LUCENE_10_0_0); diff --git a/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java b/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java index 3ae4c0eb82ad0..5a7990a4e70c5 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java +++ b/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java @@ -103,7 +103,7 @@ public final class IndexingSlowLog implements IndexingOperationListener { * characters of the source. */ private int maxSourceCharsToLog; - private final SlowLogFieldProvider slowLogFieldProvider; + private final SlowLogFields slowLogFields; /** * Reads how much of the source to log. The user can specify any value they @@ -125,8 +125,8 @@ public final class IndexingSlowLog implements IndexingOperationListener { Property.IndexScope ); - IndexingSlowLog(IndexSettings indexSettings, SlowLogFieldProvider slowLogFieldProvider) { - this.slowLogFieldProvider = slowLogFieldProvider; + IndexingSlowLog(IndexSettings indexSettings, SlowLogFields slowLogFields) { + this.slowLogFields = slowLogFields; this.index = indexSettings.getIndex(); indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING, this::setReformat); @@ -179,47 +179,19 @@ public void postIndex(ShardId shardId, Engine.Index indexOperation, Engine.Index final long tookInNanos = result.getTook(); if (indexWarnThreshold >= 0 && tookInNanos > indexWarnThreshold) { indexLogger.warn( - IndexingSlowLogMessage.of( - this.slowLogFieldProvider.indexSlowLogFields(), - index, - doc, - tookInNanos, - reformat, - maxSourceCharsToLog - ) + IndexingSlowLogMessage.of(this.slowLogFields.indexFields(), index, doc, tookInNanos, reformat, maxSourceCharsToLog) ); } else if (indexInfoThreshold >= 0 && tookInNanos > indexInfoThreshold) { indexLogger.info( - IndexingSlowLogMessage.of( - this.slowLogFieldProvider.indexSlowLogFields(), - index, - doc, - tookInNanos, - reformat, - maxSourceCharsToLog - ) + IndexingSlowLogMessage.of(this.slowLogFields.indexFields(), index, doc, tookInNanos, reformat, maxSourceCharsToLog) ); } else if (indexDebugThreshold >= 0 && tookInNanos > indexDebugThreshold) { indexLogger.debug( - IndexingSlowLogMessage.of( - this.slowLogFieldProvider.indexSlowLogFields(), - index, - doc, - tookInNanos, - reformat, - maxSourceCharsToLog - ) + IndexingSlowLogMessage.of(this.slowLogFields.indexFields(), index, doc, tookInNanos, reformat, maxSourceCharsToLog) ); } else if (indexTraceThreshold >= 0 && tookInNanos > indexTraceThreshold) { indexLogger.trace( - IndexingSlowLogMessage.of( - this.slowLogFieldProvider.indexSlowLogFields(), - index, - doc, - tookInNanos, - reformat, - maxSourceCharsToLog - ) + IndexingSlowLogMessage.of(this.slowLogFields.indexFields(), index, doc, tookInNanos, reformat, maxSourceCharsToLog) ); } } diff --git a/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java b/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java index e4836a391bfec..81e7cff862e32 100644 --- a/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java +++ b/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java @@ -45,7 +45,7 @@ public final class SearchSlowLog implements SearchOperationListener { private static final Logger queryLogger = LogManager.getLogger(INDEX_SEARCH_SLOWLOG_PREFIX + ".query"); private static final Logger fetchLogger = LogManager.getLogger(INDEX_SEARCH_SLOWLOG_PREFIX + ".fetch"); - private final SlowLogFieldProvider slowLogFieldProvider; + private final SlowLogFields slowLogFields; public static final Setting INDEX_SEARCH_SLOWLOG_INCLUDE_USER_SETTING = Setting.boolSetting( INDEX_SEARCH_SLOWLOG_PREFIX + ".include.user", @@ -126,9 +126,8 @@ public final class SearchSlowLog implements SearchOperationListener { private static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("pretty", "false")); - public SearchSlowLog(IndexSettings indexSettings, SlowLogFieldProvider slowLogFieldProvider) { - slowLogFieldProvider.init(indexSettings); - this.slowLogFieldProvider = slowLogFieldProvider; + public SearchSlowLog(IndexSettings indexSettings, SlowLogFields slowLogFields) { + this.slowLogFields = slowLogFields; indexSettings.getScopedSettings() .addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING, this::setQueryWarnThreshold); this.queryWarnThreshold = indexSettings.getValue(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING).nanos(); @@ -159,26 +158,26 @@ public SearchSlowLog(IndexSettings indexSettings, SlowLogFieldProvider slowLogFi @Override public void onQueryPhase(SearchContext context, long tookInNanos) { if (queryWarnThreshold >= 0 && tookInNanos > queryWarnThreshold) { - queryLogger.warn(SearchSlowLogMessage.of(this.slowLogFieldProvider.searchSlowLogFields(), context, tookInNanos)); + queryLogger.warn(SearchSlowLogMessage.of(this.slowLogFields.searchFields(), context, tookInNanos)); } else if (queryInfoThreshold >= 0 && tookInNanos > queryInfoThreshold) { - queryLogger.info(SearchSlowLogMessage.of(this.slowLogFieldProvider.searchSlowLogFields(), context, tookInNanos)); + queryLogger.info(SearchSlowLogMessage.of(this.slowLogFields.searchFields(), context, tookInNanos)); } else if (queryDebugThreshold >= 0 && tookInNanos > queryDebugThreshold) { - queryLogger.debug(SearchSlowLogMessage.of(this.slowLogFieldProvider.searchSlowLogFields(), context, tookInNanos)); + queryLogger.debug(SearchSlowLogMessage.of(this.slowLogFields.searchFields(), context, tookInNanos)); } else if (queryTraceThreshold >= 0 && tookInNanos > queryTraceThreshold) { - queryLogger.trace(SearchSlowLogMessage.of(this.slowLogFieldProvider.searchSlowLogFields(), context, tookInNanos)); + queryLogger.trace(SearchSlowLogMessage.of(this.slowLogFields.searchFields(), context, tookInNanos)); } } @Override public void onFetchPhase(SearchContext context, long tookInNanos) { if (fetchWarnThreshold >= 0 && tookInNanos > fetchWarnThreshold) { - fetchLogger.warn(SearchSlowLogMessage.of(this.slowLogFieldProvider.searchSlowLogFields(), context, tookInNanos)); + fetchLogger.warn(SearchSlowLogMessage.of(this.slowLogFields.searchFields(), context, tookInNanos)); } else if (fetchInfoThreshold >= 0 && tookInNanos > fetchInfoThreshold) { - fetchLogger.info(SearchSlowLogMessage.of(this.slowLogFieldProvider.searchSlowLogFields(), context, tookInNanos)); + fetchLogger.info(SearchSlowLogMessage.of(this.slowLogFields.searchFields(), context, tookInNanos)); } else if (fetchDebugThreshold >= 0 && tookInNanos > fetchDebugThreshold) { - fetchLogger.debug(SearchSlowLogMessage.of(this.slowLogFieldProvider.searchSlowLogFields(), context, tookInNanos)); + fetchLogger.debug(SearchSlowLogMessage.of(this.slowLogFields.searchFields(), context, tookInNanos)); } else if (fetchTraceThreshold >= 0 && tookInNanos > fetchTraceThreshold) { - fetchLogger.trace(SearchSlowLogMessage.of(this.slowLogFieldProvider.searchSlowLogFields(), context, tookInNanos)); + fetchLogger.trace(SearchSlowLogMessage.of(this.slowLogFields.searchFields(), context, tookInNanos)); } } diff --git a/server/src/main/java/org/elasticsearch/index/SlowLogFieldProvider.java b/server/src/main/java/org/elasticsearch/index/SlowLogFieldProvider.java index c61d4d4c85a86..e93edccc83b15 100644 --- a/server/src/main/java/org/elasticsearch/index/SlowLogFieldProvider.java +++ b/server/src/main/java/org/elasticsearch/index/SlowLogFieldProvider.java @@ -9,28 +9,14 @@ package org.elasticsearch.index; -import java.util.Map; - /** * Interface for providing additional fields to the slow log from a plugin. * Intended to be loaded through SPI. */ public interface SlowLogFieldProvider { /** - * Initialize field provider with index level settings to be able to listen for updates and set initial values + * Create a field provider with index level settings to be able to listen for updates and set initial values * @param indexSettings settings for the index */ - void init(IndexSettings indexSettings); - - /** - * Slow log fields for indexing events - * @return map of field name to value - */ - Map indexSlowLogFields(); - - /** - * Slow log fields for search events - * @return map of field name to value - */ - Map searchSlowLogFields(); + SlowLogFields create(IndexSettings indexSettings); } diff --git a/server/src/main/java/org/elasticsearch/index/SlowLogFields.java b/server/src/main/java/org/elasticsearch/index/SlowLogFields.java new file mode 100644 index 0000000000000..e018e3a4d6bb7 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/SlowLogFields.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index; + +import java.util.Map; + +/** + * Fields for the slow log. These may be different each call depending on the state of the system. + */ +public interface SlowLogFields { + + /** + * Slow log fields for indexing events + * @return map of field name to value + */ + Map indexFields(); + + /** + * Slow log fields for search events + * @return map of field name to value + */ + Map searchFields(); +} diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 0a470e86ef856..8d3d1bde316ea 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -661,7 +661,8 @@ private Translog openTranslog( translogDeletionPolicy, globalCheckpointSupplier, engineConfig.getPrimaryTermSupplier(), - persistedSequenceNumberConsumer + persistedSequenceNumberConsumer, + TranslogOperationAsserter.withEngineConfig(engineConfig) ); } diff --git a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java index 49e0ae0587085..044e6f6712c77 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java @@ -166,7 +166,8 @@ public void trimUnreferencedTranslogFiles() { translogDeletionPolicy, engineConfig.getGlobalCheckpointSupplier(), engineConfig.getPrimaryTermSupplier(), - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ) ) { translog.trimUnreferencedReaders(); diff --git a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java index c3ab2ee910805..010fc1bd9e411 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java @@ -267,7 +267,8 @@ private static TranslogStats translogStats(final EngineConfig config, final Segm translogDeletionPolicy, config.getGlobalCheckpointSupplier(), config.getPrimaryTermSupplier(), - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ) ) { return translog.stats(); diff --git a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java index b1c311af88e2f..0928b4500e6da 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java @@ -99,10 +99,6 @@ private static UnsupportedOperationException unsupported() { return new UnsupportedOperationException(); } - public TranslogLeafReader getLeafReader() { - return leafReader; - } - @Override protected DirectoryReader doOpenIfChanged() { throw unsupported(); @@ -143,6 +139,45 @@ public CacheHelper getReaderCacheHelper() { return leafReader.getReaderCacheHelper(); } + static DirectoryReader createInMemoryReader( + ShardId shardId, + EngineConfig engineConfig, + Directory directory, + DocumentParser documentParser, + MappingLookup mappingLookup, + Translog.Index operation + ) { + final ParsedDocument parsedDocs = documentParser.parseDocument( + new SourceToParse(operation.id(), operation.source(), XContentHelper.xContentType(operation.source()), operation.routing()), + mappingLookup + ); + + parsedDocs.updateSeqID(operation.seqNo(), operation.primaryTerm()); + parsedDocs.version().setLongValue(operation.version()); + // To guarantee indexability, we configure the analyzer and codec using the main engine configuration + final IndexWriterConfig writeConfig = new IndexWriterConfig(engineConfig.getAnalyzer()).setOpenMode( + IndexWriterConfig.OpenMode.CREATE + ).setCodec(engineConfig.getCodec()); + try (IndexWriter writer = new IndexWriter(directory, writeConfig)) { + writer.addDocument(parsedDocs.rootDoc()); + final DirectoryReader reader = open(writer); + if (reader.leaves().size() != 1 || reader.leaves().get(0).reader().numDocs() != 1) { + reader.close(); + throw new IllegalStateException( + "Expected a single document segment; " + + "but [" + + reader.leaves().size() + + " segments with " + + reader.leaves().get(0).reader().numDocs() + + " documents" + ); + } + return reader; + } catch (IOException e) { + throw new EngineException(shardId, "failed to create an in-memory segment for get [" + operation.id() + "]", e); + } + } + private static class TranslogLeafReader extends LeafReader { private static final FieldInfo FAKE_SOURCE_FIELD = new FieldInfo( @@ -244,7 +279,8 @@ private LeafReader getDelegate() { ensureOpen(); reader = delegate.get(); if (reader == null) { - reader = createInMemoryLeafReader(); + var indexReader = createInMemoryReader(shardId, engineConfig, directory, documentParser, mappingLookup, operation); + reader = indexReader.leaves().get(0).reader(); final LeafReader existing = delegate.getAndSet(reader); assert existing == null; onSegmentCreated.run(); @@ -254,39 +290,6 @@ private LeafReader getDelegate() { return reader; } - private LeafReader createInMemoryLeafReader() { - assert Thread.holdsLock(this); - final ParsedDocument parsedDocs = documentParser.parseDocument( - new SourceToParse(operation.id(), operation.source(), XContentHelper.xContentType(operation.source()), operation.routing()), - mappingLookup - ); - - parsedDocs.updateSeqID(operation.seqNo(), operation.primaryTerm()); - parsedDocs.version().setLongValue(operation.version()); - // To guarantee indexability, we configure the analyzer and codec using the main engine configuration - final IndexWriterConfig writeConfig = new IndexWriterConfig(engineConfig.getAnalyzer()).setOpenMode( - IndexWriterConfig.OpenMode.CREATE - ).setCodec(engineConfig.getCodec()); - try (IndexWriter writer = new IndexWriter(directory, writeConfig)) { - writer.addDocument(parsedDocs.rootDoc()); - final DirectoryReader reader = open(writer); - if (reader.leaves().size() != 1 || reader.leaves().get(0).reader().numDocs() != 1) { - reader.close(); - throw new IllegalStateException( - "Expected a single document segment; " - + "but [" - + reader.leaves().size() - + " segments with " - + reader.leaves().get(0).reader().numDocs() - + " documents" - ); - } - return reader.leaves().get(0).reader(); - } catch (IOException e) { - throw new EngineException(shardId, "failed to create an in-memory segment for get [" + operation.id() + "]", e); - } - } - @Override public CacheHelper getCoreCacheHelper() { return getDelegate().getCoreCacheHelper(); diff --git a/server/src/main/java/org/elasticsearch/index/engine/TranslogOperationAsserter.java b/server/src/main/java/org/elasticsearch/index/engine/TranslogOperationAsserter.java new file mode 100644 index 0000000000000..00de13b8e8d8e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/engine/TranslogOperationAsserter.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.engine; + +import org.apache.lucene.search.similarities.BM25Similarity; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.elasticsearch.index.cache.query.TrivialQueryCachingPolicy; +import org.elasticsearch.index.mapper.DocumentParser; +import org.elasticsearch.index.mapper.MappingLookup; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.translog.Translog; + +import java.io.IOException; + +/** + * + * A utility class to assert that translog operations with the same sequence number + * in the same generation are either identical or equivalent when synthetic sources are used. + */ +public abstract class TranslogOperationAsserter { + public static final TranslogOperationAsserter DEFAULT = new TranslogOperationAsserter() { + }; + + private TranslogOperationAsserter() { + + } + + public static TranslogOperationAsserter withEngineConfig(EngineConfig engineConfig) { + return new TranslogOperationAsserter() { + @Override + public boolean assertSameIndexOperation(Translog.Index o1, Translog.Index o2) throws IOException { + if (super.assertSameIndexOperation(o1, o2)) { + return true; + } + if (engineConfig.getIndexSettings().isRecoverySourceSyntheticEnabled()) { + return super.assertSameIndexOperation(synthesizeSource(engineConfig, o1), o2) + || super.assertSameIndexOperation(o1, synthesizeSource(engineConfig, o2)); + } + return false; + } + }; + } + + static Translog.Index synthesizeSource(EngineConfig engineConfig, Translog.Index op) throws IOException { + final ShardId shardId = engineConfig.getShardId(); + final MappingLookup mappingLookup = engineConfig.getMapperService().mappingLookup(); + final DocumentParser documentParser = engineConfig.getMapperService().documentParser(); + try ( + var directory = new ByteBuffersDirectory(); + var reader = TranslogDirectoryReader.createInMemoryReader(shardId, engineConfig, directory, documentParser, mappingLookup, op) + ) { + final Engine.Searcher searcher = new Engine.Searcher( + "assert_translog", + reader, + new BM25Similarity(), + null, + TrivialQueryCachingPolicy.NEVER, + () -> {} + ); + try ( + LuceneSyntheticSourceChangesSnapshot snapshot = new LuceneSyntheticSourceChangesSnapshot( + mappingLookup, + searcher, + LuceneSyntheticSourceChangesSnapshot.DEFAULT_BATCH_SIZE, + Integer.MAX_VALUE, + op.seqNo(), + op.seqNo(), + true, + false, + engineConfig.getIndexSettings().getIndexVersionCreated() + ) + ) { + final Translog.Operation normalized = snapshot.next(); + assert normalized != null : "expected one operation; got zero"; + return (Translog.Index) normalized; + } + } + } + + public boolean assertSameIndexOperation(Translog.Index o1, Translog.Index o2) throws IOException { + return Translog.Index.equalsWithoutAutoGeneratedTimestamp(o1, o2); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CustomSyntheticSourceFieldLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/CustomSyntheticSourceFieldLookup.java deleted file mode 100644 index dbbee8a9035d4..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/mapper/CustomSyntheticSourceFieldLookup.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.mapper; - -import org.elasticsearch.core.Nullable; -import org.elasticsearch.index.IndexSettings; - -import java.util.HashMap; -import java.util.Map; - -/** - * Contains lookup information needed to perform custom synthetic source logic. - * For example fields that use fallback synthetic source implementation or fields that preserve array ordering - * in synthetic source; - */ -public class CustomSyntheticSourceFieldLookup { - private final Map fieldsWithCustomSyntheticSourceHandling; - - public CustomSyntheticSourceFieldLookup(Mapping mapping, @Nullable IndexSettings indexSettings, boolean isSourceSynthetic) { - var fields = new HashMap(); - if (isSourceSynthetic && indexSettings != null) { - populateFields(fields, mapping.getRoot(), indexSettings.sourceKeepMode()); - } - this.fieldsWithCustomSyntheticSourceHandling = Map.copyOf(fields); - } - - private void populateFields(Map fields, ObjectMapper currentLevel, Mapper.SourceKeepMode defaultSourceKeepMode) { - if (currentLevel.isEnabled() == false) { - fields.put(currentLevel.fullPath(), Reason.DISABLED_OBJECT); - return; - } - if (sourceKeepMode(currentLevel, defaultSourceKeepMode) == Mapper.SourceKeepMode.ALL) { - fields.put(currentLevel.fullPath(), Reason.SOURCE_KEEP_ALL); - return; - } - if (currentLevel.isNested() == false && sourceKeepMode(currentLevel, defaultSourceKeepMode) == Mapper.SourceKeepMode.ARRAYS) { - fields.put(currentLevel.fullPath(), Reason.SOURCE_KEEP_ARRAYS); - } - - for (Mapper child : currentLevel) { - if (child instanceof ObjectMapper objectMapper) { - populateFields(fields, objectMapper, defaultSourceKeepMode); - } else if (child instanceof FieldMapper fieldMapper) { - // The order here is important. - // If fallback logic is used, it should be always correctly marked as FALLBACK_SYNTHETIC_SOURCE. - // This allows us to apply an optimization for SOURCE_KEEP_ARRAYS and don't store arrays that have one element. - // If this order is changed and a field that both has SOURCE_KEEP_ARRAYS and FALLBACK_SYNTHETIC_SOURCE - // is marked as SOURCE_KEEP_ARRAYS we would lose data for this field by applying such an optimization. - if (fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK) { - fields.put(fieldMapper.fullPath(), Reason.FALLBACK_SYNTHETIC_SOURCE); - } else if (sourceKeepMode(fieldMapper, defaultSourceKeepMode) == Mapper.SourceKeepMode.ALL) { - fields.put(fieldMapper.fullPath(), Reason.SOURCE_KEEP_ALL); - } else if (sourceKeepMode(fieldMapper, defaultSourceKeepMode) == Mapper.SourceKeepMode.ARRAYS) { - fields.put(fieldMapper.fullPath(), Reason.SOURCE_KEEP_ARRAYS); - } - } - } - } - - private Mapper.SourceKeepMode sourceKeepMode(ObjectMapper mapper, Mapper.SourceKeepMode defaultSourceKeepMode) { - return mapper.sourceKeepMode().orElse(defaultSourceKeepMode); - } - - private Mapper.SourceKeepMode sourceKeepMode(FieldMapper mapper, Mapper.SourceKeepMode defaultSourceKeepMode) { - return mapper.sourceKeepMode().orElse(defaultSourceKeepMode); - } - - public Map getFieldsWithCustomSyntheticSourceHandling() { - return fieldsWithCustomSyntheticSourceHandling; - } - - /** - * Specifies why this field needs custom handling. - */ - public enum Reason { - SOURCE_KEEP_ARRAYS, - SOURCE_KEEP_ALL, - FALLBACK_SYNTHETIC_SOURCE, - DISABLED_OBJECT - } -} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 5abb7b5a1b728..03e6c343c7ab9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -49,7 +49,6 @@ public static DocumentMapper createEmpty(MapperService mapperService) { mapping, mapping.toCompressedXContent(), IndexVersion.current(), - mapperService.getIndexSettings(), mapperService.getMapperMetrics(), mapperService.index().getName() ); @@ -60,13 +59,12 @@ public static DocumentMapper createEmpty(MapperService mapperService) { Mapping mapping, CompressedXContent source, IndexVersion version, - IndexSettings indexSettings, MapperMetrics mapperMetrics, String indexName ) { this.documentParser = documentParser; this.type = mapping.getRoot().fullPath(); - this.mappingLookup = MappingLookup.fromMapping(mapping, indexSettings); + this.mappingLookup = MappingLookup.fromMapping(mapping); this.mappingSource = source; this.mapperMetrics = mapperMetrics; this.indexVersion = version; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 0fc14d4dbeeb9..9ddb6f0d496a0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -27,7 +27,6 @@ import org.elasticsearch.plugins.internal.XContentMeteringParserDecorator; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.Source; -import org.elasticsearch.xcontent.FilterXContentParserWrapper; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentLocation; import org.elasticsearch.xcontent.XContentParseException; @@ -44,7 +43,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.function.BiFunction; import java.util.function.Consumer; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; @@ -62,22 +60,10 @@ public final class DocumentParser { private final XContentParserConfiguration parserConfiguration; private final MappingParserContext mappingParserContext; - private final BiFunction listenersFactory; DocumentParser(XContentParserConfiguration parserConfiguration, MappingParserContext mappingParserContext) { this.mappingParserContext = mappingParserContext; this.parserConfiguration = parserConfiguration; - this.listenersFactory = this::createDefaultListeners; - } - - DocumentParser( - XContentParserConfiguration parserConfiguration, - MappingParserContext mappingParserContext, - BiFunction listenersFactory - ) { - this.mappingParserContext = mappingParserContext; - this.parserConfiguration = parserConfiguration; - this.listenersFactory = listenersFactory; } /** @@ -95,11 +81,13 @@ public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingL final RootDocumentParserContext context; final XContentType xContentType = source.getXContentType(); - Listeners listeners = listenersFactory.apply(mappingLookup, xContentType); - XContentMeteringParserDecorator meteringParserDecorator = source.getMeteringParserDecorator(); - try (XContentParser parser = meteringParserDecorator.decorate(createParser(source, xContentType, listeners))) { - context = new RootDocumentParserContext(mappingLookup, mappingParserContext, source, listeners, parser); + try ( + XContentParser parser = meteringParserDecorator.decorate( + XContentHelper.createParser(parserConfiguration, source.source(), xContentType) + ) + ) { + context = new RootDocumentParserContext(mappingLookup, mappingParserContext, source, parser); validateStart(context.parser()); MetadataFieldMapper[] metadataFieldsMappers = mappingLookup.getMapping().getSortedMetadataMappers(); internalParseDocument(metadataFieldsMappers, context); @@ -133,152 +121,6 @@ public String documentDescription() { }; } - private Listeners createDefaultListeners(MappingLookup mappingLookup, XContentType xContentType) { - if (mappingLookup.isSourceSynthetic() && mappingParserContext.getIndexSettings().getSkipIgnoredSourceWrite() == false) { - return new Listeners.Single(new SyntheticSourceDocumentParserListener(mappingLookup, xContentType)); - } - - return Listeners.NOOP; - } - - private XContentParser createParser(SourceToParse sourceToParse, XContentType xContentType, Listeners listeners) throws IOException { - XContentParser plainParser = XContentHelper.createParser(parserConfiguration, sourceToParse.source(), xContentType); - - if (listeners.isNoop()) { - return plainParser; - } - - return new ListenerAwareXContentParser(plainParser, listeners); - } - - static class ListenerAwareXContentParser extends FilterXContentParserWrapper { - private final Listeners listeners; - - ListenerAwareXContentParser(XContentParser parser, Listeners listeners) { - super(parser); - this.listeners = listeners; - } - - @Override - public Token nextToken() throws IOException { - var token = delegate().nextToken(); - - if (listeners.anyActive()) { - var listenerToken = DocumentParserListener.Token.current(delegate()); - listeners.publish(listenerToken); - } - - return token; - } - - @Override - public void skipChildren() throws IOException { - // We can not use "native" implementation because some listeners may want to see - // skipped parts. - Token token = currentToken(); - if (token != Token.START_OBJECT && token != Token.START_ARRAY) { - return; - } - - int depth = 0; - while (token != null) { - if (token == Token.START_OBJECT || token == Token.START_ARRAY) { - depth += 1; - } - if (token == Token.END_OBJECT || token == Token.END_ARRAY) { - depth -= 1; - if (depth == 0) { - return; - } - } - - token = nextToken(); - } - } - } - - /** - * Encapsulates listeners that are subscribed to this document parser. This allows to generalize logic without knowing - * how many listeners are present (and if they are present at all). - */ - public interface Listeners { - void publish(DocumentParserListener.Event event, DocumentParserContext context) throws IOException; - - void publish(DocumentParserListener.Token token) throws IOException; - - DocumentParserListener.Output finish(); - - boolean isNoop(); - - boolean anyActive(); - - /** - * No listeners are present. - */ - Listeners NOOP = new Listeners() { - @Override - public void publish(DocumentParserListener.Event event, DocumentParserContext context) {} - - @Override - public void publish(DocumentParserListener.Token token) { - - } - - @Override - public DocumentParserListener.Output finish() { - return DocumentParserListener.Output.empty(); - } - - @Override - public boolean isNoop() { - return true; - } - - @Override - public boolean anyActive() { - return false; - } - }; - - /** - * One or more listeners are present. - */ - class Single implements Listeners { - private final DocumentParserListener listener; - - public Single(DocumentParserListener listener) { - this.listener = listener; - } - - @Override - public void publish(DocumentParserListener.Event event, DocumentParserContext context) throws IOException { - listener.consume(event); - } - - @Override - public void publish(DocumentParserListener.Token token) throws IOException { - if (listener.isActive()) { - listener.consume(token); - } - } - - @Override - public DocumentParserListener.Output finish() { - return listener.finish(); - } - - @Override - public boolean isNoop() { - return false; - } - - @Override - public boolean anyActive() { - return listener.isActive(); - } - } - } - private void internalParseDocument(MetadataFieldMapper[] metadataFieldsMappers, DocumentParserContext context) { try { final boolean emptyDoc = isEmptyDoc(context.root(), context.parser()); @@ -287,19 +129,26 @@ private void internalParseDocument(MetadataFieldMapper[] metadataFieldsMappers, metadataMapper.preParse(context); } - context.publishEvent(new DocumentParserListener.Event.DocumentStart(context.root(), context.doc())); - if (context.root().isEnabled() == false) { // entire type is disabled - context.parser().skipChildren(); + if (context.canAddIgnoredField()) { + context.addIgnoredField( + new IgnoredSourceFieldMapper.NameValue( + MapperService.SINGLE_MAPPING_NAME, + 0, + context.encodeFlattenedToken(), + context.doc() + ) + ); + } else { + context.parser().skipChildren(); + } } else if (emptyDoc == false) { parseObjectOrNested(context); } executeIndexTimeScripts(context); - context.finishListeners(); - for (MetadataFieldMapper metadataMapper : metadataFieldsMappers) { metadataMapper.postParse(context); } @@ -425,12 +274,22 @@ static Mapping createDynamicUpdate(DocumentParserContext context) { } static void parseObjectOrNested(DocumentParserContext context) throws IOException { - XContentParser parser = context.parser(); String currentFieldName = parser.currentName(); if (context.parent().isEnabled() == false) { // entire type is disabled - parser.skipChildren(); + if (context.canAddIgnoredField()) { + context.addIgnoredField( + new IgnoredSourceFieldMapper.NameValue( + context.parent().fullPath(), + context.parent().fullPath().lastIndexOf(context.parent().leafName()), + context.encodeFlattenedToken(), + context.doc() + ) + ); + } else { + parser.skipChildren(); + } return; } XContentParser.Token token = parser.currentToken(); @@ -443,6 +302,22 @@ static void parseObjectOrNested(DocumentParserContext context) throws IOExceptio throwOnConcreteValue(context.parent(), currentFieldName, context); } + var sourceKeepMode = getSourceKeepMode(context, context.parent().sourceKeepMode()); + if (context.canAddIgnoredField() + && (sourceKeepMode == Mapper.SourceKeepMode.ALL + || (sourceKeepMode == Mapper.SourceKeepMode.ARRAYS && context.inArrayScope()))) { + context = context.addIgnoredFieldFromContext( + new IgnoredSourceFieldMapper.NameValue( + context.parent().fullPath(), + context.parent().fullPath().lastIndexOf(context.parent().leafName()), + null, + context.doc() + ) + ); + token = context.parser().currentToken(); + parser = context.parser(); + } + if (context.parent().isNested()) { // Handle a nested object that doesn't contain an array. Arrays are handled in #parseNonDynamicArray. context = context.createNestedContext((NestedObjectMapper) context.parent()); @@ -566,9 +441,6 @@ private static void addFields(IndexVersion indexCreatedVersion, LuceneDocument n static void parseObjectOrField(DocumentParserContext context, Mapper mapper) throws IOException { if (mapper instanceof ObjectMapper objectMapper) { - context.publishEvent( - new DocumentParserListener.Event.ObjectStart(objectMapper, context.inArrayScope(), context.parent(), context.doc()) - ); parseObjectOrNested(context.createChildContext(objectMapper)); } else if (mapper instanceof FieldMapper fieldMapper) { if (shouldFlattenObject(context, fieldMapper)) { @@ -578,19 +450,12 @@ static void parseObjectOrField(DocumentParserContext context, Mapper mapper) thr parseObjectOrNested(context.createFlattenContext(currentFieldName)); context.path().add(currentFieldName); } else { - context.publishEvent( - new DocumentParserListener.Event.LeafValue( - fieldMapper, - context.inArrayScope(), - context.parent(), - context.doc(), - context.parser() - ) - ); - + var sourceKeepMode = getSourceKeepMode(context, fieldMapper.sourceKeepMode()); if (context.canAddIgnoredField() - && context.isWithinCopyTo() == false - && context.isCopyToDestinationField(mapper.fullPath())) { + && (fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK + || sourceKeepMode == Mapper.SourceKeepMode.ALL + || (sourceKeepMode == Mapper.SourceKeepMode.ARRAYS && context.inArrayScope()) + || (context.isWithinCopyTo() == false && context.isCopyToDestinationField(mapper.fullPath())))) { context = context.addIgnoredFieldFromContext( IgnoredSourceFieldMapper.NameValue.fromContext(context, fieldMapper.fullPath(), null) ); @@ -820,17 +685,40 @@ private static void parseNonDynamicArray( ) throws IOException { String fullPath = context.path().pathAsText(arrayFieldName); - if (mapper instanceof ObjectMapper objectMapper) { - context.publishEvent(new DocumentParserListener.Event.ObjectArrayStart(objectMapper, context.parent(), context.doc())); - } else if (mapper instanceof FieldMapper fieldMapper) { - context.publishEvent(new DocumentParserListener.Event.LeafArrayStart(fieldMapper, context.parent(), context.doc())); - } - // Check if we need to record the array source. This only applies to synthetic source. + boolean canRemoveSingleLeafElement = false; if (context.canAddIgnoredField()) { + Mapper.SourceKeepMode mode = Mapper.SourceKeepMode.NONE; + boolean objectWithFallbackSyntheticSource = false; + if (mapper instanceof ObjectMapper objectMapper) { + mode = getSourceKeepMode(context, objectMapper.sourceKeepMode()); + objectWithFallbackSyntheticSource = mode == Mapper.SourceKeepMode.ALL + || (mode == Mapper.SourceKeepMode.ARRAYS && objectMapper instanceof NestedObjectMapper == false); + } + boolean fieldWithFallbackSyntheticSource = false; + boolean fieldWithStoredArraySource = false; + if (mapper instanceof FieldMapper fieldMapper) { + mode = getSourceKeepMode(context, fieldMapper.sourceKeepMode()); + fieldWithFallbackSyntheticSource = fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK; + fieldWithStoredArraySource = mode != Mapper.SourceKeepMode.NONE; + } boolean copyToFieldHasValuesInDocument = context.isWithinCopyTo() == false && context.isCopyToDestinationField(fullPath); - if (copyToFieldHasValuesInDocument) { + + canRemoveSingleLeafElement = mapper instanceof FieldMapper + && mode == Mapper.SourceKeepMode.ARRAYS + && fieldWithFallbackSyntheticSource == false + && copyToFieldHasValuesInDocument == false; + + if (objectWithFallbackSyntheticSource + || fieldWithFallbackSyntheticSource + || fieldWithStoredArraySource + || copyToFieldHasValuesInDocument) { context = context.addIgnoredFieldFromContext(IgnoredSourceFieldMapper.NameValue.fromContext(context, fullPath, null)); + } else if (mapper instanceof ObjectMapper objectMapper && (objectMapper.isEnabled() == false)) { + // No need to call #addIgnoredFieldFromContext as both singleton and array instances of this object + // get tracked through ignored source. + context.addIgnoredField(IgnoredSourceFieldMapper.NameValue.fromContext(context, fullPath, context.encodeFlattenedToken())); + return; } } @@ -841,20 +729,28 @@ private static void parseNonDynamicArray( XContentParser parser = context.parser(); XContentParser.Token token; + int elements = 0; while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token == XContentParser.Token.START_OBJECT) { + elements = Integer.MAX_VALUE; parseObject(context, lastFieldName); } else if (token == XContentParser.Token.START_ARRAY) { + elements = Integer.MAX_VALUE; parseArray(context, lastFieldName); } else if (token == XContentParser.Token.VALUE_NULL) { + elements++; parseNullValue(context, lastFieldName); } else if (token == null) { throwEOFOnParseArray(arrayFieldName, context); } else { assert token.isValue(); + elements++; parseValue(context, lastFieldName); } } + if (elements <= 1 && canRemoveSingleLeafElement) { + context.removeLastIgnoredField(fullPath); + } postProcessDynamicArrayMapping(context, lastFieldName); } @@ -1153,14 +1049,12 @@ private static class RootDocumentParserContext extends DocumentParserContext { MappingLookup mappingLookup, MappingParserContext mappingParserContext, SourceToParse source, - Listeners listeners, XContentParser parser ) throws IOException { super( mappingLookup, mappingParserContext, source, - listeners, mappingLookup.getMapping().getRoot(), ObjectMapper.Dynamic.getRootDynamic(mappingLookup) ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 625f3a3d19af7..51e4e9f4c1b5e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -117,7 +117,6 @@ private enum Scope { private final MappingLookup mappingLookup; private final MappingParserContext mappingParserContext; private final SourceToParse sourceToParse; - private final DocumentParser.Listeners listeners; private final Set ignoredFields; private final List ignoredFieldValues; @@ -150,7 +149,6 @@ private DocumentParserContext( MappingLookup mappingLookup, MappingParserContext mappingParserContext, SourceToParse sourceToParse, - DocumentParser.Listeners listeners, Set ignoreFields, List ignoredFieldValues, Scope currentScope, @@ -171,7 +169,6 @@ private DocumentParserContext( this.mappingLookup = mappingLookup; this.mappingParserContext = mappingParserContext; this.sourceToParse = sourceToParse; - this.listeners = listeners; this.ignoredFields = ignoreFields; this.ignoredFieldValues = ignoredFieldValues; this.currentScope = currentScope; @@ -195,7 +192,6 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, in.mappingLookup, in.mappingParserContext, in.sourceToParse, - in.listeners, in.ignoredFields, in.ignoredFieldValues, in.currentScope, @@ -219,7 +215,6 @@ protected DocumentParserContext( MappingLookup mappingLookup, MappingParserContext mappingParserContext, SourceToParse source, - DocumentParser.Listeners listeners, ObjectMapper parent, ObjectMapper.Dynamic dynamic ) { @@ -227,7 +222,6 @@ protected DocumentParserContext( mappingLookup, mappingParserContext, source, - listeners, new HashSet<>(), new ArrayList<>(), Scope.SINGLETON, @@ -307,6 +301,12 @@ public final void addIgnoredField(IgnoredSourceFieldMapper.NameValue values) { } } + final void removeLastIgnoredField(String name) { + if (ignoredFieldValues.isEmpty() == false && ignoredFieldValues.getLast().name().equals(name)) { + ignoredFieldValues.removeLast(); + } + } + /** * Return the collection of values for fields that have been ignored so far. */ @@ -470,15 +470,6 @@ public Set getCopyToFields() { return copyToFields; } - public void publishEvent(DocumentParserListener.Event event) throws IOException { - listeners.publish(event, this); - } - - public void finishListeners() { - var output = listeners.finish(); - ignoredFieldValues.addAll(output.ignoredSourceValues()); - } - /** * Add a new mapper dynamically created while parsing. * @@ -668,7 +659,7 @@ public final DocumentParserContext createChildContext(ObjectMapper parent) { /** * Return a new context that will be used within a nested document. */ - public final DocumentParserContext createNestedContext(NestedObjectMapper nestedMapper) throws IOException { + public final DocumentParserContext createNestedContext(NestedObjectMapper nestedMapper) { if (isWithinCopyTo()) { // nested context will already have been set up for copy_to fields return this; @@ -697,7 +688,7 @@ public final DocumentParserContext createNestedContext(NestedObjectMapper nested /** * Return a new context that has the provided document as the current document. */ - public final DocumentParserContext switchDoc(final LuceneDocument document) throws IOException { + public final DocumentParserContext switchDoc(final LuceneDocument document) { DocumentParserContext cloned = new Wrapper(this.parent, this) { @Override public LuceneDocument doc() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserListener.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserListener.java deleted file mode 100644 index 7ea902235da1c..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserListener.java +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.mapper; - -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.xcontent.XContentParser; - -import java.io.IOException; -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.List; - -/** - * Component that listens to events produced by {@link DocumentParser} in order to implement some parsing related logic. - * It allows to keep such logic separate from actual document parsing workflow which is by itself complex. - */ -public interface DocumentParserListener { - /** - * Specifies if this listener is currently actively consuming tokens. - * This is used to avoid doing unnecessary work. - * @return - */ - boolean isActive(); - - /** - * Sends a {@link Token} to this listener. - * This is only called when {@link #isActive()} returns true since it involves a somewhat costly operation of creating a token instance - * and tokens are low level meaning this is called very frequently. - * @param token - * @throws IOException - */ - void consume(Token token) throws IOException; - - /** - * Sends an {@link Event} to this listener. Unlike tokens events are always sent to a listener. - * The logic here is that based on the event listener can decide to change the return value of {@link #isActive()}. - * @param event - * @throws IOException - */ - void consume(Event event) throws IOException; - - Output finish(); - - /** - * A lower level notification passed from the parser to a listener. - * This token is closely related to {@link org.elasticsearch.xcontent.XContentParser.Token} and is used for use cases like - * preserving the exact structure of the parsed document. - */ - sealed interface Token permits Token.FieldName, Token.StartObject, Token.EndObject, Token.StartArray, Token.EndArray, - Token.StringAsCharArrayValue, Token.NullValue, Token.ValueToken { - - record FieldName(String name) implements Token {} - - record StartObject() implements Token {} - - record EndObject() implements Token {} - - record StartArray() implements Token {} - - record EndArray() implements Token {} - - record NullValue() implements Token {} - - final class StringAsCharArrayValue implements Token { - private final XContentParser parser; - - public StringAsCharArrayValue(XContentParser parser) { - this.parser = parser; - } - - char[] buffer() throws IOException { - return parser.textCharacters(); - } - - int length() throws IOException { - return parser.textLength(); - } - - int offset() throws IOException { - return parser.textOffset(); - } - } - - non-sealed interface ValueToken extends Token { - T value() throws IOException; - } - - Token START_OBJECT = new StartObject(); - Token END_OBJECT = new EndObject(); - Token START_ARRAY = new StartArray(); - Token END_ARRAY = new EndArray(); - - static Token current(XContentParser parser) throws IOException { - return switch (parser.currentToken()) { - case START_OBJECT -> Token.START_OBJECT; - case END_OBJECT -> Token.END_OBJECT; - case START_ARRAY -> Token.START_ARRAY; - case END_ARRAY -> Token.END_ARRAY; - case FIELD_NAME -> new FieldName(parser.currentName()); - case VALUE_STRING -> { - if (parser.hasTextCharacters()) { - yield new StringAsCharArrayValue(parser); - } else { - yield (ValueToken) parser::text; - } - } - case VALUE_NUMBER -> switch (parser.numberType()) { - case INT -> (ValueToken) parser::intValue; - case BIG_INTEGER -> (ValueToken) () -> (BigInteger) parser.numberValue(); - case LONG -> (ValueToken) parser::longValue; - case FLOAT -> (ValueToken) parser::floatValue; - case DOUBLE -> (ValueToken) parser::doubleValue; - case BIG_DECIMAL -> { - // See @XContentGenerator#copyCurrentEvent - assert false : "missing xcontent number handling for type [" + parser.numberType() + "]"; - yield null; - } - }; - case VALUE_BOOLEAN -> (ValueToken) parser::booleanValue; - case VALUE_EMBEDDED_OBJECT -> (ValueToken) parser::binaryValue; - case VALUE_NULL -> new NullValue(); - case null -> null; - }; - } - } - - /** - * High level notification passed from the parser to a listener. - * Events represent meaningful logical operations during parsing and contain relevant context for the operation - * like a mapper being used. - * A listener can use events and/or tokens depending on the use case. For example, it can wait for a specific event and then switch - * to consuming tokens instead. - */ - sealed interface Event permits Event.DocumentStart, Event.ObjectStart, Event.ObjectArrayStart, Event.LeafArrayStart, Event.LeafValue { - record DocumentStart(RootObjectMapper rootObjectMapper, LuceneDocument document) implements Event {} - - record ObjectStart(ObjectMapper objectMapper, boolean insideObjectArray, ObjectMapper parentMapper, LuceneDocument document) - implements - Event {} - - record ObjectArrayStart(ObjectMapper objectMapper, ObjectMapper parentMapper, LuceneDocument document) implements Event {} - - final class LeafValue implements Event { - private final FieldMapper fieldMapper; - private final boolean insideObjectArray; - private final ObjectMapper parentMapper; - private final LuceneDocument document; - private final XContentParser parser; - private final boolean isObjectOrArray; - private final boolean isArray; - - public LeafValue( - FieldMapper fieldMapper, - boolean insideObjectArray, - ObjectMapper parentMapper, - LuceneDocument document, - XContentParser parser - ) { - this.fieldMapper = fieldMapper; - this.insideObjectArray = insideObjectArray; - this.parentMapper = parentMapper; - this.document = document; - this.parser = parser; - this.isObjectOrArray = parser.currentToken().isValue() == false && parser.currentToken() != XContentParser.Token.VALUE_NULL; - this.isArray = parser.currentToken() == XContentParser.Token.START_ARRAY; - } - - public FieldMapper fieldMapper() { - return fieldMapper; - } - - public boolean insideObjectArray() { - return insideObjectArray; - } - - public ObjectMapper parentMapper() { - return parentMapper; - } - - public LuceneDocument document() { - return document; - } - - /** - * @return whether a value is an object or an array vs a single value like a long. - */ - boolean isContainer() { - return isObjectOrArray; - } - - boolean isArray() { - return isArray; - } - - BytesRef encodeValue() throws IOException { - assert isContainer() == false : "Objects should not be handled with direct encoding"; - - return XContentDataHelper.encodeToken(parser); - } - } - - record LeafArrayStart(FieldMapper fieldMapper, ObjectMapper parentMapper, LuceneDocument document) implements Event {} - } - - record Output(List ignoredSourceValues) { - static Output empty() { - return new Output(new ArrayList<>()); - } - - void merge(Output part) { - this.ignoredSourceValues.addAll(part.ignoredSourceValues); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index ffd60efc772f9..5bbecbf117dba 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -57,7 +57,6 @@ import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; -import java.util.stream.Stream; import static org.elasticsearch.core.Strings.format; @@ -444,7 +443,11 @@ protected void doXContentBody(XContentBuilder builder, Params params) throws IOE @Override public int getTotalFieldsCount() { - return 1 + Stream.of(builderParams.multiFields.mappers).mapToInt(FieldMapper::getTotalFieldsCount).sum(); + int sum = 1; + for (FieldMapper mapper : builderParams.multiFields.mappers) { + sum += mapper.getTotalFieldsCount(); + } + return sum; } public Map indexAnalyzers() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/InferenceMetadataFieldsMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/InferenceMetadataFieldsMapper.java index 6051aafb9f742..80fee58e93110 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/InferenceMetadataFieldsMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/InferenceMetadataFieldsMapper.java @@ -86,8 +86,12 @@ public abstract ValueFetcher valueFetcher( * @return {@code true} if the new format is enabled; {@code false} otherwise */ public static boolean isEnabled(Settings settings) { - return IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.INFERENCE_METADATA_FIELDS) - && USE_LEGACY_SEMANTIC_TEXT_FORMAT.get(settings) == false; + var version = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings); + if (version.before(IndexVersions.INFERENCE_METADATA_FIELDS) + && version.between(IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0) == false) { + return false; + } + return USE_LEGACY_SEMANTIC_TEXT_FORMAT.get(settings) == false; } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 5cbdffc28ba74..916de60dc80ce 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -107,9 +107,8 @@ public abstract static class Builder { private String leafName; - @SuppressWarnings("this-escape") protected Builder(String leafName) { - setLeafName(leafName); + this.leafName = leafName; } public final String leafName() { @@ -120,7 +119,7 @@ public final String leafName() { public abstract Mapper build(MapperBuilderContext context); void setLeafName(String leafName) { - this.leafName = internFieldName(leafName); + this.leafName = leafName; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 689cbdcdbda7b..fb4f86c3cba98 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -592,7 +592,6 @@ private DocumentMapper newDocumentMapper(Mapping mapping, MergeReason reason, Co mapping, mappingSource, indexVersionCreated, - indexSettings, mapperMetrics, index().getName() ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index 80dfa37a0ee03..ed02e5fc29617 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -43,7 +43,7 @@ private CacheKey() {} * A lookup representing an empty mapping. It can be used to look up fields, although it won't hold any, but it does not * hold a valid {@link DocumentParser}, {@link IndexSettings} or {@link IndexAnalyzers}. */ - public static final MappingLookup EMPTY = fromMappers(Mapping.EMPTY, List.of(), List.of(), null); + public static final MappingLookup EMPTY = fromMappers(Mapping.EMPTY, List.of(), List.of()); private final CacheKey cacheKey = new CacheKey(); @@ -59,16 +59,14 @@ private CacheKey() {} private final List indexTimeScriptMappers; private final Mapping mapping; private final int totalFieldsCount; - private final CustomSyntheticSourceFieldLookup customSyntheticSourceFieldLookup; /** * Creates a new {@link MappingLookup} instance by parsing the provided mapping and extracting its field definitions. * * @param mapping the mapping source - * @param indexSettings index settings * @return the newly created lookup instance */ - public static MappingLookup fromMapping(Mapping mapping, IndexSettings indexSettings) { + public static MappingLookup fromMapping(Mapping mapping) { List newObjectMappers = new ArrayList<>(); List newFieldMappers = new ArrayList<>(); List newFieldAliasMappers = new ArrayList<>(); @@ -81,7 +79,7 @@ public static MappingLookup fromMapping(Mapping mapping, IndexSettings indexSett for (Mapper child : mapping.getRoot()) { collect(child, newObjectMappers, newFieldMappers, newFieldAliasMappers, newPassThroughMappers); } - return new MappingLookup(mapping, newFieldMappers, newObjectMappers, newFieldAliasMappers, newPassThroughMappers, indexSettings); + return new MappingLookup(mapping, newFieldMappers, newObjectMappers, newFieldAliasMappers, newPassThroughMappers); } private static void collect( @@ -122,7 +120,6 @@ private static void collect( * @param objectMappers the object mappers * @param aliasMappers the field alias mappers * @param passThroughMappers the pass-through mappers - * @param indexSettings index settings * @return the newly created lookup instance */ public static MappingLookup fromMappers( @@ -130,19 +127,13 @@ public static MappingLookup fromMappers( Collection mappers, Collection objectMappers, Collection aliasMappers, - Collection passThroughMappers, - @Nullable IndexSettings indexSettings + Collection passThroughMappers ) { - return new MappingLookup(mapping, mappers, objectMappers, aliasMappers, passThroughMappers, indexSettings); + return new MappingLookup(mapping, mappers, objectMappers, aliasMappers, passThroughMappers); } - public static MappingLookup fromMappers( - Mapping mapping, - Collection mappers, - Collection objectMappers, - @Nullable IndexSettings indexSettings - ) { - return new MappingLookup(mapping, mappers, objectMappers, List.of(), List.of(), indexSettings); + public static MappingLookup fromMappers(Mapping mapping, Collection mappers, Collection objectMappers) { + return new MappingLookup(mapping, mappers, objectMappers, List.of(), List.of()); } private MappingLookup( @@ -150,8 +141,7 @@ private MappingLookup( Collection mappers, Collection objectMappers, Collection aliasMappers, - Collection passThroughMappers, - @Nullable IndexSettings indexSettings + Collection passThroughMappers ) { this.totalFieldsCount = mapping.getRoot().getTotalFieldsCount(); this.mapping = mapping; @@ -217,7 +207,6 @@ private MappingLookup( this.runtimeFieldMappersCount = runtimeFields.size(); this.indexAnalyzersMap = Map.copyOf(indexAnalyzersMap); this.indexTimeScriptMappers = List.copyOf(indexTimeScriptMappers); - this.customSyntheticSourceFieldLookup = new CustomSyntheticSourceFieldLookup(mapping, indexSettings, isSourceSynthetic()); runtimeFields.stream().flatMap(RuntimeField::asMappedFieldTypes).map(MappedFieldType::name).forEach(this::validateDoesNotShadow); assert assertMapperNamesInterned(this.fieldMappers, this.objectMappers); @@ -554,8 +543,4 @@ public void validateDoesNotShadow(String name) { throw new MapperParsingException("Field [" + name + "] attempted to shadow a time_series_metric"); } } - - public CustomSyntheticSourceFieldLookup getCustomSyntheticSourceFieldLookup() { - return customSyntheticSourceFieldLookup; - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 6a107dbaa9e63..f4084b3ede24f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -263,7 +263,11 @@ public ObjectMapper build(MapperBuilderContext context) { @Override public int getTotalFieldsCount() { - return 1 + mappers.values().stream().mapToInt(Mapper::getTotalFieldsCount).sum(); + int sum = 1; + for (Mapper mapper : mappers.values()) { + sum += mapper.getTotalFieldsCount(); + } + return sum; } public static class TypeParser implements Mapper.TypeParser { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java index ce983e8a327c9..2fe82b4eacfc5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java @@ -538,6 +538,6 @@ private static boolean processField( @Override public int getTotalFieldsCount() { - return mappers.values().stream().mapToInt(Mapper::getTotalFieldsCount).sum() + runtimeFields.size(); + return super.getTotalFieldsCount() - 1 + runtimeFields.size(); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SyntheticSourceDocumentParserListener.java b/server/src/main/java/org/elasticsearch/index/mapper/SyntheticSourceDocumentParserListener.java deleted file mode 100644 index eabb117635570..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/mapper/SyntheticSourceDocumentParserListener.java +++ /dev/null @@ -1,398 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.mapper; - -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentType; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Listens for document parsing events and stores an additional copy of source data when it is needed for synthetic _source. - *
- * Note that synthetic source logic for dynamic fields and fields involved in copy_to logic is still handled in {@link DocumentParser}. - */ -class SyntheticSourceDocumentParserListener implements DocumentParserListener { - private final CustomSyntheticSourceFieldLookup customSyntheticSourceFieldLookup; - private final XContentType xContentType; - - private final Map>> ignoredSourceValues; - - private State state; - - SyntheticSourceDocumentParserListener(MappingLookup mappingLookup, XContentType xContentType) { - this.customSyntheticSourceFieldLookup = mappingLookup.getCustomSyntheticSourceFieldLookup(); - this.xContentType = xContentType; - - this.ignoredSourceValues = new HashMap<>(); - this.state = new Tracking(); - } - - @Override - public boolean isActive() { - return state instanceof Storing; - } - - @Override - public void consume(Token token) throws IOException { - if (token == null) { - return; - } - - this.state = state.consume(token); - } - - @Override - public void consume(Event event) throws IOException { - if (event == null) { - return; - } - - this.state = state.consume(event); - } - - @Override - public Output finish() { - var values = new ArrayList(); - - for (var fieldToValueMap : ignoredSourceValues.values()) { - for (var fieldValues : fieldToValueMap.values()) { - long singleElementArrays = 0; - long stashedValuesForSourceKeepArrays = 0; - - for (var fieldValue : fieldValues) { - if (fieldValue instanceof StoredValue.Array arr) { - // Arrays are stored to preserve the order of elements. - // If there is a single element it does not matter and we can drop such data. - if (arr.length == 1 && arr.reason() == StoreReason.LEAF_STORED_ARRAY) { - singleElementArrays += 1; - } - } - if (fieldValue instanceof StoredValue.Singleton singleton) { - // Stash values are values of fields that are inside object arrays and have synthetic_source_keep: "arrays". - // With current logic either all field values should be in ignored source - // or none of them. - // With object arrays the same field can be parsed multiple times (one time for every object array entry) - // and it is possible that one of the value is an array. - // Due to the rule above we need to proactively store all values of such fields because we may later discover - // that there is an array and we need to "switch" to ignored source usage. - // However if we stored all values but the array is not there, the field will be correctly constructed - // using regular logic and therefore we can drop this and save some space. - if (singleton.reason() == StoreReason.LEAF_VALUE_STASH_FOR_STORED_ARRAYS) { - stashedValuesForSourceKeepArrays += 1; - } - } - } - - // Only if all values match one of the optimization criteria we skip them, otherwise add all of them to resulting list. - if (singleElementArrays != fieldValues.size() && stashedValuesForSourceKeepArrays != fieldValues.size()) { - for (var storedValue : fieldValues) { - values.add(storedValue.nameValue()); - } - } - } - } - - return new Output(values); - } - - sealed interface StoredValue permits StoredValue.Array, StoredValue.Singleton { - IgnoredSourceFieldMapper.NameValue nameValue(); - - /** - * An array of values is stored f.e. due to synthetic_source_keep: "arrays". - */ - record Array(IgnoredSourceFieldMapper.NameValue nameValue, StoreReason reason, long length) implements StoredValue {} - - /** - * A single value. - */ - record Singleton(IgnoredSourceFieldMapper.NameValue nameValue, StoreReason reason) implements StoredValue {} - - } - - /** - * Reason for storing this value. - */ - enum StoreReason { - /** - * Leaf array that is stored due to "synthetic_source_keep": "arrays". - */ - LEAF_STORED_ARRAY, - - /** - * "Stashed" value needed to only in case there are mixed arrays and single values - * for this field. - * Can be dropped in some cases. - */ - LEAF_VALUE_STASH_FOR_STORED_ARRAYS, - - /** - * There is currently no need to distinguish other reasons. - */ - OTHER - } - - private void addIgnoredSourceValue(StoredValue storedValue, String fullPath, LuceneDocument luceneDocument) { - var values = ignoredSourceValues.computeIfAbsent(luceneDocument, ld -> new HashMap<>()) - .computeIfAbsent(fullPath, p -> new ArrayList<>()); - - values.add(storedValue); - } - - interface State { - State consume(Token token) throws IOException; - - State consume(Event event) throws IOException; - } - - class Storing implements State { - private final State returnState; - private final String fullPath; - private final ObjectMapper parentMapper; - private final StoreReason reason; - private final LuceneDocument document; - - private final XContentBuilder builder; - // Current object/array depth, needed to understand when the top-most object/arrays ends vs a nested one. - private int depth; - // If we are storing an array this is the length of the array. - private int length; - - Storing( - State returnState, - Token startingToken, - String fullPath, - ObjectMapper parentMapper, - StoreReason reason, - LuceneDocument document - ) throws IOException { - this.returnState = returnState; - this.fullPath = fullPath; - this.parentMapper = parentMapper; - this.reason = reason; - this.document = document; - - this.builder = XContentBuilder.builder(xContentType.xContent()); - - this.depth = 0; - this.length = 0; - - consume(startingToken); - } - - public State consume(Token token) throws IOException { - switch (token) { - case Token.StartObject startObject -> { - builder.startObject(); - if (depth == 1) { - length += 1; - } - depth += 1; - } - case Token.EndObject endObject -> { - builder.endObject(); - - if (processEndObjectOrArray(endObject)) { - return returnState; - } - } - case Token.StartArray startArray -> { - builder.startArray(); - depth += 1; - } - case Token.EndArray endArray -> { - builder.endArray(); - - if (processEndObjectOrArray(endArray)) { - return returnState; - } - } - case Token.FieldName fieldName -> builder.field(fieldName.name()); - case Token.StringAsCharArrayValue stringAsCharArrayValue -> { - if (depth == 1) { - length += 1; - } - builder.generator() - .writeString(stringAsCharArrayValue.buffer(), stringAsCharArrayValue.offset(), stringAsCharArrayValue.length()); - } - case Token.ValueToken valueToken -> { - if (depth == 1) { - length += 1; - } - builder.value(valueToken.value()); - } - case Token.NullValue nullValue -> { - if (depth == 1) { - length += 1; - } - builder.nullValue(); - } - case null -> { - } - } - - return this; - } - - public State consume(Event event) { - // We are currently storing something so events are not relevant. - return this; - } - - private boolean processEndObjectOrArray(Token token) throws IOException { - assert token instanceof Token.EndObject || token instanceof Token.EndArray - : "Unexpected token when storing ignored source value"; - - depth -= 1; - if (depth == 0) { - var parentOffset = parentMapper.isRoot() ? 0 : parentMapper.fullPath().length() + 1; - var nameValue = new IgnoredSourceFieldMapper.NameValue( - fullPath, - parentOffset, - XContentDataHelper.encodeXContentBuilder(builder), - document - ); - var storedValue = token instanceof Token.EndObject - ? new StoredValue.Singleton(nameValue, reason) - : new StoredValue.Array(nameValue, reason, length); - - addIgnoredSourceValue(storedValue, fullPath, document); - - return true; - } - - return false; - } - } - - class Tracking implements State { - public State consume(Token token) throws IOException { - return this; - } - - public State consume(Event event) throws IOException { - switch (event) { - case Event.DocumentStart documentStart -> { - if (documentStart.rootObjectMapper().isEnabled() == false) { - return new Storing( - this, - Token.START_OBJECT, - documentStart.rootObjectMapper().fullPath(), - documentStart.rootObjectMapper(), - StoreReason.OTHER, - documentStart.document() - ); - } - } - case Event.ObjectStart objectStart -> { - var reason = customSyntheticSourceFieldLookup.getFieldsWithCustomSyntheticSourceHandling() - .get(objectStart.objectMapper().fullPath()); - if (reason == null) { - return this; - } - if (reason == CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS && objectStart.insideObjectArray() == false) { - return this; - } - - return new Storing( - this, - Token.START_OBJECT, - objectStart.objectMapper().fullPath(), - objectStart.parentMapper(), - StoreReason.OTHER, - objectStart.document() - ); - } - case Event.ObjectArrayStart objectArrayStart -> { - var reason = customSyntheticSourceFieldLookup.getFieldsWithCustomSyntheticSourceHandling() - .get(objectArrayStart.objectMapper().fullPath()); - if (reason == null) { - return this; - } - - return new Storing( - this, - Token.START_ARRAY, - objectArrayStart.objectMapper().fullPath(), - objectArrayStart.parentMapper(), - StoreReason.OTHER, - objectArrayStart.document() - ); - } - case Event.LeafValue leafValue -> { - var reason = customSyntheticSourceFieldLookup.getFieldsWithCustomSyntheticSourceHandling() - .get(leafValue.fieldMapper().fullPath()); - if (reason == null) { - return this; - } - if (reason == CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS && leafValue.insideObjectArray() == false) { - return this; - } - - var storeReason = reason == CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS - ? StoreReason.LEAF_VALUE_STASH_FOR_STORED_ARRAYS - : StoreReason.OTHER; - - if (leafValue.isContainer()) { - return new Storing( - this, - leafValue.isArray() ? Token.START_ARRAY : Token.START_OBJECT, - leafValue.fieldMapper().fullPath(), - leafValue.parentMapper(), - storeReason, - leafValue.document() - ); - } - - var parentMapper = leafValue.parentMapper(); - var parentOffset = parentMapper.isRoot() ? 0 : parentMapper.fullPath().length() + 1; - - var nameValue = new IgnoredSourceFieldMapper.NameValue( - leafValue.fieldMapper().fullPath(), - parentOffset, - leafValue.encodeValue(), - leafValue.document() - ); - addIgnoredSourceValue( - new StoredValue.Singleton(nameValue, storeReason), - leafValue.fieldMapper().fullPath(), - leafValue.document() - ); - } - case Event.LeafArrayStart leafArrayStart -> { - var reason = customSyntheticSourceFieldLookup.getFieldsWithCustomSyntheticSourceHandling() - .get(leafArrayStart.fieldMapper().fullPath()); - if (reason == null) { - return this; - } - - var storeReason = reason == CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS - ? StoreReason.LEAF_STORED_ARRAY - : StoreReason.OTHER; - return new Storing( - this, - Token.START_ARRAY, - leafArrayStart.fieldMapper().fullPath(), - leafArrayStart.parentMapper(), - storeReason, - leafArrayStart.document() - ); - } - } - - return this; - } - } -} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 3584523505aea..a99f21803556f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -102,7 +102,7 @@ public class DenseVectorFieldMapper extends FieldMapper { public static final String COSINE_MAGNITUDE_FIELD_SUFFIX = "._magnitude"; private static final float EPS = 1e-3f; - static final int BBQ_MIN_DIMS = 64; + public static final int BBQ_MIN_DIMS = 64; public static boolean isNotUnitVector(float magnitude) { return Math.abs(magnitude - 1.0f) > EPS; @@ -487,8 +487,12 @@ private VectorData parseHexEncodedVector( } @Override - VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity) - throws IOException { + public VectorData parseKnnVector( + DocumentParserContext context, + int dims, + IntBooleanConsumer dimChecker, + VectorSimilarity similarity + ) throws IOException { XContentParser.Token token = context.parser().currentToken(); return switch (token) { case START_ARRAY -> parseVectorArray(context, dims, dimChecker, similarity); @@ -518,17 +522,17 @@ public void parseKnnVectorAndIndex(DocumentParserContext context, DenseVectorFie } @Override - int getNumBytes(int dimensions) { + public int getNumBytes(int dimensions) { return dimensions; } @Override - ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) { + public ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) { return ByteBuffer.wrap(new byte[numBytes]); } @Override - int parseDimensionCount(DocumentParserContext context) throws IOException { + public int parseDimensionCount(DocumentParserContext context) throws IOException { XContentParser.Token currentToken = context.parser().currentToken(); return switch (currentToken) { case START_ARRAY -> { @@ -692,8 +696,12 @@ && isNotUnitVector(squaredMagnitude)) { } @Override - VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity) - throws IOException { + public VectorData parseKnnVector( + DocumentParserContext context, + int dims, + IntBooleanConsumer dimChecker, + VectorSimilarity similarity + ) throws IOException { int index = 0; float squaredMagnitude = 0; float[] vector = new float[dims]; @@ -712,12 +720,12 @@ VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanCon } @Override - int getNumBytes(int dimensions) { + public int getNumBytes(int dimensions) { return dimensions * Float.BYTES; } @Override - ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) { + public ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) { return indexVersion.onOrAfter(LITTLE_ENDIAN_FLOAT_STORED_INDEX_VERSION) ? ByteBuffer.wrap(new byte[numBytes]).order(ByteOrder.LITTLE_ENDIAN) : ByteBuffer.wrap(new byte[numBytes]); @@ -890,8 +898,12 @@ private VectorData parseHexEncodedVector(DocumentParserContext context, IntBoole } @Override - VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity) - throws IOException { + public VectorData parseKnnVector( + DocumentParserContext context, + int dims, + IntBooleanConsumer dimChecker, + VectorSimilarity similarity + ) throws IOException { XContentParser.Token token = context.parser().currentToken(); return switch (token) { case START_ARRAY -> parseVectorArray(context, dims, dimChecker, similarity); @@ -921,18 +933,18 @@ public void parseKnnVectorAndIndex(DocumentParserContext context, DenseVectorFie } @Override - int getNumBytes(int dimensions) { + public int getNumBytes(int dimensions) { assert dimensions % Byte.SIZE == 0; return dimensions / Byte.SIZE; } @Override - ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) { + public ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) { return ByteBuffer.wrap(new byte[numBytes]); } @Override - int parseDimensionCount(DocumentParserContext context) throws IOException { + public int parseDimensionCount(DocumentParserContext context) throws IOException { XContentParser.Token currentToken = context.parser().currentToken(); return switch (currentToken) { case START_ARRAY -> { @@ -975,16 +987,16 @@ public void checkDimensions(Integer dvDims, int qvDims) { abstract void parseKnnVectorAndIndex(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException; - abstract VectorData parseKnnVector( + public abstract VectorData parseKnnVector( DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity ) throws IOException; - abstract int getNumBytes(int dimensions); + public abstract int getNumBytes(int dimensions); - abstract ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes); + public abstract ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes); public abstract void checkVectorBounds(float[] vector); @@ -1002,7 +1014,7 @@ public void checkDimensions(Integer dvDims, int qvDims) { } } - int parseDimensionCount(DocumentParserContext context) throws IOException { + public int parseDimensionCount(DocumentParserContext context) throws IOException { int index = 0; for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) { index++; @@ -1089,7 +1101,7 @@ public static ElementType fromString(String name) { } } - static final Map namesToElementType = Map.of( + public static final Map namesToElementType = Map.of( ElementType.BYTE.toString(), ElementType.BYTE, ElementType.FLOAT.toString(), @@ -2501,9 +2513,10 @@ public String fieldName() { } /** - * @FunctionalInterface for a function that takes a int and boolean + * Interface for a function that takes a int and boolean */ - interface IntBooleanConsumer { + @FunctionalInterface + public interface IntBooleanConsumer { void accept(int value, boolean isComplete); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsDVLeafFieldData.java deleted file mode 100644 index 0125d0249ec2b..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsDVLeafFieldData.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.mapper.vectors; - -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.LeafReader; -import org.elasticsearch.index.fielddata.LeafFieldData; -import org.elasticsearch.index.fielddata.SortedBinaryDocValues; -import org.elasticsearch.script.field.DocValuesScriptFieldFactory; -import org.elasticsearch.script.field.vectors.BitRankVectorsDocValuesField; -import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField; -import org.elasticsearch.script.field.vectors.FloatRankVectorsDocValuesField; - -import java.io.IOException; - -final class RankVectorsDVLeafFieldData implements LeafFieldData { - private final LeafReader reader; - private final String field; - private final DenseVectorFieldMapper.ElementType elementType; - private final int dims; - - RankVectorsDVLeafFieldData(LeafReader reader, String field, DenseVectorFieldMapper.ElementType elementType, int dims) { - this.reader = reader; - this.field = field; - this.elementType = elementType; - this.dims = dims; - } - - @Override - public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { - try { - BinaryDocValues values = DocValues.getBinary(reader, field); - BinaryDocValues magnitudeValues = DocValues.getBinary(reader, field + RankVectorsFieldMapper.VECTOR_MAGNITUDES_SUFFIX); - return switch (elementType) { - case BYTE -> new ByteRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims); - case FLOAT -> new FloatRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims); - case BIT -> new BitRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims); - }; - } catch (IOException e) { - throw new IllegalStateException("Cannot load doc values for multi-vector field!", e); - } - } - - @Override - public SortedBinaryDocValues getBytesValues() { - throw new UnsupportedOperationException("String representation of doc values for multi-vector fields is not supported"); - } - - @Override - public long ramBytesUsed() { - return 0; - } -} diff --git a/server/src/main/java/org/elasticsearch/index/translog/Translog.java b/server/src/main/java/org/elasticsearch/index/translog/Translog.java index dd13f905b6cb8..b1a203616b120 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -27,6 +27,7 @@ import org.elasticsearch.core.Releasable; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.TranslogOperationAsserter; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Uid; @@ -123,6 +124,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC private final TranslogDeletionPolicy deletionPolicy; private final LongConsumer persistedSequenceNumberConsumer; private final OperationListener operationListener; + private final TranslogOperationAsserter operationAsserter; /** * Creates a new Translog instance. This method will create a new transaction log unless the given {@link TranslogGeneration} is @@ -150,7 +152,8 @@ public Translog( TranslogDeletionPolicy deletionPolicy, final LongSupplier globalCheckpointSupplier, final LongSupplier primaryTermSupplier, - final LongConsumer persistedSequenceNumberConsumer + final LongConsumer persistedSequenceNumberConsumer, + final TranslogOperationAsserter operationAsserter ) throws IOException { super(config.getShardId(), config.getIndexSettings()); this.config = config; @@ -158,6 +161,7 @@ public Translog( this.primaryTermSupplier = primaryTermSupplier; this.persistedSequenceNumberConsumer = persistedSequenceNumberConsumer; this.operationListener = config.getOperationListener(); + this.operationAsserter = operationAsserter; this.deletionPolicy = deletionPolicy; this.translogUUID = translogUUID; this.bigArrays = config.getBigArrays(); @@ -586,6 +590,7 @@ TranslogWriter createWriter( bigArrays, diskIoBufferPool, operationListener, + operationAsserter, config.fsync() ); } catch (final IOException e) { @@ -1269,17 +1274,8 @@ public boolean equals(Object o) { return false; } - Index index = (Index) o; - - if (version != index.version - || seqNo != index.seqNo - || primaryTerm != index.primaryTerm - || id.equals(index.id) == false - || autoGeneratedIdTimestamp != index.autoGeneratedIdTimestamp - || source.equals(index.source) == false) { - return false; - } - return Objects.equals(routing, index.routing); + Index other = (Index) o; + return autoGeneratedIdTimestamp == other.autoGeneratedIdTimestamp && equalsWithoutAutoGeneratedTimestamp(this, other); } @Override @@ -1315,6 +1311,15 @@ public long getAutoGeneratedIdTimestamp() { return autoGeneratedIdTimestamp; } + public static boolean equalsWithoutAutoGeneratedTimestamp(Translog.Index o1, Translog.Index o2) { + return o1.version == o2.version + && o1.seqNo == o2.seqNo + && o1.primaryTerm == o2.primaryTerm + && o1.id.equals(o2.id) + && o1.source.equals(o2.source) + && Objects.equals(o1.routing, o2.routing); + } + } public static final class Delete extends Operation { @@ -1962,6 +1967,7 @@ public static String createEmptyTranslog( BigArrays.NON_RECYCLING_INSTANCE, DiskIoBufferPool.INSTANCE, TranslogConfig.NOOP_OPERATION_LISTENER, + TranslogOperationAsserter.DEFAULT, true ); writer.close(); diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java index 3dda44ff5a6db..8cf631b660b1e 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java +++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java @@ -26,6 +26,7 @@ import org.elasticsearch.core.Releasables; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.engine.TranslogOperationAsserter; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.ShardId; @@ -39,7 +40,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; import java.util.function.LongConsumer; @@ -69,6 +69,7 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable { // callback that's called whenever an operation with a given sequence number is successfully persisted. private final LongConsumer persistedSequenceNumberConsumer; private final OperationListener operationListener; + private final TranslogOperationAsserter operationAsserter; private final boolean fsync; protected final AtomicBoolean closed = new AtomicBoolean(false); @@ -108,6 +109,7 @@ private TranslogWriter( BigArrays bigArrays, DiskIoBufferPool diskIoBufferPool, OperationListener operationListener, + TranslogOperationAsserter operationAsserter, boolean fsync ) throws IOException { super(initialCheckpoint.generation, channel, path, header); @@ -136,6 +138,7 @@ private TranslogWriter( this.seenSequenceNumbers = Assertions.ENABLED ? new HashMap<>() : null; this.tragedy = tragedy; this.operationListener = operationListener; + this.operationAsserter = operationAsserter; this.fsync = fsync; this.lastModifiedTimeCache = new LastModifiedTimeCache(-1, -1, -1); } @@ -157,6 +160,7 @@ public static TranslogWriter create( BigArrays bigArrays, DiskIoBufferPool diskIoBufferPool, OperationListener operationListener, + TranslogOperationAsserter operationAsserter, boolean fsync ) throws IOException { final Path checkpointFile = file.getParent().resolve(Translog.CHECKPOINT_FILE_NAME); @@ -201,6 +205,7 @@ public static TranslogWriter create( bigArrays, diskIoBufferPool, operationListener, + operationAsserter, fsync ); } catch (Exception exception) { @@ -276,25 +281,16 @@ private synchronized boolean assertNoSeqNumberConflict(long seqNo, BytesReferenc Translog.Operation prvOp = Translog.readOperation( new BufferedChecksumStreamInput(previous.v1().streamInput(), "assertion") ); - // TODO: We haven't had timestamp for Index operations in Lucene yet, we need to loosen this check without timestamp. final boolean sameOp; if (newOp instanceof final Translog.Index o2 && prvOp instanceof final Translog.Index o1) { - sameOp = Objects.equals(o1.id(), o2.id()) - && Objects.equals(o1.source(), o2.source()) - && Objects.equals(o1.routing(), o2.routing()) - && o1.primaryTerm() == o2.primaryTerm() - && o1.seqNo() == o2.seqNo() - && o1.version() == o2.version(); + sameOp = operationAsserter.assertSameIndexOperation(o1, o2); } else if (newOp instanceof final Translog.Delete o1 && prvOp instanceof final Translog.Delete o2) { - sameOp = Objects.equals(o1.id(), o2.id()) - && o1.primaryTerm() == o2.primaryTerm() - && o1.seqNo() == o2.seqNo() - && o1.version() == o2.version(); + sameOp = o1.equals(o2); } else { sameOp = false; } - if (sameOp == false) { - throw new AssertionError( + assert sameOp + : new AssertionError( "seqNo [" + seqNo + "] was processed twice in generation [" @@ -307,7 +303,6 @@ private synchronized boolean assertNoSeqNumberConflict(long seqNo, BytesReferenc + "]", previous.v2() ); - } } } else { seenSequenceNumbers.put( diff --git a/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java b/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java index 7f97edb2ff695..6ef1e2eff785b 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java +++ b/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.engine.TranslogOperationAsserter; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.RemoveCorruptedShardDataCommand; import org.elasticsearch.index.shard.ShardPath; @@ -171,7 +172,8 @@ private static boolean isTranslogClean(ShardPath shardPath, ClusterState cluster translogDeletionPolicy, () -> translogGlobalCheckpoint, () -> primaryTerm, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ); Translog.Snapshot snapshot = translog.newSnapshot(0, Long.MAX_VALUE) ) { diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 3dc25b058b1d6..09be98630d5c4 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -67,7 +67,6 @@ import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; -import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction; import org.elasticsearch.index.seqno.RetentionLeaseSyncAction; @@ -211,9 +210,6 @@ public static Map getMappers(List mappe mappers.put(DenseVectorFieldMapper.CONTENT_TYPE, DenseVectorFieldMapper.PARSER); mappers.put(SparseVectorFieldMapper.CONTENT_TYPE, SparseVectorFieldMapper.PARSER); - if (RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()) { - mappers.put(RankVectorsFieldMapper.CONTENT_TYPE, RankVectorsFieldMapper.PARSER); - } for (MapperPlugin mapperPlugin : mapperPlugins) { for (Map.Entry entry : mapperPlugin.getMappers().entrySet()) { diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index a5765a1a707d2..f22a99cb27faf 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -264,6 +264,7 @@ public class IndicesService extends AbstractLifecycleComponent private final PostRecoveryMerger postRecoveryMerger; private final List searchOperationListeners; private final QueryRewriteInterceptor queryRewriteInterceptor; + final SlowLogFieldProvider slowLogFieldProvider; // pkg-private for testingå @Override protected void doStart() { @@ -383,6 +384,7 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon this.timestampFieldMapperService = new TimestampFieldMapperService(settings, threadPool, this); this.postRecoveryMerger = new PostRecoveryMerger(settings, threadPool.executor(ThreadPool.Names.FORCE_MERGE), this::getShardOrNull); this.searchOperationListeners = builder.searchOperationListener; + this.slowLogFieldProvider = builder.slowLogFieldProvider; } private static final String DANGLING_INDICES_UPDATE_THREAD_NAME = "DanglingIndices#updateTask"; @@ -753,7 +755,7 @@ private synchronized IndexService createIndexService( () -> allowExpensiveQueries, indexNameExpressionResolver, recoveryStateFactories, - loadSlowLogFieldProvider(), + slowLogFieldProvider, mapperMetrics, searchOperationListeners ); @@ -833,7 +835,7 @@ public synchronized MapperService createIndexMapperServiceForValidation(IndexMet () -> allowExpensiveQueries, indexNameExpressionResolver, recoveryStateFactories, - loadSlowLogFieldProvider(), + slowLogFieldProvider, mapperMetrics, searchOperationListeners ); @@ -1439,31 +1441,6 @@ int numPendingDeletes(Index index) { } } - // pkg-private for testing - SlowLogFieldProvider loadSlowLogFieldProvider() { - List slowLogFieldProviders = pluginsService.loadServiceProviders(SlowLogFieldProvider.class); - return new SlowLogFieldProvider() { - @Override - public void init(IndexSettings indexSettings) { - slowLogFieldProviders.forEach(provider -> provider.init(indexSettings)); - } - - @Override - public Map indexSlowLogFields() { - return slowLogFieldProviders.stream() - .flatMap(provider -> provider.indexSlowLogFields().entrySet().stream()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - @Override - public Map searchSlowLogFields() { - return slowLogFieldProviders.stream() - .flatMap(provider -> provider.searchSlowLogFields().entrySet().stream()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - }; - } - /** * Checks if all pending deletes have completed. Used by tests to ensure we don't check directory contents * while deletion still ongoing. * The reason is that, on Windows, browsing the directory contents can interfere diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java b/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java index f0f0f453e3be8..d88bbfa3eba17 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java @@ -23,6 +23,8 @@ import org.elasticsearch.features.FeatureService; import org.elasticsearch.gateway.MetaStateService; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.SlowLogFieldProvider; +import org.elasticsearch.index.SlowLogFields; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.mapper.MapperMetrics; @@ -79,6 +81,22 @@ public class IndicesServiceBuilder { MapperMetrics mapperMetrics; List searchOperationListener = List.of(); QueryRewriteInterceptor queryRewriteInterceptor = null; + SlowLogFieldProvider slowLogFieldProvider = new SlowLogFieldProvider() { + @Override + public SlowLogFields create(IndexSettings indexSettings) { + return new SlowLogFields() { + @Override + public Map indexFields() { + return Map.of(); + } + + @Override + public Map searchFields() { + return Map.of(); + } + }; + } + }; public IndicesServiceBuilder settings(Settings settings) { this.settings = settings; @@ -191,6 +209,11 @@ public IndicesServiceBuilder searchOperationListeners(List slowLogFieldProviders = pluginsService.loadServiceProviders(SlowLogFieldProvider.class); + // NOTE: the response of index/search slow log fields below must be calculated dynamically on every call + // because the responses may change dynamically at runtime + SlowLogFieldProvider slowLogFieldProvider = indexSettings -> { + final List fields = new ArrayList<>(); + for (var provider : slowLogFieldProviders) { + fields.add(provider.create(indexSettings)); + } + return new SlowLogFields() { + @Override + public Map indexFields() { + return fields.stream() + .flatMap(f -> f.indexFields().entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + @Override + public Map searchFields() { + return fields.stream() + .flatMap(f -> f.searchFields().entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + }; + }; + IndicesService indicesService = new IndicesServiceBuilder().settings(settings) .pluginsService(pluginsService) .nodeEnvironment(nodeEnvironment) @@ -826,6 +853,7 @@ private void construct( .requestCacheKeyDifferentiator(searchModule.getRequestCacheKeyDifferentiator()) .mapperMetrics(mapperMetrics) .searchOperationListeners(searchOperationListeners) + .slowLogFieldProvider(slowLogFieldProvider) .build(); final var parameters = new IndexSettingProvider.Parameters(clusterService, indicesService::createIndexMapperServiceForValidation); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java index 63bd4523f9bd1..690f3155971ca 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java @@ -33,7 +33,8 @@ public class RestClusterStatsAction extends BaseRestHandler { "human-readable-total-docs-size", "verbose-dense-vector-mapping-stats", "ccs-stats", - "retrievers-usage-stats" + "retrievers-usage-stats", + "esql-stats" ); private static final Set SUPPORTED_QUERY_PARAMETERS = Set.of("include_remotes", "nodeId", REST_TIMEOUT_PARAM); diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java index 06f8f8f3c1be6..d9e2eaa8f92b6 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java @@ -10,7 +10,6 @@ package org.elasticsearch.rest.action.search; import org.elasticsearch.Build; -import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; import java.util.HashSet; import java.util.Set; @@ -34,14 +33,8 @@ private SearchCapabilities() {} private static final String TRANSFORM_RANK_RRF_TO_RETRIEVER = "transform_rank_rrf_to_retriever"; /** Support kql query. */ private static final String KQL_QUERY_SUPPORTED = "kql_query"; - /** Support rank-vectors field mapper. */ - private static final String RANK_VECTORS_FIELD_MAPPER = "rank_vectors_field_mapper"; /** Support propagating nested retrievers' inner_hits to top-level compound retrievers . */ private static final String NESTED_RETRIEVER_INNER_HITS_SUPPORT = "nested_retriever_inner_hits_support"; - /** Support rank-vectors script field access. */ - private static final String RANK_VECTORS_SCRIPT_ACCESS = "rank_vectors_script_access"; - /** Initial support for rank-vectors maxSim functions access. */ - private static final String RANK_VECTORS_SCRIPT_MAX_SIM = "rank_vectors_script_max_sim_with_bugfix"; /** Fixed the math in {@code moving_fn}'s {@code linearWeightedAvg}. */ private static final String MOVING_FN_RIGHT_MATH = "moving_fn_right_math"; @@ -62,11 +55,6 @@ private SearchCapabilities() {} capabilities.add(OPTIMIZED_SCALAR_QUANTIZATION_BBQ); capabilities.add(KNN_QUANTIZED_VECTOR_RESCORE); capabilities.add(MOVING_FN_RIGHT_MATH); - if (RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()) { - capabilities.add(RANK_VECTORS_FIELD_MAPPER); - capabilities.add(RANK_VECTORS_SCRIPT_ACCESS); - capabilities.add(RANK_VECTORS_SCRIPT_MAX_SIM); - } if (Build.current().isSnapshot()) { capabilities.add(KQL_QUERY_SUPPORTED); } diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java index db81bb6ebe1cb..1bff1b50fb5ac 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java @@ -111,13 +111,13 @@ public boolean isEmpty() { return value == null; } - static class ByteVectorIterator implements VectorIterator { + public static class ByteVectorIterator implements VectorIterator { private final byte[] buffer; private final BytesRef vectorValues; private final int size; private int idx = 0; - ByteVectorIterator(BytesRef vectorValues, byte[] buffer, int size) { + public ByteVectorIterator(BytesRef vectorValues, byte[] buffer, int size) { assert vectorValues.length == (buffer.length * size); this.vectorValues = vectorValues; this.size = size; diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java index 39bc1e621113b..d47795a3b2401 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java @@ -110,14 +110,14 @@ private void decodeVectorIfNecessary() { } } - static class FloatVectorIterator implements VectorIterator { + public static class FloatVectorIterator implements VectorIterator { private final float[] buffer; private final FloatBuffer vectorValues; private final BytesRef vectorValueBytesRef; private final int size; private int idx = 0; - FloatVectorIterator(BytesRef vectorValues, float[] buffer, int size) { + public FloatVectorIterator(BytesRef vectorValues, float[] buffer, int size) { assert vectorValues.length == (buffer.length * Float.BYTES * size); this.vectorValueBytesRef = vectorValues; this.vectorValues = ByteBuffer.wrap(vectorValues.bytes, vectorValues.offset, vectorValues.length) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorsReducer.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorsReducer.java index 6682fb2a83418..4c89877b7b1c6 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorsReducer.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorsReducer.java @@ -12,7 +12,10 @@ import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; /** @@ -54,7 +57,12 @@ public void accept(InternalAggregations aggregations) { * returns the reduced {@link InternalAggregations}. */ public InternalAggregations get() { - return InternalAggregations.from(aggByName.values().stream().map(AggregatorReducer::get).toList()); + final Collection reducers = aggByName.values(); + final List aggs = new ArrayList<>(reducers.size()); + for (AggregatorReducer reducer : reducers) { + aggs.add(reducer.get()); + } + return InternalAggregations.from(aggs); } @Override diff --git a/server/src/main/java/org/elasticsearch/usage/UsageService.java b/server/src/main/java/org/elasticsearch/usage/UsageService.java index dd4895eb4bdc2..5b4fa0f27bf48 100644 --- a/server/src/main/java/org/elasticsearch/usage/UsageService.java +++ b/server/src/main/java/org/elasticsearch/usage/UsageService.java @@ -26,11 +26,13 @@ public class UsageService { private final Map handlers; private final SearchUsageHolder searchUsageHolder; private final CCSUsageTelemetry ccsUsageHolder; + private final CCSUsageTelemetry esqlUsageHolder; public UsageService() { this.handlers = new HashMap<>(); this.searchUsageHolder = new SearchUsageHolder(); this.ccsUsageHolder = new CCSUsageTelemetry(); + this.esqlUsageHolder = new CCSUsageTelemetry(false); } /** @@ -89,4 +91,8 @@ public SearchUsageHolder getSearchUsageHolder() { public CCSUsageTelemetry getCcsUsageHolder() { return ccsUsageHolder; } + + public CCSUsageTelemetry getEsqlUsageHolder() { + return esqlUsageHolder; + } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java index a72630c327ea2..6444caf08f831 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java @@ -352,4 +352,20 @@ public void testRanges() throws IOException { assertThat(value2Read.count(), equalTo(count1 + count2)); assertThat(value2Read.max(), equalTo(max1)); } + + public void testUseMRTFalse() { + CCSTelemetrySnapshot empty = new CCSTelemetrySnapshot(); + // Ignore MRT data + empty.setUseMRT(false); + + var randomWithMRT = randomValueOtherThanMany( + v -> v.getTookMrtTrue().count() == 0 || v.getTookMrtFalse().count() == 0, + this::randomCCSTelemetrySnapshot + ); + + empty.add(randomWithMRT); + assertThat(empty.getTook().count(), equalTo(randomWithMRT.getTook().count())); + assertThat(empty.getTookMrtFalse().count(), equalTo(0L)); + assertThat(empty.getTookMrtTrue().count(), equalTo(0L)); + } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetryTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetryTests.java index c4a2fdee1111e..5eb2224ec5f8e 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetryTests.java @@ -340,4 +340,23 @@ public void testConcurrentUpdates() throws InterruptedException { CCSTelemetrySnapshot expectedSnapshot = ccsUsageHolder.getCCSTelemetrySnapshot(); assertThat(snapshot, equalTo(expectedSnapshot)); } + + public void testUseMRTFalse() { + // Ignore MRT counters if instructed. + CCSUsageTelemetry ccsUsageHolder = new CCSUsageTelemetry(false); + + CCSUsage.Builder builder = new CCSUsage.Builder(); + builder.took(10L).setRemotesCount(1).setClient("kibana"); + builder.setFeature(MRT_FEATURE); + ccsUsageHolder.updateUsage(builder.build()); + + builder = new CCSUsage.Builder(); + builder.took(11L).setRemotesCount(1).setClient("kibana"); + ccsUsageHolder.updateUsage(builder.build()); + + CCSTelemetrySnapshot snapshot = ccsUsageHolder.getCCSTelemetrySnapshot(); + assertThat(snapshot.getTook().count(), equalTo(2L)); + assertThat(snapshot.getTookMrtFalse().count(), equalTo(0L)); + assertThat(snapshot.getTookMrtTrue().count(), equalTo(0L)); + } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java index 9bf4ad7c3cb64..fbd6e0916eefe 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java @@ -130,6 +130,7 @@ public void testCreation() { new ShardStats[] { shardStats }, new SearchUsageStats(), RepositoryUsageStats.EMPTY, + null, null ); diff --git a/server/src/test/java/org/elasticsearch/action/get/TransportMultiGetActionTests.java b/server/src/test/java/org/elasticsearch/action/get/TransportMultiGetActionTests.java index 667d6686767fa..dad392970b6a1 100644 --- a/server/src/test/java/org/elasticsearch/action/get/TransportMultiGetActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/get/TransportMultiGetActionTests.java @@ -145,15 +145,9 @@ public static void beforeClass() throws Exception { when( operationRouting.getShards(eq(clusterState), eq(index1.getName()), anyString(), nullable(String.class), nullable(String.class)) ).thenReturn(index1ShardIterator); - when(operationRouting.shardId(eq(clusterState), eq(index1.getName()), nullable(String.class), nullable(String.class))).thenReturn( - new ShardId(index1, randomInt()) - ); when( operationRouting.getShards(eq(clusterState), eq(index2.getName()), anyString(), nullable(String.class), nullable(String.class)) ).thenReturn(index2ShardIterator); - when(operationRouting.shardId(eq(clusterState), eq(index2.getName()), nullable(String.class), nullable(String.class))).thenReturn( - new ShardId(index2, randomInt()) - ); clusterService = mock(ClusterService.class); when(clusterService.localNode()).thenReturn(transportService.getLocalNode()); diff --git a/server/src/test/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsActionTests.java b/server/src/test/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsActionTests.java index df2b716ab691c..4655d2e47bac5 100644 --- a/server/src/test/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsActionTests.java @@ -146,15 +146,9 @@ public static void beforeClass() throws Exception { when( operationRouting.getShards(eq(clusterState), eq(index1.getName()), anyString(), nullable(String.class), nullable(String.class)) ).thenReturn(index1ShardIterator); - when(operationRouting.shardId(eq(clusterState), eq(index1.getName()), nullable(String.class), nullable(String.class))).thenReturn( - new ShardId(index1, randomInt()) - ); when( operationRouting.getShards(eq(clusterState), eq(index2.getName()), anyString(), nullable(String.class), nullable(String.class)) ).thenReturn(index2ShardIterator); - when(operationRouting.shardId(eq(clusterState), eq(index2.getName()), nullable(String.class), nullable(String.class))).thenReturn( - new ShardId(index2, randomInt()) - ); clusterService = mock(ClusterService.class); when(clusterService.localNode()).thenReturn(transportService.getLocalNode()); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index d45b2c119d2ae..633a678733f07 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -43,7 +43,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexModule; @@ -1496,8 +1495,6 @@ public void testRejectTranslogRetentionSettings() { ); } - @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT) - @AwaitsFix(bugUrl = "looks like a test that's not applicable to 9.0 after version bump") public void testDeprecateTranslogRetentionSettings() { request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); final Settings.Builder settings = Settings.builder(); @@ -1585,7 +1582,7 @@ public void testClusterStateCreateIndexWithClusterBlockTransformer() { .numberOfShards(1) .numberOfReplicas(nbReplicas) .build() - .withTimestampRanges(IndexLongFieldRange.UNKNOWN, IndexLongFieldRange.UNKNOWN, minTransportVersion), + .withTimestampRanges(IndexLongFieldRange.UNKNOWN, IndexLongFieldRange.UNKNOWN), null, MetadataCreateIndexService.createClusterBlocksTransformerForIndexCreation(settings), TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index e07ef15f7cf84..fc46afd10c18d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -34,7 +34,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Predicates; import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexNotFoundException; @@ -1987,8 +1986,6 @@ public void testHiddenAliasValidation() { } } - @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT) - @AwaitsFix(bugUrl = "this test needs to be updated or removed after the version 9.0 bump") public void testSystemAliasValidationMixedVersionSystemAndRegularFails() { final IndexVersion random7xVersion = IndexVersionUtils.randomVersionBetween( random(), @@ -2039,8 +2036,6 @@ public void testSystemAliasValidationNewSystemAndRegularFails() { ); } - @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT) - @AwaitsFix(bugUrl = "this test needs to be updated or removed after the version 9.0 bump") public void testSystemAliasOldSystemAndNewRegular() { final IndexVersion random7xVersion = IndexVersionUtils.randomVersionBetween( random(), @@ -2054,8 +2049,6 @@ public void testSystemAliasOldSystemAndNewRegular() { metadataWithIndices(oldVersionSystem, regularIndex); } - @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT) - @AwaitsFix(bugUrl = "this test needs to be updated or removed after the version 9.0 bump") public void testSystemIndexValidationAllRegular() { final IndexVersion random7xVersion = IndexVersionUtils.randomVersionBetween( random(), @@ -2070,8 +2063,6 @@ public void testSystemIndexValidationAllRegular() { metadataWithIndices(currentVersionSystem, currentVersionSystem2, oldVersionSystem); } - @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT) - @AwaitsFix(bugUrl = "this test needs to be updated or removed after the version 9.0 bump") public void testSystemAliasValidationAllSystemSomeOld() { final IndexVersion random7xVersion = IndexVersionUtils.randomVersionBetween( random(), diff --git a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java index c626be7983c46..ce8b0e457b762 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java @@ -81,7 +81,7 @@ public void testLevelPrecedence() { String uuid = UUIDs.randomBase64UUID(); IndexMetadata metadata = createIndexMetadata("index-precedence", settings(uuid)); IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - IndexingSlowLog log = new IndexingSlowLog(settings, mock(SlowLogFieldProvider.class)); + IndexingSlowLog log = new IndexingSlowLog(settings, mock(SlowLogFields.class)); ParsedDocument doc = EngineTestCase.createParsedDoc("1", null); Engine.Index index = new Engine.Index(Uid.encodeId("doc_id"), randomNonNegativeLong(), doc); @@ -142,7 +142,7 @@ public void testTwoLoggersDifferentLevel() { ), Settings.EMPTY ); - IndexingSlowLog log1 = new IndexingSlowLog(index1Settings, mock(SlowLogFieldProvider.class)); + IndexingSlowLog log1 = new IndexingSlowLog(index1Settings, mock(SlowLogFields.class)); IndexSettings index2Settings = new IndexSettings( createIndexMetadata( @@ -155,7 +155,7 @@ public void testTwoLoggersDifferentLevel() { ), Settings.EMPTY ); - IndexingSlowLog log2 = new IndexingSlowLog(index2Settings, mock(SlowLogFieldProvider.class)); + IndexingSlowLog log2 = new IndexingSlowLog(index2Settings, mock(SlowLogFields.class)); ParsedDocument doc = EngineTestCase.createParsedDoc("1", null); Engine.Index index = new Engine.Index(Uid.encodeId("doc_id"), randomNonNegativeLong(), doc); @@ -179,12 +179,12 @@ public void testMultipleSlowLoggersUseSingleLog4jLogger() { LoggerContext context = (LoggerContext) LogManager.getContext(false); IndexSettings index1Settings = new IndexSettings(createIndexMetadata("index1", settings(UUIDs.randomBase64UUID())), Settings.EMPTY); - IndexingSlowLog log1 = new IndexingSlowLog(index1Settings, mock(SlowLogFieldProvider.class)); + IndexingSlowLog log1 = new IndexingSlowLog(index1Settings, mock(SlowLogFields.class)); int numberOfLoggersBefore = context.getLoggers().size(); IndexSettings index2Settings = new IndexSettings(createIndexMetadata("index2", settings(UUIDs.randomBase64UUID())), Settings.EMPTY); - IndexingSlowLog log2 = new IndexingSlowLog(index2Settings, mock(SlowLogFieldProvider.class)); + IndexingSlowLog log2 = new IndexingSlowLog(index2Settings, mock(SlowLogFields.class)); context = (LoggerContext) LogManager.getContext(false); int numberOfLoggersAfter = context.getLoggers().size(); @@ -355,7 +355,7 @@ public void testReformatSetting() { .build() ); IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - IndexingSlowLog log = new IndexingSlowLog(settings, mock(SlowLogFieldProvider.class)); + IndexingSlowLog log = new IndexingSlowLog(settings, mock(SlowLogFields.class)); assertFalse(log.isReformat()); settings.updateIndexMetadata( newIndexMeta("index", Settings.builder().put(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING.getKey(), "true").build()) @@ -372,7 +372,7 @@ public void testReformatSetting() { metadata = newIndexMeta("index", Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build()); settings = new IndexSettings(metadata, Settings.EMPTY); - log = new IndexingSlowLog(settings, mock(SlowLogFieldProvider.class)); + log = new IndexingSlowLog(settings, mock(SlowLogFields.class)); assertTrue(log.isReformat()); try { settings.updateIndexMetadata( @@ -405,7 +405,7 @@ public void testSetLevels() { .build() ); IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - IndexingSlowLog log = new IndexingSlowLog(settings, mock(SlowLogFieldProvider.class)); + IndexingSlowLog log = new IndexingSlowLog(settings, mock(SlowLogFields.class)); assertEquals(TimeValue.timeValueMillis(100).nanos(), log.getIndexTraceThreshold()); assertEquals(TimeValue.timeValueMillis(200).nanos(), log.getIndexDebugThreshold()); assertEquals(TimeValue.timeValueMillis(300).nanos(), log.getIndexInfoThreshold()); @@ -436,7 +436,7 @@ public void testSetLevels() { assertEquals(TimeValue.timeValueMillis(-1).nanos(), log.getIndexWarnThreshold()); settings = new IndexSettings(metadata, Settings.EMPTY); - log = new IndexingSlowLog(settings, mock(SlowLogFieldProvider.class)); + log = new IndexingSlowLog(settings, mock(SlowLogFields.class)); assertEquals(TimeValue.timeValueMillis(-1).nanos(), log.getIndexTraceThreshold()); assertEquals(TimeValue.timeValueMillis(-1).nanos(), log.getIndexDebugThreshold()); diff --git a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java index 50e3269a6b9ba..359118c7cb5a1 100644 --- a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java +++ b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java @@ -103,7 +103,7 @@ public void testLevelPrecedence() { try (SearchContext ctx = searchContextWithSourceAndTask(createIndex("index"))) { String uuid = UUIDs.randomBase64UUID(); IndexSettings settings = new IndexSettings(createIndexMetadata("index", settings(uuid)), Settings.EMPTY); - SearchSlowLog log = new SearchSlowLog(settings, mock(SlowLogFieldProvider.class)); + SearchSlowLog log = new SearchSlowLog(settings, mock(SlowLogFields.class)); // For this test, when level is not breached, the level below should be used. { @@ -187,7 +187,7 @@ public void testTwoLoggersDifferentLevel() { ), Settings.EMPTY ); - SearchSlowLog log1 = new SearchSlowLog(settings1, mock(SlowLogFieldProvider.class)); + SearchSlowLog log1 = new SearchSlowLog(settings1, mock(SlowLogFields.class)); IndexSettings settings2 = new IndexSettings( createIndexMetadata( @@ -200,7 +200,7 @@ public void testTwoLoggersDifferentLevel() { ), Settings.EMPTY ); - SearchSlowLog log2 = new SearchSlowLog(settings2, mock(SlowLogFieldProvider.class)); + SearchSlowLog log2 = new SearchSlowLog(settings2, mock(SlowLogFields.class)); { // threshold set on WARN only, should not log @@ -223,7 +223,7 @@ public void testMultipleSlowLoggersUseSingleLog4jLogger() { try (SearchContext ctx1 = searchContextWithSourceAndTask(createIndex("index-1"))) { IndexSettings settings1 = new IndexSettings(createIndexMetadata("index-1", settings(UUIDs.randomBase64UUID())), Settings.EMPTY); - SearchSlowLog log1 = new SearchSlowLog(settings1, mock(SlowLogFieldProvider.class)); + SearchSlowLog log1 = new SearchSlowLog(settings1, mock(SlowLogFields.class)); int numberOfLoggersBefore = context.getLoggers().size(); try (SearchContext ctx2 = searchContextWithSourceAndTask(createIndex("index-2"))) { @@ -231,7 +231,7 @@ public void testMultipleSlowLoggersUseSingleLog4jLogger() { createIndexMetadata("index-2", settings(UUIDs.randomBase64UUID())), Settings.EMPTY ); - SearchSlowLog log2 = new SearchSlowLog(settings2, mock(SlowLogFieldProvider.class)); + SearchSlowLog log2 = new SearchSlowLog(settings2, mock(SlowLogFields.class)); int numberOfLoggersAfter = context.getLoggers().size(); assertThat(numberOfLoggersAfter, equalTo(numberOfLoggersBefore)); @@ -323,7 +323,7 @@ public void testSetQueryLevels() { .build() ); IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - SearchSlowLog log = new SearchSlowLog(settings, mock(SlowLogFieldProvider.class)); + SearchSlowLog log = new SearchSlowLog(settings, mock(SlowLogFields.class)); assertEquals(TimeValue.timeValueMillis(100).nanos(), log.getQueryTraceThreshold()); assertEquals(TimeValue.timeValueMillis(200).nanos(), log.getQueryDebugThreshold()); assertEquals(TimeValue.timeValueMillis(300).nanos(), log.getQueryInfoThreshold()); @@ -354,7 +354,7 @@ public void testSetQueryLevels() { assertEquals(TimeValue.timeValueMillis(-1).nanos(), log.getQueryWarnThreshold()); settings = new IndexSettings(metadata, Settings.EMPTY); - log = new SearchSlowLog(settings, mock(SlowLogFieldProvider.class)); + log = new SearchSlowLog(settings, mock(SlowLogFields.class)); assertEquals(TimeValue.timeValueMillis(-1).nanos(), log.getQueryTraceThreshold()); assertEquals(TimeValue.timeValueMillis(-1).nanos(), log.getQueryDebugThreshold()); @@ -429,7 +429,7 @@ public void testSetFetchLevels() { .build() ); IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - SearchSlowLog log = new SearchSlowLog(settings, mock(SlowLogFieldProvider.class)); + SearchSlowLog log = new SearchSlowLog(settings, mock(SlowLogFields.class)); assertEquals(TimeValue.timeValueMillis(100).nanos(), log.getFetchTraceThreshold()); assertEquals(TimeValue.timeValueMillis(200).nanos(), log.getFetchDebugThreshold()); assertEquals(TimeValue.timeValueMillis(300).nanos(), log.getFetchInfoThreshold()); @@ -460,7 +460,7 @@ public void testSetFetchLevels() { assertEquals(TimeValue.timeValueMillis(-1).nanos(), log.getFetchWarnThreshold()); settings = new IndexSettings(metadata, Settings.EMPTY); - log = new SearchSlowLog(settings, mock(SlowLogFieldProvider.class)); + log = new SearchSlowLog(settings, mock(SlowLogFields.class)); assertEquals(TimeValue.timeValueMillis(-1).nanos(), log.getFetchTraceThreshold()); assertEquals(TimeValue.timeValueMillis(-1).nanos(), log.getFetchDebugThreshold()); diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index d07c775da7e2a..26de6a7897786 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -3568,7 +3568,8 @@ public void testRecoverFromForeignTranslog() throws IOException { new TranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ); translog.add(TranslogOperationsUtils.indexOp("SomeBogusId", 0, primaryTerm.get())); assertEquals(generation.translogFileGeneration(), translog.currentFileGeneration()); diff --git a/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java b/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java new file mode 100644 index 0000000000000..b764bce464d15 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.engine; + +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; + +import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING; + +public class TranslogOperationAsserterTests extends EngineTestCase { + + @Override + protected Settings indexSettings() { + return Settings.builder() + .put(super.indexSettings()) + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) + .put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()) + .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) + .build(); + } + + Translog.Index toIndexOp(String source) throws IOException { + XContentParser parser = createParser(XContentType.JSON.xContent(), source); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.copyCurrentStructure(parser); + return new Translog.Index( + "1", + 0, + 1, + 1, + new BytesArray(Strings.toString(builder)), + null, + IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP + ); + } + + EngineConfig engineConfig(boolean useSyntheticSource) { + EngineConfig config = engine.config(); + Settings.Builder settings = Settings.builder().put(config.getIndexSettings().getSettings()); + if (useSyntheticSource) { + settings.put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()); + settings.put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true); + } else { + settings.put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.name()); + settings.put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), false); + } + IndexMetadata imd = IndexMetadata.builder(config.getIndexSettings().getIndexMetadata()).settings(settings).build(); + return config( + new IndexSettings(imd, Settings.EMPTY), + config.getStore(), + config.getTranslogConfig().getTranslogPath(), + config.getMergePolicy(), + null + ); + } + + public void testBasic() throws Exception { + TranslogOperationAsserter syntheticAsserter = TranslogOperationAsserter.withEngineConfig(engineConfig(true)); + TranslogOperationAsserter regularAsserter = TranslogOperationAsserter.withEngineConfig(engineConfig(false)); + { + var o1 = toIndexOp(""" + { + "value": "value-1" + } + """); + var o2 = toIndexOp(""" + { + "value": [ "value-1" ] + } + """); + assertTrue(syntheticAsserter.assertSameIndexOperation(o1, o2)); + assertFalse(regularAsserter.assertSameIndexOperation(o1, o2)); + } + { + var o1 = toIndexOp(""" + { + "value": [ "value-1", "value-2" ] + } + """); + var o2 = toIndexOp(""" + { + "value": [ "value-1", "value-2" ] + } + """); + assertTrue(syntheticAsserter.assertSameIndexOperation(o1, o2)); + assertTrue(regularAsserter.assertSameIndexOperation(o1, o2)); + } + { + var o1 = toIndexOp(""" + { + "value": [ "value-2", "value-1" ] + } + """); + var o2 = toIndexOp(""" + { + "value": [ "value-1", "value-2" ] + } + """); + assertTrue(syntheticAsserter.assertSameIndexOperation(o1, o2)); + assertFalse(regularAsserter.assertSameIndexOperation(o1, o2)); + } + { + var o1 = toIndexOp(""" + { + "value": [ "value-1", "value-2" ] + } + """); + var o2 = toIndexOp(""" + { + "value": [ "value-1", "value-2", "value-2" ] + } + """); + assertTrue(syntheticAsserter.assertSameIndexOperation(o1, o2)); + assertFalse(regularAsserter.assertSameIndexOperation(o1, o2)); + } + { + var o1 = toIndexOp(""" + { + "value": [ "value-1", "value-2" ] + } + """); + var o2 = toIndexOp(""" + { + "value": [ "value-1", "value-2", "value-3" ] + } + """); + assertFalse(syntheticAsserter.assertSameIndexOperation(o1, o2)); + assertFalse(regularAsserter.assertSameIndexOperation(o1, o2)); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java index 9d344a319055b..130c10130c4f3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java @@ -15,8 +15,8 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.geo.Orientation; +import org.elasticsearch.core.Strings; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geo.ShapeTestUtils; import org.elasticsearch.geometry.Geometry; @@ -56,7 +56,7 @@ public void ignoreTestGeoBoundsBlockLoader() throws IOException { ); } - private void testBoundsBlockLoaderAux( + private static void testBoundsBlockLoaderAux( CoordinateEncoder encoder, Supplier generator, Function indexerFactory, @@ -74,34 +74,36 @@ private void testBoundsBlockLoaderAux( iw.addDocument(doc); } } - // We specifically check just the even indices, to verify the loader can skip documents correctly. - var evenIndices = evenArray(geometries.size()); + + var expected = new ArrayList(); + var byteRefResults = new ArrayList(); + int currentIndex = 0; try (DirectoryReader reader = DirectoryReader.open(directory)) { - var byteRefResults = new ArrayList(); for (var leaf : reader.leaves()) { LeafReader leafReader = leaf.reader(); int numDocs = leafReader.numDocs(); - try ( - TestBlock block = (TestBlock) loader.reader(leaf) - .read(TestBlock.factory(leafReader.numDocs()), TestBlock.docs(evenArray(numDocs))) - ) { + // We specifically check just the even indices, to verify the loader can skip documents correctly. + int[] array = evenArray(numDocs); + for (int i = 0; i < array.length; i += 1) { + expected.add(visitor.apply(geometries.get(array[i] + currentIndex)).get()); + } + try (var block = (TestBlock) loader.reader(leaf).read(TestBlock.factory(leafReader.numDocs()), TestBlock.docs(array))) { for (int i = 0; i < block.size(); i++) { byteRefResults.add((BytesRef) block.get(i)); } } + currentIndex += numDocs; } - for (int i = 0; i < evenIndices.length; i++) { - var idx = evenIndices[i]; - var geometry = geometries.get(idx); - var geoString = geometry.toString(); - var geometryString = geoString.length() > 200 ? geoString.substring(0, 200) + "..." : geoString; - Rectangle r = visitor.apply(geometry).get(); - assertThat( - Strings.format("geometries[%d] ('%s') wasn't extracted correctly", idx, geometryString), - byteRefResults.get(i), - WellKnownBinaryBytesRefMatcher.encodes(RectangleMatcher.closeToFloat(r, 1e-3, encoder)) - ); - } + } + + for (int i = 0; i < expected.size(); i++) { + Rectangle rectangle = expected.get(i); + var geoString = rectangle.toString(); + assertThat( + Strings.format("geometry '%s' wasn't extracted correctly", geoString), + byteRefResults.get(i), + WellKnownBinaryBytesRefMatcher.encodes(RectangleMatcher.closeToFloat(rectangle, 1e-3, encoder)) + ); } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CustomSyntheticSourceFieldLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CustomSyntheticSourceFieldLookupTests.java deleted file mode 100644 index c3d6bbf285d77..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/mapper/CustomSyntheticSourceFieldLookupTests.java +++ /dev/null @@ -1,263 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.mapper; - -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.IndexVersion; - -import java.io.IOException; -import java.util.Map; - -public class CustomSyntheticSourceFieldLookupTests extends MapperServiceTestCase { - private static String MAPPING = """ - { - "_doc": { - "properties": { - "keep_all": { - "type": "keyword", - "synthetic_source_keep": "all" - }, - "keep_arrays": { - "type": "keyword", - "synthetic_source_keep": "arrays" - }, - "fallback_impl": { - "type": "long", - "doc_values": "false" - }, - "object_keep_all": { - "properties": {}, - "synthetic_source_keep": "all" - }, - "object_keep_arrays": { - "properties": {}, - "synthetic_source_keep": "arrays" - }, - "object_disabled": { - "properties": {}, - "enabled": "false" - }, - "nested_keep_all": { - "type": "nested", - "properties": {}, - "synthetic_source_keep": "all" - }, - "nested_disabled": { - "type": "nested", - "properties": {}, - "enabled": "false" - }, - "just_field": { - "type": "boolean" - }, - "just_object": { - "properties": {} - }, - "nested_obj": { - "properties": { - "keep_all": { - "type": "keyword", - "synthetic_source_keep": "all" - }, - "keep_arrays": { - "type": "keyword", - "synthetic_source_keep": "arrays" - }, - "fallback_impl": { - "type": "long", - "doc_values": "false" - }, - "object_keep_all": { - "properties": {}, - "synthetic_source_keep": "all" - }, - "object_keep_arrays": { - "properties": {}, - "synthetic_source_keep": "arrays" - }, - "object_disabled": { - "properties": {}, - "enabled": "false" - }, - "nested_keep_all": { - "type": "nested", - "properties": {}, - "synthetic_source_keep": "all" - }, - "nested_disabled": { - "type": "nested", - "properties": {}, - "enabled": "false" - }, - "just_field": { - "type": "boolean" - }, - "just_object": { - "properties": {} - } - } - }, - "nested_nested": { - "properties": { - "keep_all": { - "type": "keyword", - "synthetic_source_keep": "all" - }, - "keep_arrays": { - "type": "keyword", - "synthetic_source_keep": "arrays" - }, - "fallback_impl": { - "type": "long", - "doc_values": "false" - }, - "object_keep_all": { - "properties": {}, - "synthetic_source_keep": "all" - }, - "object_keep_arrays": { - "properties": {}, - "synthetic_source_keep": "arrays" - }, - "object_disabled": { - "properties": {}, - "enabled": "false" - }, - "nested_keep_all": { - "type": "nested", - "properties": {}, - "synthetic_source_keep": "all" - }, - "nested_disabled": { - "type": "nested", - "properties": {}, - "enabled": "false" - }, - "just_field": { - "type": "boolean" - }, - "just_object": { - "properties": {} - } - } - } - } - } - } - """; - - public void testIsNoopWhenSourceIsNotSynthetic() throws IOException { - var mapping = createMapperService(MAPPING).mappingLookup().getMapping(); - var indexSettings = indexSettings(Mapper.SourceKeepMode.NONE); - var sut = new CustomSyntheticSourceFieldLookup(mapping, indexSettings, false); - - assertEquals(sut.getFieldsWithCustomSyntheticSourceHandling(), Map.of()); - } - - public void testDetectsLeafWithKeepAll() throws IOException { - var mapping = createMapperService(MAPPING).mappingLookup().getMapping(); - var indexSettings = indexSettings(Mapper.SourceKeepMode.NONE); - var sut = new CustomSyntheticSourceFieldLookup(mapping, indexSettings, true); - - var fields = sut.getFieldsWithCustomSyntheticSourceHandling(); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ALL, fields.get("keep_all")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ALL, fields.get("nested_obj.keep_all")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ALL, fields.get("nested_nested.keep_all")); - } - - public void testDetectsLeafWithKeepArrays() throws IOException { - var mapping = createMapperService(MAPPING).mappingLookup().getMapping(); - var indexSettings = indexSettings(Mapper.SourceKeepMode.NONE); - var sut = new CustomSyntheticSourceFieldLookup(mapping, indexSettings, true); - - var fields = sut.getFieldsWithCustomSyntheticSourceHandling(); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("keep_arrays")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("nested_obj.keep_arrays")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("nested_nested.keep_arrays")); - } - - public void testDetectsLeafWithFallback() throws IOException { - var mapping = createMapperService(MAPPING).mappingLookup().getMapping(); - var indexSettings = indexSettings(Mapper.SourceKeepMode.NONE); - var sut = new CustomSyntheticSourceFieldLookup(mapping, indexSettings, true); - - var fields = sut.getFieldsWithCustomSyntheticSourceHandling(); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.FALLBACK_SYNTHETIC_SOURCE, fields.get("fallback_impl")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.FALLBACK_SYNTHETIC_SOURCE, fields.get("nested_obj.fallback_impl")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.FALLBACK_SYNTHETIC_SOURCE, fields.get("nested_nested.fallback_impl")); - } - - public void testDetectsObjectWithKeepAll() throws IOException { - var mapping = createMapperService(MAPPING).mappingLookup().getMapping(); - var indexSettings = indexSettings(Mapper.SourceKeepMode.NONE); - var sut = new CustomSyntheticSourceFieldLookup(mapping, indexSettings, true); - - var fields = sut.getFieldsWithCustomSyntheticSourceHandling(); - - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ALL, fields.get("object_keep_all")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ALL, fields.get("nested_obj.object_keep_all")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ALL, fields.get("nested_nested.object_keep_all")); - - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ALL, fields.get("nested_keep_all")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ALL, fields.get("nested_obj.nested_keep_all")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ALL, fields.get("nested_nested.nested_keep_all")); - } - - public void testDetectsObjectWithKeepArrays() throws IOException { - var mapping = createMapperService(MAPPING).mappingLookup().getMapping(); - var indexSettings = indexSettings(Mapper.SourceKeepMode.NONE); - var sut = new CustomSyntheticSourceFieldLookup(mapping, indexSettings, true); - - var fields = sut.getFieldsWithCustomSyntheticSourceHandling(); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("object_keep_arrays")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("nested_obj.object_keep_arrays")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("nested_nested.object_keep_arrays")); - } - - public void testDetectsDisabledObject() throws IOException { - var mapping = createMapperService(MAPPING).mappingLookup().getMapping(); - var indexSettings = indexSettings(Mapper.SourceKeepMode.NONE); - var sut = new CustomSyntheticSourceFieldLookup(mapping, indexSettings, true); - - var fields = sut.getFieldsWithCustomSyntheticSourceHandling(); - - assertEquals(CustomSyntheticSourceFieldLookup.Reason.DISABLED_OBJECT, fields.get("object_disabled")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.DISABLED_OBJECT, fields.get("nested_obj.object_disabled")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.DISABLED_OBJECT, fields.get("nested_nested.object_disabled")); - - assertEquals(CustomSyntheticSourceFieldLookup.Reason.DISABLED_OBJECT, fields.get("nested_disabled")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.DISABLED_OBJECT, fields.get("nested_obj.nested_disabled")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.DISABLED_OBJECT, fields.get("nested_nested.nested_disabled")); - } - - public void testAppliesIndexLevelSourceKeepMode() throws IOException { - var mapping = createMapperService(MAPPING).mappingLookup().getMapping(); - var indexSettings = indexSettings(Mapper.SourceKeepMode.ARRAYS); - var sut = new CustomSyntheticSourceFieldLookup(mapping, indexSettings, true); - - var fields = sut.getFieldsWithCustomSyntheticSourceHandling(); - - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("just_field")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("nested_obj.just_field")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("nested_nested.just_field")); - - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("just_object")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("nested_obj.just_object")); - assertEquals(CustomSyntheticSourceFieldLookup.Reason.SOURCE_KEEP_ARRAYS, fields.get("nested_nested.just_object")); - } - - private static IndexSettings indexSettings(Mapper.SourceKeepMode sourceKeepMode) { - return createIndexSettings( - IndexVersion.current(), - Settings.builder().put(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING.getKey(), sourceKeepMode).build() - ); - } -} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java index 9e617f638e755..b2ba3d60d2174 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java @@ -81,7 +81,6 @@ public void testAddFields() throws Exception { merged, merged.toCompressedXContent(), IndexVersion.current(), - null, MapperMetrics.NOOP, "myIndex" ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserListenerTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserListenerTests.java deleted file mode 100644 index a1dafacfb7b23..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserListenerTests.java +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.mapper; - -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentParserConfiguration; -import org.elasticsearch.xcontent.XContentType; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -public class DocumentParserListenerTests extends MapperServiceTestCase { - private static class MemorizingDocumentParserListener implements DocumentParserListener { - private final List events = new ArrayList<>(); - private final List tokens = new ArrayList<>(); - - @Override - public boolean isActive() { - return true; - } - - @Override - public void consume(Token token) throws IOException { - // Tokens contains information tied to current parser state so we need to "materialize" them. - if (token instanceof Token.StringAsCharArrayValue charArray) { - var string = String.copyValueOf(charArray.buffer(), charArray.offset(), charArray.length()); - tokens.add((Token.ValueToken) () -> string); - } else if (token instanceof Token.ValueToken v) { - var value = v.value(); - tokens.add((Token.ValueToken) () -> value); - } else { - tokens.add(token); - } - } - - @Override - public void consume(Event event) throws IOException { - events.add(event); - } - - @Override - public Output finish() { - return new Output(List.of()); - } - - public List getTokens() { - return tokens; - } - - public List getEvents() { - return events; - } - } - - public void testEventFlow() throws IOException { - var mapping = XContentBuilder.builder(XContentType.JSON.xContent()).startObject().startObject("_doc").startObject("properties"); - { - mapping.startObject("leaf").field("type", "keyword").endObject(); - mapping.startObject("leaf_array").field("type", "keyword").endObject(); - - mapping.startObject("object").startObject("properties"); - { - mapping.startObject("leaf").field("type", "keyword").endObject(); - mapping.startObject("leaf_array").field("type", "keyword").endObject(); - } - mapping.endObject().endObject(); - - mapping.startObject("object_array").startObject("properties"); - { - mapping.startObject("leaf").field("type", "keyword").endObject(); - mapping.startObject("leaf_array").field("type", "keyword").endObject(); - } - mapping.endObject().endObject(); - } - mapping.endObject().endObject().endObject(); - var mappingService = createSytheticSourceMapperService(mapping); - - XContentType xContentType = randomFrom(XContentType.values()); - - var listener = new MemorizingDocumentParserListener(); - var documentParser = new DocumentParser( - XContentParserConfiguration.EMPTY, - mappingService.parserContext(), - (ml, xct) -> new DocumentParser.Listeners.Single(listener) - ); - - var source = XContentBuilder.builder(xContentType.xContent()); - source.startObject(); - { - source.field("leaf", "leaf"); - source.array("leaf_array", "one", "two"); - source.startObject("object"); - { - source.field("leaf", "leaf"); - source.array("leaf_array", "one", "two"); - } - source.endObject(); - source.startArray("object_array"); - { - source.startObject(); - { - source.field("leaf", "leaf"); - source.array("leaf_array", "one", "two"); - } - source.endObject(); - } - source.endArray(); - } - source.endObject(); - - documentParser.parseDocument(new SourceToParse("id1", BytesReference.bytes(source), xContentType), mappingService.mappingLookup()); - var events = listener.getEvents(); - - assertEquals("_doc", ((DocumentParserListener.Event.DocumentStart) events.get(0)).rootObjectMapper().fullPath()); - - assertLeafEvents(events, 1, "", "_doc", false); - - var objectStart = (DocumentParserListener.Event.ObjectStart) events.get(5); - assertEquals("object", objectStart.objectMapper().fullPath()); - assertEquals("_doc", objectStart.parentMapper().fullPath()); - assertFalse(objectStart.insideObjectArray()); - assertLeafEvents(events, 6, "object.", "object", false); - - var objectArrayStart = (DocumentParserListener.Event.ObjectArrayStart) events.get(10); - assertEquals("object_array", objectArrayStart.objectMapper().fullPath()); - assertEquals("_doc", objectArrayStart.parentMapper().fullPath()); - - var objectInArrayStart = (DocumentParserListener.Event.ObjectStart) events.get(11); - assertEquals("object_array", objectInArrayStart.objectMapper().fullPath()); - assertEquals("_doc", objectInArrayStart.parentMapper().fullPath()); - assertTrue(objectInArrayStart.insideObjectArray()); - assertLeafEvents(events, 12, "object_array.", "object_array", true); - } - - public void testTokenFlow() throws IOException { - var mapping = XContentBuilder.builder(XContentType.JSON.xContent()) - .startObject() - .startObject("_doc") - .field("enabled", false) - .endObject() - .endObject(); - var mappingService = createSytheticSourceMapperService(mapping); - - XContentType xContentType = randomFrom(XContentType.values()); - - var listener = new MemorizingDocumentParserListener(); - var documentParser = new DocumentParser( - XContentParserConfiguration.EMPTY, - mappingService.parserContext(), - (ml, xct) -> new DocumentParser.Listeners.Single(listener) - ); - - var source = XContentBuilder.builder(xContentType.xContent()); - source.startObject(); - { - source.field("leaf", "leaf"); - source.array("leaf_array", "one", "two"); - source.startObject("object"); - { - source.field("leaf", "leaf"); - source.array("leaf_array", "one", "two"); - } - source.endObject(); - source.startArray("object_array"); - { - source.startObject(); - { - source.field("leaf", "leaf"); - source.array("leaf_array", "one", "two"); - } - source.endObject(); - } - source.endArray(); - } - source.endObject(); - - documentParser.parseDocument(new SourceToParse("id1", BytesReference.bytes(source), xContentType), mappingService.mappingLookup()); - var tokens = listener.getTokens(); - assertTrue(tokens.get(0) instanceof DocumentParserListener.Token.StartObject); - { - assertLeafTokens(tokens, 1); - assertEquals("object", ((DocumentParserListener.Token.FieldName) tokens.get(8)).name()); - assertTrue(tokens.get(9) instanceof DocumentParserListener.Token.StartObject); - { - assertLeafTokens(tokens, 10); - } - assertTrue(tokens.get(17) instanceof DocumentParserListener.Token.EndObject); - assertEquals("object_array", ((DocumentParserListener.Token.FieldName) tokens.get(18)).name()); - assertTrue(tokens.get(19) instanceof DocumentParserListener.Token.StartArray); - { - assertTrue(tokens.get(20) instanceof DocumentParserListener.Token.StartObject); - { - assertLeafTokens(tokens, 21); - } - assertTrue(tokens.get(28) instanceof DocumentParserListener.Token.EndObject); - } - assertTrue(tokens.get(29) instanceof DocumentParserListener.Token.EndArray); - } - assertTrue(tokens.get(30) instanceof DocumentParserListener.Token.EndObject); - } - - private void assertLeafEvents( - List events, - int start, - String prefix, - String parent, - boolean inObjectArray - ) { - var leafValue = (DocumentParserListener.Event.LeafValue) events.get(start); - assertEquals(prefix + "leaf", leafValue.fieldMapper().fullPath()); - assertEquals(parent, leafValue.parentMapper().fullPath()); - assertFalse(leafValue.isArray()); - assertFalse(leafValue.isContainer()); - assertEquals(inObjectArray, leafValue.insideObjectArray()); - - var leafArray = (DocumentParserListener.Event.LeafArrayStart) events.get(start + 1); - assertEquals(prefix + "leaf_array", leafArray.fieldMapper().fullPath()); - assertEquals(parent, leafArray.parentMapper().fullPath()); - - var arrayValue1 = (DocumentParserListener.Event.LeafValue) events.get(start + 2); - assertEquals(prefix + "leaf_array", arrayValue1.fieldMapper().fullPath()); - assertEquals(parent, arrayValue1.parentMapper().fullPath()); - assertFalse(arrayValue1.isArray()); - assertFalse(arrayValue1.isContainer()); - assertEquals(inObjectArray, leafValue.insideObjectArray()); - - var arrayValue2 = (DocumentParserListener.Event.LeafValue) events.get(start + 3); - assertEquals(prefix + "leaf_array", arrayValue2.fieldMapper().fullPath()); - assertEquals(parent, arrayValue2.parentMapper().fullPath()); - assertFalse(arrayValue2.isArray()); - assertFalse(arrayValue2.isContainer()); - assertEquals(inObjectArray, leafValue.insideObjectArray()); - } - - private void assertLeafTokens(List tokens, int start) throws IOException { - assertEquals("leaf", ((DocumentParserListener.Token.FieldName) tokens.get(start)).name()); - assertEquals("leaf", ((DocumentParserListener.Token.ValueToken) tokens.get(start + 1)).value()); - assertEquals("leaf_array", ((DocumentParserListener.Token.FieldName) tokens.get(start + 2)).name()); - assertTrue(tokens.get(start + 3) instanceof DocumentParserListener.Token.StartArray); - assertEquals("one", ((DocumentParserListener.Token.ValueToken) tokens.get(start + 4)).value()); - assertEquals("two", ((DocumentParserListener.Token.ValueToken) tokens.get(start + 5)).value()); - assertTrue(tokens.get(start + 6) instanceof DocumentParserListener.Token.EndArray); - } -} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index 3699f97e243af..d128b25038a59 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -2688,7 +2688,6 @@ same name need to be part of the same mappings (hence the same document). If th newMapping, newMapping.toCompressedXContent(), IndexVersion.current(), - mapperService.getIndexSettings(), MapperMetrics.NOOP, "myIndex" ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java index 1f020520e7e35..d4d0e67ff4141 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java @@ -9,11 +9,8 @@ package org.elasticsearch.index.mapper; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; @@ -78,14 +75,7 @@ public void testCreateDynamicStringFieldAsKeywordForDimension() throws IOExcepti ).build(MapperBuilderContext.root(false, false)); Mapping mapping = new Mapping(root, new MetadataFieldMapper[] { sourceMapper }, Map.of()); - IndexMetadata indexMetadata = IndexMetadata.builder("index") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - .build(); - IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); - - DocumentParserContext ctx = new TestDocumentParserContext(MappingLookup.fromMapping(mapping, indexSettings), sourceToParse) { + DocumentParserContext ctx = new TestDocumentParserContext(MappingLookup.fromMapping(mapping), sourceToParse) { @Override public XContentParser parser() { return parser; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index dbfc1f114fffb..e385177b87147 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -204,6 +204,6 @@ private static MappingLookup createMappingLookup( new MetadataFieldMapper[0], Collections.emptyMap() ); - return MappingLookup.fromMappers(mapping, fieldMappers, objectMappers, fieldAliasMappers, emptyList(), null); + return MappingLookup.fromMappers(mapping, fieldMappers, objectMappers, fieldAliasMappers, emptyList()); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java index 67b62530f3443..4a45824342c74 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java @@ -22,6 +22,8 @@ import java.util.List; import java.util.stream.Stream; +import static java.util.Collections.emptyList; + public class FieldNamesFieldTypeTests extends ESTestCase { public void testTermQuery() { @@ -34,6 +36,7 @@ public void testTermQuery() { settings ); List mappers = Stream.of(fieldNamesFieldType, fieldType).map(MockFieldMapper::new).toList(); + MappingLookup mappingLookup = MappingLookup.fromMappers(Mapping.EMPTY, mappers, emptyList()); SearchExecutionContext searchExecutionContext = SearchExecutionContextHelper.createSimple(indexSettings, null, null); Query termQuery = fieldNamesFieldType.termQuery("field_name", searchExecutionContext); assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.CONTENT_TYPE, "field_name")), termQuery); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 5c9765eefb98d..14902aa419b9f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -848,7 +848,7 @@ public void testIndexStoredArraySourceRootObjectArrayWithBypass() throws IOExcep b.field("bool_value", true); }); assertEquals(""" - {"bool_value":true,"path":{"int_value":[10,20]}}""", syntheticSource); + {"bool_value":true,"path":{"int_value":[20,10]}}""", syntheticSource); } public void testIndexStoredArraySourceNestedValueArray() throws IOException { @@ -912,7 +912,7 @@ public void testIndexStoredArraySourceNestedValueArrayDisabled() throws IOExcept b.endObject(); }); assertEquals(""" - {"path":{"bool_value":true,"int_value":[10,20,30],"obj":{"foo":[1,2]}}}""", syntheticSource); + {"path":{"bool_value":true,"int_value":[10,20,30],"obj":{"foo":[2,1]}}}""", syntheticSource); } public void testFieldStoredArraySourceNestedValueArray() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java index 71edce3d1549a..fd44e68df19a8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java @@ -13,12 +13,8 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.TimeSeriesParams.MetricType; @@ -53,13 +49,7 @@ private static MappingLookup createMappingLookup( new MetadataFieldMapper[0], Collections.emptyMap() ); - IndexMetadata indexMetadata = IndexMetadata.builder("index") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - .build(); - IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); - return MappingLookup.fromMappers(mapping, fieldMappers, objectMappers, indexSettings); + return MappingLookup.fromMappers(mapping, fieldMappers, objectMappers); } public void testOnlyRuntimeField() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java index 289ec24f7d3b9..b87ab09c530d6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java @@ -107,7 +107,7 @@ public void testFieldNameWithDeepDots() throws Exception { b.endObject(); }); Mapping mapping = createMappingParser(Settings.EMPTY).parse("_doc", new CompressedXContent(BytesReference.bytes(builder))); - MappingLookup mappingLookup = MappingLookup.fromMapping(mapping, null); + MappingLookup mappingLookup = MappingLookup.fromMapping(mapping); assertNotNull(mappingLookup.getMapper("foo.bar")); assertNotNull(mappingLookup.getMapper("foo.baz.deep.field")); assertNotNull(mappingLookup.objectMappers().get("foo")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SyntheticSourceDocumentParserListenerTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SyntheticSourceDocumentParserListenerTests.java deleted file mode 100644 index b5496e1001bf4..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/mapper/SyntheticSourceDocumentParserListenerTests.java +++ /dev/null @@ -1,385 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.mapper; - -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentType; - -import java.io.IOException; -import java.math.BigInteger; - -public class SyntheticSourceDocumentParserListenerTests extends MapperServiceTestCase { - public void testStoreLeafValue() throws IOException { - XContentType xContentType = randomFrom(XContentType.values()); - - var mapping = fieldMapping(b -> b.field("type", "long").field("synthetic_source_keep", "all")); - var mappingLookup = createSytheticSourceMapperService(mapping).mappingLookup(); - var sut = new SyntheticSourceDocumentParserListener(mappingLookup, xContentType); - - var doc = new LuceneDocument(); - - var value = XContentBuilder.builder(xContentType.xContent()).value(1234L); - var parser = createParser(value); - parser.nextToken(); - - sut.consume( - new DocumentParserListener.Event.LeafValue( - (FieldMapper) mappingLookup.getMapper("field"), - false, - mappingLookup.getMapping().getRoot(), - doc, - parser - ) - ); - - var output = sut.finish(); - - assertEquals(1, output.ignoredSourceValues().size()); - var valueToStore = output.ignoredSourceValues().get(0); - assertEquals("field", valueToStore.name()); - var decoded = XContentBuilder.builder(xContentType.xContent()); - XContentDataHelper.decodeAndWrite(decoded, valueToStore.value()); - assertEquals(BytesReference.bytes(value), BytesReference.bytes(decoded)); - } - - public void testStoreLeafArray() throws IOException { - XContentType xContentType = randomFrom(XContentType.values()); - - var mapping = fieldMapping(b -> b.field("type", "long").field("synthetic_source_keep", "all")); - var mappingLookup = createSytheticSourceMapperService(mapping).mappingLookup(); - var sut = new SyntheticSourceDocumentParserListener(mappingLookup, xContentType); - - var values = randomList(0, 10, ESTestCase::randomLong); - - var doc = new LuceneDocument(); - - sut.consume( - new DocumentParserListener.Event.LeafArrayStart( - (FieldMapper) mappingLookup.getMapper("field"), - mappingLookup.getMapping().getRoot(), - doc - ) - ); - for (long l : values) { - sut.consume((DocumentParserListener.Token.ValueToken) () -> l); - } - sut.consume(DocumentParserListener.Token.END_ARRAY); - - var output = sut.finish(); - - assertEquals(1, output.ignoredSourceValues().size()); - var valueToStore = output.ignoredSourceValues().get(0); - assertEquals("field", valueToStore.name()); - - var decoded = XContentBuilder.builder(xContentType.xContent()); - XContentDataHelper.decodeAndWrite(decoded, valueToStore.value()); - - var parser = createParser(decoded); - assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken()); - for (long l : values) { - parser.nextToken(); - assertEquals(XContentParser.Token.VALUE_NUMBER, parser.currentToken()); - assertEquals(l, parser.longValue()); - } - assertEquals(XContentParser.Token.END_ARRAY, parser.nextToken()); - } - - public void testStoreObject() throws IOException { - XContentType xContentType = randomFrom(XContentType.values()); - - var mapping = fieldMapping(b -> b.field("type", "object").field("synthetic_source_keep", "all")); - var mappingLookup = createSytheticSourceMapperService(mapping).mappingLookup(); - var sut = new SyntheticSourceDocumentParserListener(mappingLookup, xContentType); - - var names = randomList(0, 10, () -> randomAlphaOfLength(10)); - var values = randomList(names.size(), names.size(), () -> randomAlphaOfLength(10)); - - var doc = new LuceneDocument(); - - sut.consume( - new DocumentParserListener.Event.ObjectStart( - mappingLookup.objectMappers().get("field"), - false, - mappingLookup.getMapping().getRoot(), - doc - ) - ); - for (int i = 0; i < names.size(); i++) { - sut.consume(new DocumentParserListener.Token.FieldName(names.get(i))); - var value = values.get(i); - sut.consume((DocumentParserListener.Token.ValueToken) () -> value); - } - sut.consume(DocumentParserListener.Token.END_OBJECT); - - var output = sut.finish(); - - assertEquals(1, output.ignoredSourceValues().size()); - var valueToStore = output.ignoredSourceValues().get(0); - assertEquals("field", valueToStore.name()); - - var decoded = XContentBuilder.builder(xContentType.xContent()); - XContentDataHelper.decodeAndWrite(decoded, valueToStore.value()); - - var parser = createParser(decoded); - assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken()); - for (int i = 0; i < names.size(); i++) { - parser.nextToken(); - assertEquals(XContentParser.Token.FIELD_NAME, parser.currentToken()); - assertEquals(names.get(i), parser.currentName()); - - assertEquals(XContentParser.Token.VALUE_STRING, parser.nextToken()); - assertEquals(values.get(i), parser.text()); - } - assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken()); - } - - public void testStoreObjectArray() throws IOException { - XContentType xContentType = randomFrom(XContentType.values()); - - var mapping = fieldMapping(b -> b.field("type", "object").field("synthetic_source_keep", "all")); - var mappingLookup = createSytheticSourceMapperService(mapping).mappingLookup(); - var sut = new SyntheticSourceDocumentParserListener(mappingLookup, xContentType); - - var names = randomList(0, 10, () -> randomAlphaOfLength(10)); - var values = randomList(names.size(), names.size(), () -> randomAlphaOfLength(10)); - - var doc = new LuceneDocument(); - - sut.consume( - new DocumentParserListener.Event.ObjectArrayStart( - mappingLookup.objectMappers().get("field"), - mappingLookup.getMapping().getRoot(), - doc - ) - ); - for (int i = 0; i < names.size(); i++) { - sut.consume(DocumentParserListener.Token.START_OBJECT); - - sut.consume(new DocumentParserListener.Token.FieldName(names.get(i))); - var value = values.get(i); - sut.consume((DocumentParserListener.Token.ValueToken) () -> value); - - sut.consume(DocumentParserListener.Token.END_OBJECT); - } - sut.consume(DocumentParserListener.Token.END_ARRAY); - - var output = sut.finish(); - - assertEquals(1, output.ignoredSourceValues().size()); - var valueToStore = output.ignoredSourceValues().get(0); - assertEquals("field", valueToStore.name()); - - var decoded = XContentBuilder.builder(xContentType.xContent()); - XContentDataHelper.decodeAndWrite(decoded, valueToStore.value()); - - var parser = createParser(decoded); - assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken()); - for (int i = 0; i < names.size(); i++) { - assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken()); - assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken()); - assertEquals(names.get(i), parser.currentName()); - - assertEquals(XContentParser.Token.VALUE_STRING, parser.nextToken()); - assertEquals(values.get(i), parser.text()); - - assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken()); - } - assertEquals(XContentParser.Token.END_ARRAY, parser.nextToken()); - } - - public void testStashedLeafValue() throws IOException { - XContentType xContentType = randomFrom(XContentType.values()); - - var mapping = fieldMapping(b -> b.field("type", "boolean").field("synthetic_source_keep", "arrays")); - var mappingLookup = createSytheticSourceMapperService(mapping).mappingLookup(); - var sut = new SyntheticSourceDocumentParserListener(mappingLookup, xContentType); - - var doc = new LuceneDocument(); - - var value = XContentBuilder.builder(xContentType.xContent()).value(false); - var parser = createParser(value); - parser.nextToken(); - - sut.consume( - new DocumentParserListener.Event.LeafValue( - (FieldMapper) mappingLookup.getMapper("field"), - true, - mappingLookup.getMapping().getRoot(), - doc, - parser - ) - ); - - sut.consume( - new DocumentParserListener.Event.LeafValue( - (FieldMapper) mappingLookup.getMapper("field"), - true, - mappingLookup.getMapping().getRoot(), - doc, - parser - ) - ); - - var output = sut.finish(); - - // Single values are optimized away because there are no arrays mixed in and regular synthetic source logic is sufficient - assertEquals(0, output.ignoredSourceValues().size()); - } - - public void testStashedMixedValues() throws IOException { - XContentType xContentType = randomFrom(XContentType.values()); - - var mapping = fieldMapping(b -> b.field("type", "boolean").field("synthetic_source_keep", "arrays")); - var mappingLookup = createSytheticSourceMapperService(mapping).mappingLookup(); - var sut = new SyntheticSourceDocumentParserListener(mappingLookup, xContentType); - - var doc = new LuceneDocument(); - - var value = XContentBuilder.builder(xContentType.xContent()).value(false); - var parser = createParser(value); - parser.nextToken(); - - sut.consume( - new DocumentParserListener.Event.LeafValue( - (FieldMapper) mappingLookup.getMapper("field"), - true, - mappingLookup.getMapping().getRoot(), - doc, - parser - ) - ); - - sut.consume( - new DocumentParserListener.Event.LeafValue( - (FieldMapper) mappingLookup.getMapper("field"), - true, - mappingLookup.getMapping().getRoot(), - doc, - parser - ) - ); - - sut.consume( - new DocumentParserListener.Event.LeafArrayStart( - (FieldMapper) mappingLookup.getMapper("field"), - mappingLookup.getMapping().getRoot(), - doc - ) - ); - sut.consume((DocumentParserListener.Token.ValueToken) () -> true); - sut.consume((DocumentParserListener.Token.ValueToken) () -> true); - sut.consume(DocumentParserListener.Token.END_ARRAY); - - var output = sut.finish(); - - // Both arrays and individual values are stored. - assertEquals(3, output.ignoredSourceValues().size()); - } - - public void testStashedObjectValue() throws IOException { - XContentType xContentType = randomFrom(XContentType.values()); - - var mapping = fieldMapping(b -> b.field("type", "object").field("synthetic_source_keep", "arrays")); - var mappingLookup = createSytheticSourceMapperService(mapping).mappingLookup(); - var sut = new SyntheticSourceDocumentParserListener(mappingLookup, xContentType); - - var doc = new LuceneDocument(); - - var value = XContentBuilder.builder(xContentType.xContent()).value(1234L); - var parser = createParser(value); - parser.nextToken(); - - sut.consume( - new DocumentParserListener.Event.ObjectStart( - mappingLookup.objectMappers().get("field"), - true, - mappingLookup.getMapping().getRoot(), - doc - ) - ); - sut.consume(new DocumentParserListener.Token.FieldName("hello")); - sut.consume((DocumentParserListener.Token.ValueToken) () -> BigInteger.valueOf(13)); - sut.consume(DocumentParserListener.Token.END_OBJECT); - - var output = sut.finish(); - - // Single value optimization does not work for objects because it is possible that one of the fields - // of this object needs to be stored in ignored source. - // Because we stored the entire object we didn't store individual fields separately. - // Optimizing this away would lead to missing data from synthetic source in some cases. - // We could do both, but we don't do it now. - assertEquals(1, output.ignoredSourceValues().size()); - } - - public void testSingleElementArray() throws IOException { - XContentType xContentType = randomFrom(XContentType.values()); - - var mapping = fieldMapping(b -> b.field("type", "boolean").field("synthetic_source_keep", "arrays")); - var mappingLookup = createSytheticSourceMapperService(mapping).mappingLookup(); - var sut = new SyntheticSourceDocumentParserListener(mappingLookup, xContentType); - - var doc = new LuceneDocument(); - - sut.consume( - new DocumentParserListener.Event.LeafArrayStart( - (FieldMapper) mappingLookup.getMapper("field"), - mappingLookup.getMapping().getRoot(), - doc - ) - ); - sut.consume((DocumentParserListener.Token.ValueToken) () -> true); - sut.consume(DocumentParserListener.Token.END_ARRAY); - - var output = sut.finish(); - - // Since there is only one value in the array, order does not matter, - // and we can drop ignored source value and use standard synthetic source logic. - assertEquals(0, output.ignoredSourceValues().size()); - } - - public void testMultipleSingleElementArrays() throws IOException { - XContentType xContentType = randomFrom(XContentType.values()); - - var mapping = fieldMapping(b -> b.field("type", "boolean").field("synthetic_source_keep", "arrays")); - var mappingLookup = createSytheticSourceMapperService(mapping).mappingLookup(); - var sut = new SyntheticSourceDocumentParserListener(mappingLookup, xContentType); - - var doc = new LuceneDocument(); - - sut.consume( - new DocumentParserListener.Event.LeafArrayStart( - (FieldMapper) mappingLookup.getMapper("field"), - mappingLookup.getMapping().getRoot(), - doc - ) - ); - sut.consume((DocumentParserListener.Token.ValueToken) () -> true); - sut.consume(DocumentParserListener.Token.END_ARRAY); - - sut.consume( - new DocumentParserListener.Event.LeafArrayStart( - (FieldMapper) mappingLookup.getMapper("field"), - mappingLookup.getMapping().getRoot(), - doc - ) - ); - sut.consume((DocumentParserListener.Token.ValueToken) () -> false); - sut.consume(DocumentParserListener.Token.END_ARRAY); - - var output = sut.finish(); - - // Since there is only one value in the array, order does not matter, - // and we can drop ignored source value. - assertEquals(0, output.ignoredSourceValues().size()); - } -} diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index df2c9466f3b7d..dc70c44a89128 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -297,7 +297,7 @@ private static MappingLookup createMappingLookup(List concreteF new MetadataFieldMapper[0], Collections.emptyMap() ); - return MappingLookup.fromMappers(mapping, mappers, Collections.emptyList(), null); + return MappingLookup.fromMappers(mapping, mappers, Collections.emptyList()); } public void testSearchRequestRuntimeFields() { @@ -389,13 +389,7 @@ public void testSyntheticSourceSearchLookup() throws IOException { new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100) ).build(MapperBuilderContext.root(true, false)); Mapping mapping = new Mapping(root, new MetadataFieldMapper[] { sourceMapper }, Map.of()); - IndexMetadata indexMetadata = IndexMetadata.builder("index") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - .build(); - IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); - MappingLookup lookup = MappingLookup.fromMapping(mapping, indexSettings); + MappingLookup lookup = MappingLookup.fromMapping(mapping); SearchExecutionContext sec = createSearchExecutionContext("index", "", lookup, Map.of()); assertTrue(sec.isSourceSynthetic()); diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java index f72152bd7ff8b..9aa847c837e96 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.engine.TranslogOperationAsserter; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import org.mockito.Mockito; @@ -95,6 +96,7 @@ private Tuple, TranslogWriter> createReadersAndWriter() thr BigArrays.NON_RECYCLING_INSTANCE, TranslogTests.RANDOMIZING_IO_BUFFERS, TranslogConfig.NOOP_OPERATION_LISTENER, + TranslogOperationAsserter.DEFAULT, true ); writer = Mockito.spy(writer); diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index 97f49df41d099..99f2e2a562eec 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -53,6 +53,7 @@ import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.Engine.Operation.Origin; +import org.elasticsearch.index.engine.TranslogOperationAsserter; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.ParsedDocument; @@ -224,7 +225,8 @@ protected Translog createTranslog(TranslogConfig config) throws IOException { new TranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - getPersistedSeqNoConsumer() + getPersistedSeqNoConsumer(), + TranslogOperationAsserter.DEFAULT ); } @@ -235,7 +237,8 @@ protected Translog openTranslog(TranslogConfig config, String translogUUID) thro new TranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - getPersistedSeqNoConsumer() + getPersistedSeqNoConsumer(), + TranslogOperationAsserter.DEFAULT ); } @@ -270,7 +273,8 @@ private Translog create(Path path) throws IOException { new TranslogDeletionPolicy(), () -> globalCheckpoint.get(), primaryTerm::get, - getPersistedSeqNoConsumer() + getPersistedSeqNoConsumer(), + TranslogOperationAsserter.DEFAULT ); } @@ -1444,7 +1448,8 @@ public int write(ByteBuffer src) throws IOException { new TranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - persistedSeqNos::add + persistedSeqNos::add, + TranslogOperationAsserter.DEFAULT ) { @Override ChannelFactory getChannelFactory() { @@ -1559,7 +1564,8 @@ public void force(boolean metaData) throws IOException { new TranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - persistedSeqNos::add + persistedSeqNos::add, + TranslogOperationAsserter.DEFAULT ) { @Override ChannelFactory getChannelFactory() { @@ -1746,7 +1752,8 @@ public void testBasicRecovery() throws IOException { translog.getDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ); assertEquals( "lastCommitted must be 1 less than current", @@ -1803,7 +1810,8 @@ public void testRecoveryUncommitted() throws IOException { deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ) ) { assertNotNull(translogGeneration); @@ -1830,7 +1838,8 @@ public void testRecoveryUncommitted() throws IOException { deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ) ) { assertNotNull(translogGeneration); @@ -1894,7 +1903,8 @@ public void testRecoveryUncommittedFileExists() throws IOException { deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ) ) { assertNotNull(translogGeneration); @@ -1922,7 +1932,8 @@ public void testRecoveryUncommittedFileExists() throws IOException { deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ) ) { assertNotNull(translogGeneration); @@ -1984,7 +1995,15 @@ public void testRecoveryUncommittedCorruptedCheckpoint() throws IOException { final TranslogDeletionPolicy deletionPolicy = translog.getDeletionPolicy(); final TranslogCorruptedException translogCorruptedException = expectThrows( TranslogCorruptedException.class, - () -> new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}) + () -> new Translog( + config, + translogUUID, + deletionPolicy, + () -> SequenceNumbers.NO_OPS_PERFORMED, + primaryTerm::get, + seqNo -> {}, + TranslogOperationAsserter.DEFAULT + ) ); assertThat( translogCorruptedException.getMessage(), @@ -2010,7 +2029,8 @@ public void testRecoveryUncommittedCorruptedCheckpoint() throws IOException { deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ) ) { assertNotNull(translogGeneration); @@ -2293,7 +2313,8 @@ public void testOpenForeignTranslog() throws IOException { new TranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ); fail("translog doesn't belong to this UUID"); } catch (TranslogCorruptedException ex) { @@ -2305,7 +2326,8 @@ public void testOpenForeignTranslog() throws IOException { deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ); try (Translog.Snapshot snapshot = this.translog.newSnapshot(randomLongBetween(0, firstUncommitted), Long.MAX_VALUE)) { for (int i = firstUncommitted; i < translogOperations; i++) { @@ -2503,7 +2525,8 @@ public void testFailFlush() throws IOException { deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ) ) { assertEquals( @@ -2649,7 +2672,8 @@ protected void afterAdd() throws IOException { new TranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ); Translog.Snapshot snapshot = tlog.newSnapshot() ) { @@ -2708,7 +2732,8 @@ public void testRecoveryFromAFutureGenerationCleansUp() throws IOException { deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ); assertThat(translog.getMinFileGeneration(), equalTo(1L)); // no trimming done yet, just recovered @@ -2771,7 +2796,8 @@ public void testRecoveryFromFailureOnTrimming() throws IOException { deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ) ) { // we don't know when things broke exactly @@ -2875,7 +2901,15 @@ private Translog getFailableTranslog( primaryTerm.get() ); } - return new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}) { + return new Translog( + config, + translogUUID, + deletionPolicy, + () -> SequenceNumbers.NO_OPS_PERFORMED, + primaryTerm::get, + seqNo -> {}, + TranslogOperationAsserter.DEFAULT + ) { @Override ChannelFactory getChannelFactory() { return channelFactory; @@ -3019,7 +3053,8 @@ public void testFailWhileCreateWriteWithRecoveredTLogs() throws IOException { new TranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ) { @Override protected TranslogWriter createWriter( @@ -3087,7 +3122,8 @@ public void testRecoverWithUnbackedNextGenInIllegalState() throws IOException { translog.getDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ) ); assertEquals(ex.getMessage(), "failed to create new translog file"); @@ -3114,7 +3150,8 @@ public void testRecoverWithUnbackedNextGenAndFutureFile() throws IOException { deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ) ) { assertFalse(tlog.syncNeeded()); @@ -3130,7 +3167,15 @@ public void testRecoverWithUnbackedNextGenAndFutureFile() throws IOException { TranslogException ex = expectThrows( TranslogException.class, - () -> new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}) + () -> new Translog( + config, + translogUUID, + deletionPolicy, + () -> SequenceNumbers.NO_OPS_PERFORMED, + primaryTerm::get, + seqNo -> {}, + TranslogOperationAsserter.DEFAULT + ) ); assertEquals(ex.getMessage(), "failed to create new translog file"); assertEquals(ex.getCause().getClass(), FileAlreadyExistsException.class); @@ -3256,7 +3301,8 @@ public void testWithRandomException() throws IOException { deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ); Translog.Snapshot snapshot = translog.newSnapshot(localCheckpointOfSafeCommit + 1, Long.MAX_VALUE) ) { @@ -3351,7 +3397,8 @@ public void testPendingDelete() throws IOException { deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ); translog.add(TranslogOperationsUtils.indexOp("2", 1, primaryTerm.get())); translog.rollGeneration(); @@ -3365,7 +3412,8 @@ public void testPendingDelete() throws IOException { deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ); } @@ -3713,7 +3761,15 @@ class MisbehavingTranslog extends Translog { LongSupplier globalCheckpointSupplier, LongSupplier primaryTermSupplier ) throws IOException { - super(config, translogUUID, deletionPolicy, globalCheckpointSupplier, primaryTermSupplier, seqNo -> {}); + super( + config, + translogUUID, + deletionPolicy, + globalCheckpointSupplier, + primaryTermSupplier, + seqNo -> {}, + TranslogOperationAsserter.DEFAULT + ); } void callCloseDirectly() throws IOException { @@ -3855,7 +3911,8 @@ public void copy(Path source, Path target, CopyOption... options) throws IOExcep brokenTranslog.getDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ) ) { recoveredTranslog.rollGeneration(); @@ -3889,7 +3946,8 @@ public void testSyncConcurrently() throws Exception { new TranslogDeletionPolicy(), globalCheckpointSupplier, primaryTerm::get, - persistedSeqNos::add + persistedSeqNos::add, + TranslogOperationAsserter.DEFAULT ) ) { Thread[] threads = new Thread[between(2, 8)]; @@ -3974,7 +4032,8 @@ public void force(boolean metaData) throws IOException { new TranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ) { @Override ChannelFactory getChannelFactory() { @@ -4040,7 +4099,8 @@ public void testDisabledFsync() throws IOException { new TranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, - getPersistedSeqNoConsumer() + getPersistedSeqNoConsumer(), + TranslogOperationAsserter.DEFAULT ) { @Override ChannelFactory getChannelFactory() { diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java index 8afedbd63f14e..773c660caa1c6 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java @@ -204,7 +204,7 @@ public void testCacheDifferentReaders() throws Exception { public void testCacheDifferentMapping() throws Exception { IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY); MappingLookup.CacheKey mappingKey1 = MappingLookup.EMPTY.cacheKey(); - MappingLookup.CacheKey mappingKey2 = MappingLookup.fromMappers(Mapping.EMPTY, emptyList(), emptyList(), null).cacheKey(); + MappingLookup.CacheKey mappingKey2 = MappingLookup.fromMappers(Mapping.EMPTY, emptyList(), emptyList()).cacheKey(); AtomicBoolean indexShard = new AtomicBoolean(true); ShardRequestCache requestCacheStats = new ShardRequestCache(); Directory dir = newDirectory(); @@ -364,13 +364,13 @@ public void testClearAllEntityIdentity() throws Exception { writer.updateDocument(new Term("id", "0"), newDoc(0, "bar")); DirectoryReader secondReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1)); - MappingLookup.CacheKey secondMappingKey = MappingLookup.fromMappers(Mapping.EMPTY, emptyList(), emptyList(), null).cacheKey(); + MappingLookup.CacheKey secondMappingKey = MappingLookup.fromMappers(Mapping.EMPTY, emptyList(), emptyList()).cacheKey(); TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard); Loader secondLoader = new Loader(secondReader, 0); writer.updateDocument(new Term("id", "0"), newDoc(0, "baz")); DirectoryReader thirdReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1)); - MappingLookup.CacheKey thirdMappingKey = MappingLookup.fromMappers(Mapping.EMPTY, emptyList(), emptyList(), null).cacheKey(); + MappingLookup.CacheKey thirdMappingKey = MappingLookup.fromMappers(Mapping.EMPTY, emptyList(), emptyList()).cacheKey(); AtomicBoolean differentIdentity = new AtomicBoolean(true); TestEntity thirdEntity = new TestEntity(requestCacheStats, differentIdentity); Loader thirdLoader = new Loader(thirdReader, 0); @@ -506,7 +506,7 @@ public void testKeyEqualsAndHashCode() throws IOException { AtomicBoolean trueBoolean = new AtomicBoolean(true); AtomicBoolean falseBoolean = new AtomicBoolean(false); MappingLookup.CacheKey mKey1 = MappingLookup.EMPTY.cacheKey(); - MappingLookup.CacheKey mKey2 = MappingLookup.fromMappers(Mapping.EMPTY, emptyList(), emptyList(), null).cacheKey(); + MappingLookup.CacheKey mKey2 = MappingLookup.fromMappers(Mapping.EMPTY, emptyList(), emptyList()).cacheKey(); Directory dir = newDirectory(); IndexWriterConfig config = newIndexWriterConfig(); IndexWriter writer = new IndexWriter(dir, config); diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java index 17975b7d18dd8..b56afadb14924 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java @@ -44,6 +44,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.SlowLogFieldProvider; +import org.elasticsearch.index.SlowLogFields; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.engine.EngineFactory; @@ -209,16 +210,18 @@ static void setFields(Map fields) { } @Override - public void init(IndexSettings indexSettings) {} - - @Override - public Map indexSlowLogFields() { - return fields; - } - - @Override - public Map searchSlowLogFields() { - return fields; + public SlowLogFields create(IndexSettings indexSettings) { + return new SlowLogFields() { + @Override + public Map indexFields() { + return fields; + } + + @Override + public Map searchFields() { + return fields; + } + }; } } @@ -231,16 +234,18 @@ static void setFields(Map fields) { } @Override - public void init(IndexSettings indexSettings) {} - - @Override - public Map indexSlowLogFields() { - return fields; - } - - @Override - public Map searchSlowLogFields() { - return fields; + public SlowLogFields create(IndexSettings indexSettings) { + return new SlowLogFields() { + @Override + public Map indexFields() { + return fields; + } + + @Override + public Map searchFields() { + return fields; + } + }; } } @@ -805,33 +810,34 @@ public void testLoadSlowLogFieldProvider() { TestAnotherSlowLogFieldProvider.setFields(Map.of("key2", "value2")); var indicesService = getIndicesService(); - SlowLogFieldProvider fieldProvider = indicesService.loadSlowLogFieldProvider(); + SlowLogFieldProvider fieldProvider = indicesService.slowLogFieldProvider; + SlowLogFields fields = fieldProvider.create(null); // The map of fields from the two providers are merged to a single map of fields - assertEquals(Map.of("key1", "value1", "key2", "value2"), fieldProvider.searchSlowLogFields()); - assertEquals(Map.of("key1", "value1", "key2", "value2"), fieldProvider.indexSlowLogFields()); + assertEquals(Map.of("key1", "value1", "key2", "value2"), fields.searchFields()); + assertEquals(Map.of("key1", "value1", "key2", "value2"), fields.indexFields()); TestSlowLogFieldProvider.setFields(Map.of("key1", "value1")); TestAnotherSlowLogFieldProvider.setFields(Map.of("key1", "value2")); // There is an overlap of field names, since this isn't deterministic and probably a // programming error (two providers provide the same field) throw an exception - assertThrows(IllegalStateException.class, fieldProvider::searchSlowLogFields); - assertThrows(IllegalStateException.class, fieldProvider::indexSlowLogFields); + assertThrows(IllegalStateException.class, fields::searchFields); + assertThrows(IllegalStateException.class, fields::indexFields); TestSlowLogFieldProvider.setFields(Map.of("key1", "value1")); TestAnotherSlowLogFieldProvider.setFields(Map.of()); // One provider has no fields - assertEquals(Map.of("key1", "value1"), fieldProvider.searchSlowLogFields()); - assertEquals(Map.of("key1", "value1"), fieldProvider.indexSlowLogFields()); + assertEquals(Map.of("key1", "value1"), fields.searchFields()); + assertEquals(Map.of("key1", "value1"), fields.indexFields()); TestSlowLogFieldProvider.setFields(Map.of()); TestAnotherSlowLogFieldProvider.setFields(Map.of()); // Both providers have no fields - assertEquals(Map.of(), fieldProvider.searchSlowLogFields()); - assertEquals(Map.of(), fieldProvider.indexSlowLogFields()); + assertEquals(Map.of(), fields.searchFields()); + assertEquals(Map.of(), fields.indexFields()); } public void testWithTempIndexServiceHandlesExistingIndex() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/lucene/util/automaton/MinimizationOperationsTests.java b/server/src/test/java/org/elasticsearch/lucene/util/automaton/MinimizationOperationsTests.java index bc9cafc792a8c..0f231ecd57c13 100644 --- a/server/src/test/java/org/elasticsearch/lucene/util/automaton/MinimizationOperationsTests.java +++ b/server/src/test/java/org/elasticsearch/lucene/util/automaton/MinimizationOperationsTests.java @@ -1,10 +1,18 @@ /* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * @notice + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Sourced from: https://github.com/apache/lucene/blob/main/lucene/core/src/test/org/apache/lucene/util/automaton/TestMinimize.java */ package org.elasticsearch.lucene.util.automaton; diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java index 926ac534164f3..d041121b8a96b 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java @@ -227,8 +227,7 @@ private SearchExecutionContext createSearchExecutionContext( MappingLookup mappingLookup = MappingLookup.fromMappers( mapping, Collections.singletonList(keywordFieldMapper), - Collections.emptyList(), - indexSettings + Collections.emptyList() ); return new SearchExecutionContext( 0, diff --git a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java index d78c697e19c81..3e4ed0ebac1ba 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java @@ -169,7 +169,7 @@ public void testBuild() throws IOException { invocation -> new TestTemplateService.MockTemplateScript.Factory(((Script) invocation.getArguments()[0]).getIdOrCode()) ); List mappers = Collections.singletonList(new MockFieldMapper(fieldType)); - MappingLookup lookup = MappingLookup.fromMappers(Mapping.EMPTY, mappers, emptyList(), idxSettings); + MappingLookup lookup = MappingLookup.fromMappers(Mapping.EMPTY, mappers, emptyList()); SearchExecutionContext mockContext = new SearchExecutionContext( 0, 0, diff --git a/server/src/test/resources/org/elasticsearch/action/admin/cluster/stats/telemetry_test.json b/server/src/test/resources/org/elasticsearch/action/admin/cluster/stats/telemetry_test.json index fe9c77cb2a183..a92bab739b37d 100644 --- a/server/src/test/resources/org/elasticsearch/action/admin/cluster/stats/telemetry_test.json +++ b/server/src/test/resources/org/elasticsearch/action/admin/cluster/stats/telemetry_test.json @@ -1,5 +1,4 @@ { - "_search" : { "total" : 10, "success" : 20, "skipped" : 5, @@ -63,5 +62,4 @@ } } } - } -} \ No newline at end of file +} diff --git a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java index 163712fb05a50..b43c516d2b246 100644 --- a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java +++ b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java @@ -14,12 +14,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.bytes.CompositeBytesReference; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.Tuple; import org.elasticsearch.rest.RestStatus; @@ -40,13 +38,11 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.function.BiFunction; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; +import static fixture.gcs.MockGcsBlobStore.failAndThrow; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.stream.Collectors.joining; import static org.elasticsearch.core.Strings.format; @@ -58,13 +54,14 @@ public class GoogleCloudStorageHttpHandler implements HttpHandler { private static final Logger logger = LogManager.getLogger(GoogleCloudStorageHttpHandler.class); + private static final String IF_GENERATION_MATCH = "ifGenerationMatch"; - private final ConcurrentMap blobs; + private final MockGcsBlobStore mockGcsBlobStore; private final String bucket; public GoogleCloudStorageHttpHandler(final String bucket) { this.bucket = Objects.requireNonNull(bucket); - this.blobs = new ConcurrentHashMap<>(); + this.mockGcsBlobStore = new MockGcsBlobStore(); } @Override @@ -83,15 +80,12 @@ public void handle(final HttpExchange exchange) throws IOException { exchange.sendResponseHeaders(RestStatus.OK.getStatus(), 0); } else if (Regex.simpleMatch("GET /storage/v1/b/" + bucket + "/o/*", request)) { final String key = exchange.getRequestURI().getPath().replace("/storage/v1/b/" + bucket + "/o/", ""); - final BytesReference blob = blobs.get(key); - if (blob == null) { - exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), -1); - } else { - final byte[] response = buildBlobInfoJson(key, blob.length()).getBytes(UTF_8); - exchange.getResponseHeaders().add("Content-Type", "application/json; charset=utf-8"); - exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); - exchange.getResponseBody().write(response); - } + final Long ifGenerationMatch = parseOptionalLongParameter(exchange, IF_GENERATION_MATCH); + final MockGcsBlobStore.BlobVersion blob = mockGcsBlobStore.getBlob(key, ifGenerationMatch); + final byte[] response = buildBlobInfoJson(blob).getBytes(UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json; charset=utf-8"); + exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); + exchange.getResponseBody().write(response); } else if (Regex.simpleMatch("GET /storage/v1/b/" + bucket + "/o*", request)) { // List Objects https://cloud.google.com/storage/docs/json_api/v1/objects/list final Map params = new HashMap<>(); @@ -102,14 +96,14 @@ public void handle(final HttpExchange exchange) throws IOException { final Set prefixes = new HashSet<>(); final List listOfBlobs = new ArrayList<>(); - for (final Map.Entry blob : blobs.entrySet()) { + for (final Map.Entry blob : mockGcsBlobStore.listBlobs().entrySet()) { final String blobName = blob.getKey(); if (prefix.isEmpty() || blobName.startsWith(prefix)) { int delimiterPos = (delimiter != null) ? blobName.substring(prefix.length()).indexOf(delimiter) : -1; if (delimiterPos > -1) { prefixes.add("\"" + blobName.substring(0, prefix.length() + delimiterPos + 1) + "\""); } else { - listOfBlobs.add(buildBlobInfoJson(blobName, blob.getValue().length())); + listOfBlobs.add(buildBlobInfoJson(blob.getValue())); } } } @@ -128,14 +122,16 @@ public void handle(final HttpExchange exchange) throws IOException { } else if (Regex.simpleMatch("GET /download/storage/v1/b/" + bucket + "/o/*", request)) { // Download Object https://cloud.google.com/storage/docs/request-body - BytesReference blob = blobs.get(exchange.getRequestURI().getPath().replace("/download/storage/v1/b/" + bucket + "/o/", "")); + final String path = exchange.getRequestURI().getPath().replace("/download/storage/v1/b/" + bucket + "/o/", ""); + final Long ifGenerationMatch = parseOptionalLongParameter(exchange, IF_GENERATION_MATCH); + final MockGcsBlobStore.BlobVersion blob = mockGcsBlobStore.getBlob(path, ifGenerationMatch); if (blob != null) { final String rangeHeader = exchange.getRequestHeaders().getFirst("Range"); final long offset; final long end; if (rangeHeader == null) { offset = 0L; - end = blob.length() - 1; + end = blob.contents().length() - 1; } else { final HttpHeaderParser.Range range = HttpHeaderParser.parseRangeHeader(rangeHeader); if (range == null) { @@ -145,13 +141,13 @@ public void handle(final HttpExchange exchange) throws IOException { end = range.end(); } - if (offset >= blob.length()) { + if (offset >= blob.contents().length()) { exchange.getResponseHeaders().add("Content-Type", "application/octet-stream"); exchange.sendResponseHeaders(RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus(), -1); return; } - BytesReference response = blob; + BytesReference response = blob.contents(); exchange.getResponseHeaders().add("Content-Type", "application/octet-stream"); final int bufferedLength = response.length(); if (offset > 0 || bufferedLength > end) { @@ -171,12 +167,12 @@ public void handle(final HttpExchange exchange) throws IOException { final String uri = "/storage/v1/b/" + bucket + "/o/"; final StringBuilder batch = new StringBuilder(); for (String line : Streams.readAllLines(requestBody.streamInput())) { - if (line.length() == 0 || line.startsWith("--") || line.toLowerCase(Locale.ROOT).startsWith("content")) { + if (line.isEmpty() || line.startsWith("--") || line.toLowerCase(Locale.ROOT).startsWith("content")) { batch.append(line).append("\r\n"); } else if (line.startsWith("DELETE")) { final String name = line.substring(line.indexOf(uri) + uri.length(), line.lastIndexOf(" HTTP")); if (Strings.hasText(name)) { - blobs.remove(URLDecoder.decode(name, UTF_8)); + mockGcsBlobStore.deleteBlob(URLDecoder.decode(name, UTF_8)); batch.append("HTTP/1.1 204 NO_CONTENT").append("\r\n"); batch.append("\r\n"); } @@ -191,11 +187,13 @@ public void handle(final HttpExchange exchange) throws IOException { // Multipart upload Optional> content = parseMultipartRequestBody(requestBody.streamInput()); if (content.isPresent()) { - blobs.put(content.get().v1(), content.get().v2()); - - byte[] response = String.format(Locale.ROOT, """ - {"bucket":"%s","name":"%s"} - """, bucket, content.get().v1()).getBytes(UTF_8); + final Long ifGenerationMatch = parseOptionalLongParameter(exchange, IF_GENERATION_MATCH); + final MockGcsBlobStore.BlobVersion newBlobVersion = mockGcsBlobStore.updateBlob( + content.get().v1(), + ifGenerationMatch, + content.get().v2() + ); + byte[] response = buildBlobInfoJson(newBlobVersion).getBytes(UTF_8); exchange.getResponseHeaders().add("Content-Type", "application/json"); exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); exchange.getResponseBody().write(response); @@ -214,7 +212,11 @@ public void handle(final HttpExchange exchange) throws IOException { final Map params = new HashMap<>(); RestUtils.decodeQueryString(exchange.getRequestURI(), params); final String blobName = params.get("name"); - blobs.put(blobName, BytesArray.EMPTY); + final Long ifGenerationMatch = parseOptionalLongParameter(exchange, IF_GENERATION_MATCH); + final MockGcsBlobStore.ResumableUpload resumableUpload = mockGcsBlobStore.createResumableUpload( + blobName, + ifGenerationMatch + ); byte[] response = requestBody.utf8ToString().getBytes(UTF_8); exchange.getResponseHeaders().add("Content-Type", "application/json"); @@ -227,10 +229,8 @@ public void handle(final HttpExchange exchange) throws IOException { + "/o?" + "uploadType=resumable" + "&upload_id=" - + UUIDs.randomBase64UUID() - + "&test_blob_name=" - + blobName - ); // not a Google Storage parameter, but it allows to pass the blob name + + resumableUpload.uploadId() + ); exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); exchange.getResponseBody().write(response); @@ -239,48 +239,62 @@ public void handle(final HttpExchange exchange) throws IOException { final Map params = new HashMap<>(); RestUtils.decodeQueryString(exchange.getRequestURI(), params); - final String blobName = params.get("test_blob_name"); - if (blobs.containsKey(blobName) == false) { - exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), -1); - return; + final String contentRangeValue = requireHeader(exchange, "Content-Range"); + final HttpHeaderParser.ContentRange contentRange = HttpHeaderParser.parseContentRangeHeader(contentRangeValue); + if (contentRange == null) { + throw failAndThrow("Invalid Content-Range: " + contentRangeValue); } - BytesReference blob = blobs.get(blobName); - final String range = exchange.getRequestHeaders().getFirst("Content-Range"); - final Integer limit = getContentRangeLimit(range); - - blob = CompositeBytesReference.of(blob, requestBody); - blobs.put(blobName, blob); - - if (limit == null) { - if ("bytes */*".equals(range) == false) { - final int start = getContentRangeStart(range); - final int end = getContentRangeEnd(range); - exchange.getResponseHeaders().add("Range", String.format(Locale.ROOT, "bytes=%d-%d", start, end)); - } - exchange.getResponseHeaders().add("Content-Length", "0"); - exchange.sendResponseHeaders(308 /* Resume Incomplete */, -1); - } else { - if (limit > blob.length()) { - throw new AssertionError("Requesting more bytes than available for blob"); - } - exchange.sendResponseHeaders(RestStatus.OK.getStatus(), -1); + + final MockGcsBlobStore.UpdateResponse updateResponse = mockGcsBlobStore.updateResumableUpload( + params.get("upload_id"), + contentRange, + requestBody + ); + + if (updateResponse.rangeHeader() != null) { + exchange.getResponseHeaders().add("Range", updateResponse.rangeHeader().headerString()); } + exchange.getResponseHeaders().add("Content-Length", "0"); + exchange.sendResponseHeaders(updateResponse.statusCode(), -1); } else { exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), -1); } + } catch (MockGcsBlobStore.GcsRestException e) { + sendError(exchange, e); } finally { exchange.close(); } } - private String buildBlobInfoJson(String blobName, int size) { - return String.format(Locale.ROOT, """ - {"kind":"storage#object","bucket":"%s","name":"%s","id":"%s","size":"%s"} - """, bucket, blobName, blobName, size); + private void sendError(HttpExchange exchange, MockGcsBlobStore.GcsRestException e) throws IOException { + final String responseBody = Strings.format(""" + { + "error": { + "errors": [], + "code": %d, + "message": "%s" + } + } + """, e.getStatus().getStatus(), e.getMessage()); + exchange.sendResponseHeaders(e.getStatus().getStatus(), responseBody.length()); + exchange.getResponseBody().write(responseBody.getBytes(UTF_8)); + } + + private String buildBlobInfoJson(MockGcsBlobStore.BlobVersion blobReference) { + return String.format( + Locale.ROOT, + """ + {"kind":"storage#object","bucket":"%s","name":"%s","id":"%s","size":"%s","generation":"%d"}""", + bucket, + blobReference.path(), + blobReference.path(), + blobReference.contents().length(), + blobReference.generation() + ); } public Map blobs() { - return blobs; + return Maps.transformValues(mockGcsBlobStore.listBlobs(), MockGcsBlobStore.BlobVersion::contents); } private static String httpServerUrl(final HttpExchange exchange) { @@ -362,34 +376,24 @@ private static boolean isEndOfPart(BytesReference fullRequestBody, int endPos) { return true; } - private static final Pattern PATTERN_CONTENT_RANGE = Pattern.compile("bytes ([^/]*)/([0-9\\*]*)"); - private static final Pattern PATTERN_CONTENT_RANGE_BYTES = Pattern.compile("([0-9]*)-([0-9]*)"); - - private static Integer parse(final Pattern pattern, final String contentRange, final BiFunction fn) { - final Matcher matcher = pattern.matcher(contentRange); - if (matcher.matches() == false || matcher.groupCount() != 2) { - throw new IllegalArgumentException("Unable to parse content range header"); + private static String requireHeader(HttpExchange exchange, String headerName) { + final String headerValue = exchange.getRequestHeaders().getFirst(headerName); + if (headerValue != null) { + return headerValue; } - return fn.apply(matcher.group(1), matcher.group(2)); + throw failAndThrow("Missing required header: " + headerName); } - public static Integer getContentRangeLimit(final String contentRange) { - return parse(PATTERN_CONTENT_RANGE, contentRange, (bytes, limit) -> "*".equals(limit) ? null : Integer.parseInt(limit)); - } - - public static int getContentRangeStart(final String contentRange) { - return parse( - PATTERN_CONTENT_RANGE, - contentRange, - (bytes, limit) -> parse(PATTERN_CONTENT_RANGE_BYTES, bytes, (start, end) -> Integer.parseInt(start)) - ); - } - - public static int getContentRangeEnd(final String contentRange) { - return parse( - PATTERN_CONTENT_RANGE, - contentRange, - (bytes, limit) -> parse(PATTERN_CONTENT_RANGE_BYTES, bytes, (start, end) -> Integer.parseInt(end)) - ); + private static Long parseOptionalLongParameter(HttpExchange exchange, String parameterName) { + final Map params = new HashMap<>(); + RestUtils.decodeQueryString(exchange.getRequestURI(), params); + if (params.containsKey(parameterName)) { + try { + return Long.parseLong(params.get(parameterName)); + } catch (NumberFormatException e) { + throw failAndThrow("Invalid long parameter: " + parameterName); + } + } + return null; } } diff --git a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/MockGcsBlobStore.java b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/MockGcsBlobStore.java new file mode 100644 index 0000000000000..a5b055566f78d --- /dev/null +++ b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/MockGcsBlobStore.java @@ -0,0 +1,202 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package fixture.gcs; + +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.bytes.CompositeBytesReference; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.fixture.HttpHeaderParser; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; + +public class MockGcsBlobStore { + + private static final int RESUME_INCOMPLETE = 308; + private final ConcurrentMap blobs = new ConcurrentHashMap<>(); + private final ConcurrentMap resumableUploads = new ConcurrentHashMap<>(); + + record BlobVersion(String path, long generation, BytesReference contents) {} + + record ResumableUpload(String uploadId, String path, Long ifGenerationMatch, BytesReference contents, boolean completed) { + + public ResumableUpload update(BytesReference contents, boolean completed) { + return new ResumableUpload(uploadId, path, ifGenerationMatch, contents, completed); + } + } + + BlobVersion getBlob(String path, Long ifGenerationMatch) { + final BlobVersion blob = blobs.get(path); + if (blob == null) { + throw new BlobNotFoundException(path); + } else { + if (ifGenerationMatch != null) { + if (blob.generation != ifGenerationMatch) { + throw new GcsRestException( + RestStatus.PRECONDITION_FAILED, + "Generation mismatch, expected " + ifGenerationMatch + " but got " + blob.generation + ); + } + } + return blob; + } + } + + BlobVersion updateBlob(String path, Long ifGenerationMatch, BytesReference contents) { + return blobs.compute(path, (name, existing) -> { + if (existing != null) { + if (ifGenerationMatch != null) { + if (ifGenerationMatch == 0) { + throw new GcsRestException( + RestStatus.PRECONDITION_FAILED, + "Blob already exists at generation " + existing.generation + ); + } else if (ifGenerationMatch != existing.generation) { + throw new GcsRestException( + RestStatus.PRECONDITION_FAILED, + "Generation mismatch, expected " + ifGenerationMatch + ", got" + existing.generation + ); + } + } + return new BlobVersion(path, existing.generation + 1, contents); + } else { + if (ifGenerationMatch != null && ifGenerationMatch != 0) { + throw new GcsRestException( + RestStatus.PRECONDITION_FAILED, + "Blob does not exist, expected generation " + ifGenerationMatch + ); + } + return new BlobVersion(path, 1, contents); + } + }); + } + + ResumableUpload createResumableUpload(String path, Long ifGenerationMatch) { + final String uploadId = UUIDs.randomBase64UUID(); + final ResumableUpload value = new ResumableUpload(uploadId, path, ifGenerationMatch, BytesArray.EMPTY, false); + resumableUploads.put(uploadId, value); + return value; + } + + /** + * Update or query a resumable upload + * + * @see GCS Resumable Uploads + * @param uploadId The upload ID + * @param contentRange The range being submitted + * @param requestBody The data for that range + * @return The response to the request + */ + UpdateResponse updateResumableUpload(String uploadId, HttpHeaderParser.ContentRange contentRange, BytesReference requestBody) { + final AtomicReference updateResponse = new AtomicReference<>(); + resumableUploads.compute(uploadId, (uid, existing) -> { + if (existing == null) { + throw failAndThrow("Attempted to update a non-existent resumable: " + uid); + } + + if (contentRange.hasRange() == false) { + // Content-Range: */... is a status check https://cloud.google.com/storage/docs/performing-resumable-uploads#status-check + if (existing.completed) { + updateResponse.set(new UpdateResponse(RestStatus.OK.getStatus(), calculateRangeHeader(blobs.get(existing.path)))); + } else { + final HttpHeaderParser.Range range = calculateRangeHeader(existing); + updateResponse.set(new UpdateResponse(RESUME_INCOMPLETE, range)); + } + return existing; + } else { + if (contentRange.start() > contentRange.end()) { + throw failAndThrow("Invalid content range " + contentRange); + } + if (contentRange.start() > existing.contents.length()) { + throw failAndThrow( + "Attempted to append after the end of the current content: size=" + + existing.contents.length() + + ", start=" + + contentRange.start() + ); + } + if (contentRange.end() < existing.contents.length()) { + throw failAndThrow("Attempted to upload no new data"); + } + final int offset = Math.toIntExact(existing.contents.length() - contentRange.start()); + final BytesReference updatedContent = CompositeBytesReference.of( + existing.contents, + requestBody.slice(offset, requestBody.length()) + ); + // We just received the last chunk, update the blob and remove the resumable upload from the map + if (contentRange.hasSize() && updatedContent.length() == contentRange.size()) { + updateBlob(existing.path(), existing.ifGenerationMatch, updatedContent); + updateResponse.set(new UpdateResponse(RestStatus.OK.getStatus(), null)); + return existing.update(BytesArray.EMPTY, true); + } + final ResumableUpload updated = existing.update(updatedContent, false); + updateResponse.set(new UpdateResponse(RESUME_INCOMPLETE, calculateRangeHeader(updated))); + return updated; + } + }); + assert updateResponse.get() != null : "Should always produce an update response"; + return updateResponse.get(); + } + + private static HttpHeaderParser.Range calculateRangeHeader(ResumableUpload resumableUpload) { + return resumableUpload.contents.length() > 0 ? new HttpHeaderParser.Range(0, resumableUpload.contents.length() - 1) : null; + } + + private static HttpHeaderParser.Range calculateRangeHeader(BlobVersion blob) { + return blob.contents.length() > 0 ? new HttpHeaderParser.Range(0, blob.contents.length() - 1) : null; + } + + record UpdateResponse(int statusCode, HttpHeaderParser.Range rangeHeader) {} + + void deleteBlob(String path) { + blobs.remove(path); + } + + Map listBlobs() { + return Map.copyOf(blobs); + } + + static class BlobNotFoundException extends GcsRestException { + + BlobNotFoundException(String path) { + super(RestStatus.NOT_FOUND, "Blob not found: " + path); + } + } + + static class GcsRestException extends RuntimeException { + + private final RestStatus status; + + GcsRestException(RestStatus status, String errorMessage) { + super(errorMessage); + this.status = status; + } + + public RestStatus getStatus() { + return status; + } + } + + /** + * Fail the test with an assertion error and throw an exception in-line + * + * @param message The message to use on the {@link Throwable}s + * @return nothing, but claim to return an exception to help with static analysis + */ + public static RuntimeException failAndThrow(String message) { + ExceptionsHelper.maybeDieOnAnotherThread(new AssertionError(message)); + throw new IllegalStateException(message); + } +} diff --git a/test/fixtures/gcs-fixture/src/test/java/fixture/gcs/GoogleCloudStorageHttpHandlerTests.java b/test/fixtures/gcs-fixture/src/test/java/fixture/gcs/GoogleCloudStorageHttpHandlerTests.java index 0caaa983f76df..f83da9e1679a9 100644 --- a/test/fixtures/gcs-fixture/src/test/java/fixture/gcs/GoogleCloudStorageHttpHandlerTests.java +++ b/test/fixtures/gcs-fixture/src/test/java/fixture/gcs/GoogleCloudStorageHttpHandlerTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.fixture.HttpHeaderParser; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -29,17 +30,21 @@ import java.io.OutputStream; import java.net.InetSocketAddress; import java.net.URI; +import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.zip.GZIPOutputStream; public class GoogleCloudStorageHttpHandlerTests extends ESTestCase { private static final String HOST = "http://127.0.0.1:12345"; private static final int RESUME_INCOMPLETE = 308; + private static final Pattern GENERATION_PATTERN = Pattern.compile("\"generation\"\\s*:\\s*\"(\\d+)\""); public void testRejectsBadUri() { assertEquals( @@ -61,48 +66,38 @@ public void testCheckEndpoint() { public void testSimpleObjectOperations() { final var bucket = randomAlphaOfLength(10); final var handler = new GoogleCloudStorageHttpHandler(bucket); + final var blobName = "path/" + randomAlphaOfLength(10); - assertEquals(RestStatus.NOT_FOUND, handleRequest(handler, "GET", "/download/storage/v1/b/" + bucket + "/o/blob").restStatus()); + assertEquals(RestStatus.NOT_FOUND, getBlobContents(handler, bucket, blobName, null, null).restStatus()); assertEquals( new TestHttpResponse(RestStatus.OK, "{\"kind\":\"storage#objects\",\"items\":[],\"prefixes\":[]}"), - handleRequest(handler, "GET", "/storage/v1/b/" + bucket + "/o") + listBlobs(handler, bucket, null) ); - // Multipart upload final var body = randomAlphaOfLength(50); assertEquals( RestStatus.OK, - handleRequest( - handler, - "POST", - "/upload/storage/v1/b/" + bucket + "/?uploadType=multipart", - createGzipCompressedMultipartUploadBody(bucket, "path/blob", body) - ).restStatus() - ); - assertEquals( - new TestHttpResponse(RestStatus.OK, body), - handleRequest(handler, "GET", "/download/storage/v1/b/" + bucket + "/o/path/blob") + executeUpload(handler, bucket, blobName, new BytesArray(body.getBytes(StandardCharsets.UTF_8)), null).restStatus() ); + assertEquals(new TestHttpResponse(RestStatus.OK, body), getBlobContents(handler, bucket, blobName, null, null)); + assertEquals(new TestHttpResponse(RestStatus.OK, Strings.format(""" - {"kind":"storage#objects","items":[{"kind":"storage#object","bucket":"%s","name":"path/blob","id":"path/blob","size":"50"} - ],"prefixes":[]}""", bucket)), handleRequest(handler, "GET", "/storage/v1/b/" + bucket + "/o")); + {"kind":"storage#objects","items":[{"kind":"storage#object","bucket":"%s","name":"%s","id":"%s","size":"50",\ + "generation":"1"}],"prefixes":[]}""", bucket, blobName, blobName)), listBlobs(handler, bucket, null)); assertEquals(new TestHttpResponse(RestStatus.OK, Strings.format(""" - {"kind":"storage#objects","items":[{"kind":"storage#object","bucket":"%s","name":"path/blob","id":"path/blob","size":"50"} - ],"prefixes":[]}""", bucket)), handleRequest(handler, "GET", "/storage/v1/b/" + bucket + "/o?prefix=path/")); + {"kind":"storage#objects","items":[{"kind":"storage#object","bucket":"%s","name":"%s","id":"%s","size":"50",\ + "generation":"1"}],"prefixes":[]}""", bucket, blobName, blobName)), listBlobs(handler, bucket, "path/")); - assertEquals( - new TestHttpResponse(RestStatus.OK, """ - {"kind":"storage#objects","items":[],"prefixes":[]}"""), - handleRequest(handler, "GET", "/storage/v1/b/" + bucket + "/o?prefix=path/other") - ); + assertEquals(new TestHttpResponse(RestStatus.OK, """ + {"kind":"storage#objects","items":[],"prefixes":[]}"""), listBlobs(handler, bucket, "some/other/path")); assertEquals( new TestHttpResponse(RestStatus.OK, """ --__END_OF_PART__d8b50acb-87dc-4630-a3d3-17d187132ebc__ - Content-Length: 162 + Content-Length: 168 Content-Type: application/http content-id: 1 content-transfer-encoding: binary @@ -118,7 +113,7 @@ public void testSimpleObjectOperations() { handler, "POST", "/batch/storage/v1", - createBatchDeleteRequest(bucket, "path/blob"), + createBatchDeleteRequest(bucket, blobName), Headers.of("Content-Type", "mixed/multipart") ) ); @@ -128,53 +123,41 @@ public void testSimpleObjectOperations() { handler, "POST", "/batch/storage/v1", - createBatchDeleteRequest(bucket, "path/blob"), + createBatchDeleteRequest(bucket, blobName), Headers.of("Content-Type", "mixed/multipart") ).restStatus() ); - assertEquals( - new TestHttpResponse(RestStatus.OK, """ - {"kind":"storage#objects","items":[],"prefixes":[]}"""), - handleRequest(handler, "GET", "/storage/v1/b/" + bucket + "/o?prefix=path/") - ); + assertEquals(new TestHttpResponse(RestStatus.OK, """ + {"kind":"storage#objects","items":[],"prefixes":[]}"""), listBlobs(handler, bucket, "path/")); } public void testGetWithBytesRange() { final var bucket = randomIdentifier(); final var handler = new GoogleCloudStorageHttpHandler(bucket); final var blobName = "blob_name_" + randomIdentifier(); - final var blobPath = "/download/storage/v1/b/" + bucket + "/o/" + blobName; final var blobBytes = randomBytesReference(256); - assertEquals( - RestStatus.OK, - handleRequest( - handler, - "POST", - "/upload/storage/v1/b/" + bucket + "/?uploadType=multipart", - createGzipCompressedMultipartUploadBody(bucket, blobName, blobBytes) - ).restStatus() - ); + assertEquals(RestStatus.OK, executeUpload(handler, bucket, blobName, blobBytes, 0L).restStatus()); assertEquals( "No Range", new TestHttpResponse(RestStatus.OK, blobBytes, TestHttpExchange.EMPTY_HEADERS), - handleRequest(handler, "GET", blobPath) + getBlobContents(handler, bucket, blobName, null, null) ); var end = blobBytes.length() - 1; assertEquals( "Exact Range: bytes=0-" + end, new TestHttpResponse(RestStatus.OK, blobBytes, TestHttpExchange.EMPTY_HEADERS), - handleRequest(handler, "GET", blobPath, BytesArray.EMPTY, rangeHeader(0, end)) + getBlobContents(handler, bucket, blobName, null, new HttpHeaderParser.Range(0, end)) ); end = randomIntBetween(blobBytes.length() - 1, Integer.MAX_VALUE); assertEquals( "Larger Range: bytes=0-" + end, new TestHttpResponse(RestStatus.OK, blobBytes, TestHttpExchange.EMPTY_HEADERS), - handleRequest(handler, "GET", blobPath, BytesArray.EMPTY, rangeHeader(0, end)) + getBlobContents(handler, bucket, blobName, null, new HttpHeaderParser.Range(0, end)) ); var start = randomIntBetween(blobBytes.length(), Integer.MAX_VALUE - 1); @@ -182,7 +165,7 @@ public void testGetWithBytesRange() { assertEquals( "Invalid Range: bytes=" + start + '-' + end, new TestHttpResponse(RestStatus.REQUESTED_RANGE_NOT_SATISFIED, BytesArray.EMPTY, TestHttpExchange.EMPTY_HEADERS), - handleRequest(handler, "GET", blobPath, BytesArray.EMPTY, rangeHeader(start, end)) + getBlobContents(handler, bucket, blobName, null, new HttpHeaderParser.Range(start, end)) ); start = randomIntBetween(0, blobBytes.length() - 1); @@ -191,7 +174,7 @@ public void testGetWithBytesRange() { assertEquals( "Range: bytes=" + start + '-' + end, new TestHttpResponse(RestStatus.OK, blobBytes.slice(start, length), TestHttpExchange.EMPTY_HEADERS), - handleRequest(handler, "GET", blobPath, BytesArray.EMPTY, rangeHeader(start, end)) + getBlobContents(handler, bucket, blobName, null, new HttpHeaderParser.Range(start, end)) ); } @@ -209,23 +192,39 @@ public void testResumableUpload() { final var sessionURI = locationHeader.substring(locationHeader.indexOf(HOST) + HOST.length()); assertEquals(RestStatus.OK, createUploadResponse.restStatus()); + // status check + assertEquals( + new TestHttpResponse(RESUME_INCOMPLETE, TestHttpExchange.EMPTY_HEADERS), + handleRequest(handler, "PUT", sessionURI, BytesArray.EMPTY, contentRangeHeader(null, null, null)) + ); + final var part1 = randomAlphaOfLength(50); final var uploadPart1Response = handleRequest(handler, "PUT", sessionURI, part1, contentRangeHeader(0, 50, null)); - assertEquals(new TestHttpResponse(RESUME_INCOMPLETE, rangeHeader(0, 50)), uploadPart1Response); + assertEquals(new TestHttpResponse(RESUME_INCOMPLETE, rangeHeader(0, 49)), uploadPart1Response); + // status check assertEquals( - new TestHttpResponse(RESUME_INCOMPLETE, TestHttpExchange.EMPTY_HEADERS), + new TestHttpResponse(RESUME_INCOMPLETE, rangeHeader(0, 49)), handleRequest(handler, "PUT", sessionURI, BytesArray.EMPTY, contentRangeHeader(null, null, null)) ); final var part2 = randomAlphaOfLength(50); - final var uploadPart2Response = handleRequest(handler, "PUT", sessionURI, part2, contentRangeHeader(51, 100, null)); - assertEquals(new TestHttpResponse(RESUME_INCOMPLETE, rangeHeader(51, 100)), uploadPart2Response); + final var uploadPart2Response = handleRequest(handler, "PUT", sessionURI, part2, contentRangeHeader(50, 99, null)); + assertEquals(new TestHttpResponse(RESUME_INCOMPLETE, rangeHeader(0, 99)), uploadPart2Response); + + // incomplete upload should not be visible yet + assertEquals(RestStatus.NOT_FOUND, getBlobContents(handler, bucket, blobName, null, null).restStatus()); final var part3 = randomAlphaOfLength(30); - final var uploadPart3Response = handleRequest(handler, "PUT", sessionURI, part3, contentRangeHeader(101, 130, 130)); + final var uploadPart3Response = handleRequest(handler, "PUT", sessionURI, part3, contentRangeHeader(100, 129, 130)); assertEquals(new TestHttpResponse(RestStatus.OK, TestHttpExchange.EMPTY_HEADERS), uploadPart3Response); + // status check + assertEquals( + new TestHttpResponse(RestStatus.OK, rangeHeader(0, 129)), + handleRequest(handler, "PUT", sessionURI, BytesArray.EMPTY, contentRangeHeader(null, null, null)) + ); + // complete upload should be visible now // can download contents @@ -235,14 +234,303 @@ public void testResumableUpload() { ); // can see in listing - assertEquals(new TestHttpResponse(RestStatus.OK, Strings.format(""" - {"kind":"storage#objects","items":[{"kind":"storage#object","bucket":"%s","name":"%s","id":"%s","size":"130"} - ],"prefixes":[]}""", bucket, blobName, blobName)), handleRequest(handler, "GET", "/storage/v1/b/" + bucket + "/o")); + assertEquals( + new TestHttpResponse(RestStatus.OK, Strings.format(""" + {"kind":"storage#objects","items":[{"kind":"storage#object","bucket":"%s","name":"%s","id":"%s","size":"130",\ + "generation":"1"}],"prefixes":[]}""", bucket, blobName, blobName)), + handleRequest(handler, "GET", "/storage/v1/b/" + bucket + "/o") + ); // can get metadata - assertEquals(new TestHttpResponse(RestStatus.OK, Strings.format(""" - {"kind":"storage#object","bucket":"%s","name":"%s","id":"%s","size":"130"} - """, bucket, blobName, blobName)), handleRequest(handler, "GET", "/storage/v1/b/" + bucket + "/o/" + blobName)); + assertEquals( + new TestHttpResponse( + RestStatus.OK, + Strings.format( + """ + {"kind":"storage#object","bucket":"%s","name":"%s","id":"%s","size":"130","generation":"1"}""", + bucket, + blobName, + blobName + ) + ), + handleRequest(handler, "GET", "/storage/v1/b/" + bucket + "/o/" + blobName) + ); + } + + public void testIfGenerationMatch_MultipartUpload() { + final var bucket = randomIdentifier(); + final var handler = new GoogleCloudStorageHttpHandler(bucket); + final var blobName = "blob_name_" + randomIdentifier(); + + assertEquals( + RestStatus.OK, + executeUpload(handler, bucket, blobName, randomBytesReference(randomIntBetween(100, 5_000)), null).restStatus() + ); + + // update, matched generation + assertEquals( + RestStatus.OK, + executeMultipartUpload( + handler, + bucket, + blobName, + randomBytesReference(randomIntBetween(100, 5_000)), + getCurrentGeneration(handler, bucket, blobName) + ).restStatus() + ); + + // update, mismatched generation + assertEquals( + RestStatus.PRECONDITION_FAILED, + executeMultipartUpload( + handler, + bucket, + blobName, + randomBytesReference(randomIntBetween(100, 5_000)), + randomValueOtherThan(getCurrentGeneration(handler, bucket, blobName), ESTestCase::randomNonNegativeLong) + ).restStatus() + ); + + // update, no generation + assertEquals( + RestStatus.OK, + executeMultipartUpload(handler, bucket, blobName, randomBytesReference(randomIntBetween(100, 5_000)), null).restStatus() + ); + + // update, zero generation + assertEquals( + RestStatus.PRECONDITION_FAILED, + executeMultipartUpload(handler, bucket, blobName, randomBytesReference(randomIntBetween(100, 5_000)), 0L).restStatus() + ); + + // new file, zero generation + assertEquals( + RestStatus.OK, + executeMultipartUpload(handler, bucket, blobName + "/new/1", randomBytesReference(randomIntBetween(100, 5_000)), 0L) + .restStatus() + ); + + // new file, non-zero generation + assertEquals( + RestStatus.PRECONDITION_FAILED, + executeMultipartUpload( + handler, + bucket, + blobName + "/new/2", + randomBytesReference(randomIntBetween(100, 5_000)), + randomLongBetween(1, Long.MAX_VALUE) + ).restStatus() + ); + } + + public void testIfGenerationMatch_ResumableUpload() { + final var bucket = randomIdentifier(); + final var handler = new GoogleCloudStorageHttpHandler(bucket); + final var blobName = "blob_name_" + randomIdentifier(); + + assertEquals( + RestStatus.OK, + executeUpload(handler, bucket, blobName, randomBytesReference(randomIntBetween(100, 5_000)), null).restStatus() + ); + + // update, matched generation + assertEquals( + RestStatus.OK, + executeResumableUpload( + handler, + bucket, + blobName, + randomBytesReference(randomIntBetween(100, 5_000)), + getCurrentGeneration(handler, bucket, blobName) + ).restStatus() + ); + + // update, mismatched generation + assertEquals( + RestStatus.PRECONDITION_FAILED, + executeResumableUpload( + handler, + bucket, + blobName, + randomBytesReference(randomIntBetween(100, 5_000)), + randomValueOtherThan(getCurrentGeneration(handler, bucket, blobName), ESTestCase::randomNonNegativeLong) + ).restStatus() + ); + + // update, no generation + assertEquals( + RestStatus.OK, + executeResumableUpload(handler, bucket, blobName, randomBytesReference(randomIntBetween(100, 5_000)), null).restStatus() + ); + + // update, zero generation + assertEquals( + RestStatus.PRECONDITION_FAILED, + executeResumableUpload(handler, bucket, blobName, randomBytesReference(randomIntBetween(100, 5_000)), 0L).restStatus() + ); + + // new file, zero generation + assertEquals( + RestStatus.OK, + executeResumableUpload(handler, bucket, blobName + "/new/1", randomBytesReference(randomIntBetween(100, 5_000)), 0L) + .restStatus() + ); + + // new file, non-zero generation + assertEquals( + RestStatus.PRECONDITION_FAILED, + executeResumableUpload( + handler, + bucket, + blobName + "/new/2", + randomBytesReference(randomIntBetween(100, 5_000)), + randomLongBetween(1, Long.MAX_VALUE) + ).restStatus() + ); + } + + public void testIfGenerationMatch_GetObject() { + final var bucket = randomIdentifier(); + final var handler = new GoogleCloudStorageHttpHandler(bucket); + final var blobName = "blob_name_" + randomIdentifier(); + + assertEquals( + RestStatus.OK, + executeUpload(handler, bucket, blobName, randomBytesReference(randomIntBetween(100, 5_000)), null).restStatus() + ); + + final long currentGeneration = getCurrentGeneration(handler, bucket, blobName); + + // Get contents, matching generation + assertEquals(RestStatus.OK, getBlobContents(handler, bucket, blobName, currentGeneration, null).restStatus()); + + // Get contents, mismatched generation + assertEquals( + RestStatus.PRECONDITION_FAILED, + getBlobContents(handler, bucket, blobName, randomValueOtherThan(currentGeneration, ESTestCase::randomNonNegativeLong), null) + .restStatus() + ); + + // Get metadata, matching generation + assertEquals(RestStatus.OK, getBlobMetadata(handler, bucket, blobName, currentGeneration).restStatus()); + + // Get metadata, mismatched generation + assertEquals( + RestStatus.PRECONDITION_FAILED, + getBlobMetadata(handler, bucket, blobName, randomValueOtherThan(currentGeneration, ESTestCase::randomNonNegativeLong)) + .restStatus() + ); + } + + private static TestHttpResponse executeUpload( + GoogleCloudStorageHttpHandler handler, + String bucket, + String blobName, + BytesReference bytes, + Long ifGenerationMatch + ) { + assert bytes.length() > 20; + if (randomBoolean()) { + return executeResumableUpload(handler, bucket, blobName, bytes, ifGenerationMatch); + } else { + return executeMultipartUpload(handler, bucket, blobName, bytes, ifGenerationMatch); + } + } + + private static TestHttpResponse executeResumableUpload( + GoogleCloudStorageHttpHandler handler, + String bucket, + String blobName, + BytesReference bytes, + Long ifGenerationMatch + ) { + final var createUploadResponse = handleRequest( + handler, + "POST", + "/upload/storage/v1/b/" + + bucket + + "/?uploadType=resumable&name=" + + blobName + + (ifGenerationMatch != null ? "&ifGenerationMatch=" + ifGenerationMatch : "") + ); + final var locationHeader = createUploadResponse.headers.getFirst("Location"); + final var sessionURI = locationHeader.substring(locationHeader.indexOf(HOST) + HOST.length()); + assertEquals(RestStatus.OK, createUploadResponse.restStatus()); + + final int partBoundary = randomIntBetween(10, bytes.length() - 1); + final var part1 = bytes.slice(0, partBoundary); + final var uploadPart1Response = handleRequest(handler, "PUT", sessionURI, part1, contentRangeHeader(0, partBoundary - 1, null)); + assertEquals(RESUME_INCOMPLETE, uploadPart1Response.status()); + + final var part2 = bytes.slice(partBoundary, bytes.length() - partBoundary); + return handleRequest(handler, "PUT", sessionURI, part2, contentRangeHeader(partBoundary, bytes.length() - 1, bytes.length())); + } + + private static TestHttpResponse executeMultipartUpload( + GoogleCloudStorageHttpHandler handler, + String bucket, + String blobName, + BytesReference bytes, + Long ifGenerationMatch + ) { + return handleRequest( + handler, + "POST", + "/upload/storage/v1/b/" + + bucket + + "/?uploadType=multipart" + + (ifGenerationMatch != null ? "&ifGenerationMatch=" + ifGenerationMatch : ""), + createGzipCompressedMultipartUploadBody(bucket, blobName, bytes) + ); + } + + private static TestHttpResponse getBlobContents( + GoogleCloudStorageHttpHandler handler, + String bucket, + String blobName, + @Nullable Long ifGenerationMatch, + @Nullable HttpHeaderParser.Range range + ) { + return handleRequest( + handler, + "GET", + "/download/storage/v1/b/" + + bucket + + "/o/" + + blobName + + (ifGenerationMatch != null ? "?ifGenerationMatch=" + ifGenerationMatch : ""), + BytesArray.EMPTY, + range != null ? rangeHeader(range.start(), range.end()) : TestHttpExchange.EMPTY_HEADERS + ); + } + + private static TestHttpResponse getBlobMetadata( + GoogleCloudStorageHttpHandler handler, + String bucket, + String blobName, + @Nullable Long ifGenerationMatch + ) { + return handleRequest( + handler, + "GET", + "/storage/v1/b/" + bucket + "/o/" + blobName + (ifGenerationMatch != null ? "?ifGenerationMatch=" + ifGenerationMatch : "") + ); + } + + private static long getCurrentGeneration(GoogleCloudStorageHttpHandler handler, String bucket, String blobName) { + TestHttpResponse blobMetadata = getBlobMetadata(handler, bucket, blobName, null); + assertEquals(RestStatus.OK, blobMetadata.restStatus()); + Matcher matcher = GENERATION_PATTERN.matcher(blobMetadata.body.utf8ToString()); + assertTrue(matcher.find()); + return Long.parseLong(matcher.group(1)); + } + + private static TestHttpResponse listBlobs(GoogleCloudStorageHttpHandler handler, String bucket, String prefix) { + return handleRequest( + handler, + "GET", + "/storage/v1/b/" + bucket + "/o" + (prefix != null ? "?prefix=" + URLEncoder.encode(prefix, StandardCharsets.UTF_8) : "") + ); } private record TestHttpResponse(int status, BytesReference body, Headers headers) { @@ -333,10 +621,6 @@ private static Headers rangeHeader(long start, long end) { return Headers.of("Range", Strings.format("bytes=%d-%d", start, end)); } - private static BytesReference createGzipCompressedMultipartUploadBody(String bucketName, String path, String content) { - return createGzipCompressedMultipartUploadBody(bucketName, path, new BytesArray(content.getBytes(StandardCharsets.UTF_8))); - } - private static BytesReference createGzipCompressedMultipartUploadBody(String bucketName, String path, BytesReference content) { final String metadataString = Strings.format("{\"bucket\":\"%s\", \"name\":\"%s\"}", bucketName, path); final BytesReference header = new BytesArray(Strings.format(""" diff --git a/test/fixtures/krb5kdc-fixture/Dockerfile b/test/fixtures/krb5kdc-fixture/Dockerfile index e862c7a71f2ba..47fc05d5aaf5b 100644 --- a/test/fixtures/krb5kdc-fixture/Dockerfile +++ b/test/fixtures/krb5kdc-fixture/Dockerfile @@ -1,9 +1,12 @@ -FROM ubuntu:14.04 -ADD . /fixture +FROM alpine:3.21.0 + +ADD src/main/resources /fixture +RUN apk update && apk add -y --no-cache python3 krb5 krb5-server + RUN echo kerberos.build.elastic.co > /etc/hostname -RUN bash /fixture/src/main/resources/provision/installkdc.sh +RUN sh /fixture/provision/installkdc.sh EXPOSE 88 EXPOSE 88/udp -CMD sleep infinity +CMD ["sleep", "infinity"] diff --git a/test/fixtures/krb5kdc-fixture/build.gradle b/test/fixtures/krb5kdc-fixture/build.gradle index c9540011d80dd..887d6a2b68761 100644 --- a/test/fixtures/krb5kdc-fixture/build.gradle +++ b/test/fixtures/krb5kdc-fixture/build.gradle @@ -16,8 +16,8 @@ apply plugin: 'elasticsearch.deploy-test-fixtures' dockerFixtures { krb5dc { dockerContext = projectDir - version = "1.0" - baseImages = ["ubuntu:14.04"] + version = "1.1" + baseImages = ["alpine:3.21.0"] } } diff --git a/test/fixtures/krb5kdc-fixture/src/main/java/org/elasticsearch/test/fixtures/krb5kdc/Krb5kDcContainer.java b/test/fixtures/krb5kdc-fixture/src/main/java/org/elasticsearch/test/fixtures/krb5kdc/Krb5kDcContainer.java index cb1f86de51b1f..f44058d0ebcc4 100644 --- a/test/fixtures/krb5kdc-fixture/src/main/java/org/elasticsearch/test/fixtures/krb5kdc/Krb5kDcContainer.java +++ b/test/fixtures/krb5kdc-fixture/src/main/java/org/elasticsearch/test/fixtures/krb5kdc/Krb5kDcContainer.java @@ -29,7 +29,7 @@ import java.util.List; public final class Krb5kDcContainer extends DockerEnvironmentAwareTestContainer { - public static final String DOCKER_BASE_IMAGE = "docker.elastic.co/elasticsearch-dev/krb5dc-fixture:1.0"; + public static final String DOCKER_BASE_IMAGE = "docker.elastic.co/elasticsearch-dev/krb5dc-fixture:1.1"; private final TemporaryFolder temporaryFolder = new TemporaryFolder(); private final ProvisioningId provisioningId; private Path krb5ConfFile; @@ -39,14 +39,14 @@ public final class Krb5kDcContainer extends DockerEnvironmentAwareTestContainer public enum ProvisioningId { HDFS( "hdfs", - "/fixture/src/main/resources/provision/hdfs.sh", + "/fixture/provision/hdfs.sh", "/fixture/build/keytabs/hdfs_hdfs.build.elastic.co.keytab", "/fixture/build/keytabs/elasticsearch.keytab", "hdfs/hdfs.build.elastic.co@BUILD.ELASTIC.CO" ), PEPPA( "peppa", - "/fixture/src/main/resources/provision/peppa.sh", + "/fixture/provision/peppa.sh", "/fixture/build/keytabs/peppa.keytab", "/fixture/build/keytabs/HTTP_localhost.keytab", "peppa@BUILD.ELASTIC.CO" @@ -94,7 +94,7 @@ public Krb5kDcContainer(ProvisioningId provisioningId) { withNetworkAliases("kerberos.build.elastic.co", "build.elastic.co"); withCopyFileToContainer(MountableFile.forHostPath("/dev/urandom"), "/dev/random"); withExtraHost("kerberos.build.elastic.co", "127.0.0.1"); - withCommand("bash", provisioningId.scriptPath); + withCommand("sh", provisioningId.scriptPath); } @Override @@ -122,7 +122,7 @@ public String getConf() { .findFirst(); String hostPortSpec = bindings.get().getHostPortSpec(); String s = copyFileFromContainer("/fixture/build/krb5.conf.template", i -> IOUtils.toString(i, StandardCharsets.UTF_8)); - return s.replace("${MAPPED_PORT}", hostPortSpec); + return s.replace("#KDC_DOCKER_HOST", "kdc = 127.0.0.1:" + hostPortSpec); } public Path getKeytab() { diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/addprinc.sh b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/addprinc.sh index 44bd7a841dedb..553bd8f85f70c 100755 --- a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/addprinc.sh +++ b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/addprinc.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one # or more contributor license agreements. Licensed under the "Elastic License @@ -24,7 +24,7 @@ PASSWD="$2" USER=$(echo $PRINC | tr "/" "_") VDIR=/fixture -RESOURCES=$VDIR/src/main/resources +RESOURCES=$VDIR PROV_DIR=$RESOURCES/provision ENVPROP_FILE=$RESOURCES/env.properties BUILD_DIR=$VDIR/build @@ -45,16 +45,16 @@ USER_KTAB=$LOCALSTATEDIR/$USER.keytab if [ -f $USER_KTAB ] && [ -z "$PASSWD" ]; then echo "Principal '${PRINC}@${REALM}' already exists. Re-copying keytab..." - sudo cp $USER_KTAB $KEYTAB_DIR/$USER.keytab + cp $USER_KTAB $KEYTAB_DIR/$USER.keytab else if [ -z "$PASSWD" ]; then echo "Provisioning '${PRINC}@${REALM}' principal and keytab..." - sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -randkey $USER_PRIN" - sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "ktadd -k $USER_KTAB $USER_PRIN" - sudo cp $USER_KTAB $KEYTAB_DIR/$USER.keytab + kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -randkey $USER_PRIN" + kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "ktadd -k $USER_KTAB $USER_PRIN" + cp $USER_KTAB $KEYTAB_DIR/$USER.keytab else echo "Provisioning '${PRINC}@${REALM}' principal with password..." - sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -pw $PASSWD $PRINC" + kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -pw $PASSWD $PRINC" fi fi diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/hdfs.sh b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/hdfs.sh index de08a52df3306..cf2eb5a1b7233 100644 --- a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/hdfs.sh +++ b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/hdfs.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh set -e diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/installkdc.sh b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/installkdc.sh index 428747075ff36..a364349c56c68 100755 --- a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/installkdc.sh +++ b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/installkdc.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one # or more contributor license agreements. Licensed under the "Elastic License @@ -12,8 +12,7 @@ set -e # KDC installation steps and considerations based on https://web.mit.edu/kerberos/krb5-latest/doc/admin/install_kdc.html # and helpful input from https://help.ubuntu.com/community/Kerberos -VDIR=/fixture -RESOURCES=$VDIR/src/main/resources +RESOURCES=/fixture PROV_DIR=$RESOURCES/provision ENVPROP_FILE=$RESOURCES/env.properties LOCALSTATEDIR=/etc @@ -49,33 +48,11 @@ touch $LOGDIR/kadmin.log touch $LOGDIR/krb5kdc.log touch $LOGDIR/krb5lib.log -# Update package manager -apt-get update -qqy - -# Installation asks a bunch of questions via debconf. Set the answers ahead of time -debconf-set-selections <<< "krb5-config krb5-config/read_conf boolean true" -debconf-set-selections <<< "krb5-config krb5-config/kerberos_servers string $KDC_NAME" -debconf-set-selections <<< "krb5-config krb5-config/add_servers boolean true" -debconf-set-selections <<< "krb5-config krb5-config/admin_server string $KDC_NAME" -debconf-set-selections <<< "krb5-config krb5-config/add_servers_realm string $REALM_NAME" -debconf-set-selections <<< "krb5-config krb5-config/default_realm string $REALM_NAME" -debconf-set-selections <<< "krb5-admin-server krb5-admin-server/kadmind boolean true" -debconf-set-selections <<< "krb5-admin-server krb5-admin-server/newrealm note" -debconf-set-selections <<< "krb5-kdc krb5-kdc/debconf boolean true" -debconf-set-selections <<< "krb5-kdc krb5-kdc/purge_data_too boolean false" - -# Install krb5 packages -apt-get install -qqy krb5-{admin-server,kdc} - -# /dev/random produces output very slowly on Ubuntu VM's. Install haveged to increase entropy. -apt-get install -qqy haveged -haveged - # Create kerberos database with stash file and garbage password kdb5_util create -s -r $REALM_NAME -P zyxwvutsrpqonmlk9876 # Set up admin acls -cat << EOF > /etc/krb5kdc/kadm5.acl +cat << EOF > /var/lib/krb5kdc/kadm5.acl */admin@$REALM_NAME * */*@$REALM_NAME i EOF diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template index b66709968839a..e79caecbcf334 100644 --- a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template +++ b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template @@ -6,6 +6,7 @@ # License v3.0 only", or the "Server Side Public License, v 1". [libdefaults] + spake_preauth_groups = edwards25519 default_realm = ${REALM_NAME} dns_canonicalize_hostname = false dns_lookup_kdc = false @@ -25,7 +26,7 @@ [realms] ${REALM_NAME} = { kdc = 127.0.0.1:88 - kdc = 127.0.0.1:${MAPPED_PORT} + #KDC_DOCKER_HOST admin_server = ${KDC_NAME}:749 default_domain = ${BUILD_ZONE} } diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/peppa.sh b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/peppa.sh index da6480d891af7..24179da5882c7 100644 --- a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/peppa.sh +++ b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/peppa.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh set -e diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java index fc0ac3286c6e5..c3ce32d4ce333 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java @@ -685,7 +685,7 @@ public static MetadataRolloverService getMetadataRolloverService( new MetadataFieldMapper[] { dtfm }, Collections.emptyMap() ); - mappingLookup = MappingLookup.fromMappers(mapping, List.of(dtfm, dateFieldMapper), List.of(), null); + mappingLookup = MappingLookup.fromMappers(mapping, List.of(dtfm, dateFieldMapper), List.of()); } IndicesService indicesService = mockIndicesServices(mappingLookup); diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index 46f6a0b503bfb..9a160fffb965c 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -141,7 +141,6 @@ import static org.elasticsearch.index.engine.Engine.Operation.Origin.PEER_RECOVERY; import static org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY; import static org.elasticsearch.index.engine.Engine.Operation.Origin.REPLICA; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -458,7 +457,13 @@ protected static ParsedDocument testParsedDocument( } public static ParsedDocument parseDocument(MapperService mapperService, String id, String routing) { - SourceToParse sourceToParse = new SourceToParse(id, new BytesArray("{ \"value\" : \"test\" }"), XContentType.JSON, routing); + String source = randomFrom( + "{ \"value\" : \"test-1\" }", + "{ \"value\" : [\"test-1\",\"test-2\"] }", + "{ \"value\" : [\"test-2\",\"test-1\"] }", + "{ \"value\" : [\"test-1\",\"test-2\",\"test-2\"] }" + ); + SourceToParse sourceToParse = new SourceToParse(id, new BytesArray(source), XContentType.JSON, routing); return mapperService.documentMapper().parse(sourceToParse); } @@ -492,7 +497,8 @@ protected Translog createTranslog(Path translogPath, LongSupplier primaryTermSup new TranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTermSupplier, - seqNo -> {} + seqNo -> {}, + TranslogOperationAsserter.DEFAULT ); } @@ -1072,10 +1078,9 @@ public List generateHistoryOnReplica( final int nestedValues = between(0, 3); final long startTime = threadPool.relativeTimeInNanos(); final int copies = allowDuplicate && rarely() ? between(2, 4) : 1; + final var nonNestedDoc = parseDocument(mapperService, id, null); for (int copy = 0; copy < copies; copy++) { - final ParsedDocument doc = isNestedDoc - ? nestedParsedDocFactory.apply(id, nestedValues) - : parseDocument(engine.engineConfig.getMapperService(), id, null); + final ParsedDocument doc = isNestedDoc ? nestedParsedDocFactory.apply(id, nestedValues) : nonNestedDoc; switch (opType) { case INDEX -> operations.add( new Engine.Index( @@ -1345,6 +1350,7 @@ public static void assertConsistentHistoryBetweenTranslogAndLuceneIndex(Engine e } else { minSeqNoToRetain = engine.getMinRetainedSeqNo(); } + TranslogOperationAsserter translogOperationAsserter = TranslogOperationAsserter.withEngineConfig(engine.engineConfig); for (Translog.Operation translogOp : translogOps) { final Translog.Operation luceneOp = luceneOps.get(translogOp.seqNo()); if (luceneOp == null) { @@ -1372,10 +1378,9 @@ public static void assertConsistentHistoryBetweenTranslogAndLuceneIndex(Engine e assertThat(luceneOp.opType(), equalTo(translogOp.opType())); if (luceneOp.opType() == Translog.Operation.Type.INDEX) { if (engine.engineConfig.getIndexSettings().isRecoverySourceSyntheticEnabled()) { - assertToXContentEquivalent( - ((Translog.Index) luceneOp).source(), - ((Translog.Index) translogOp).source(), - XContentFactory.xContentType(((Translog.Index) luceneOp).source().array()) + assertTrue( + "luceneOp=" + luceneOp + " != translogOp=" + translogOp, + translogOperationAsserter.assertSameIndexOperation((Translog.Index) luceneOp, (Translog.Index) translogOp) ); } else { assertThat(((Translog.Index) luceneOp).source(), equalTo(((Translog.Index) translogOp).source())); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java index 02ae0853909fc..49fe9d30239ae 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java @@ -70,7 +70,6 @@ private TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse sou } ), source, - DocumentParser.Listeners.NOOP, mappingLookup.getMapping().getRoot(), ObjectMapper.Dynamic.getRootDynamic(mappingLookup) ); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 4fe41692e1500..9e2dee4d94212 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -112,7 +112,6 @@ import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; -import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.IndexShard; @@ -206,7 +205,6 @@ public abstract class AggregatorTestCase extends ESTestCase { private static final List TYPE_TEST_BLACKLIST = List.of( ObjectMapper.CONTENT_TYPE, // Cannot aggregate objects DenseVectorFieldMapper.CONTENT_TYPE, // Cannot aggregate dense vectors - RankVectorsFieldMapper.CONTENT_TYPE, // Cannot aggregate dense vectors SparseVectorFieldMapper.CONTENT_TYPE, // Sparse vectors are no longer supported NestedObjectMapper.CONTENT_TYPE, // TODO support for nested @@ -357,8 +355,7 @@ private AggregationContext createAggregationContext( Arrays.stream(fieldTypes) .map(ft -> new FieldAliasMapper(ft.name() + "-alias", ft.name() + "-alias", ft.name())) .collect(toList()), - List.of(), - indexSettings + List.of() ); BiFunction> fieldDataBuilder = (fieldType, context) -> fieldType .fielddataBuilder( @@ -466,7 +463,7 @@ private SubSearchContext buildSubSearchContext( * of stuff. */ SearchExecutionContext subContext = spy(searchExecutionContext); - MappingLookup disableNestedLookup = MappingLookup.fromMappers(Mapping.EMPTY, Set.of(), Set.of(), indexSettings); + MappingLookup disableNestedLookup = MappingLookup.fromMappers(Mapping.EMPTY, Set.of(), Set.of()); doReturn(new NestedDocuments(disableNestedLookup, bitsetFilterCache::getBitSetProducer, indexSettings.getIndexVersionCreated())) .when(subContext) .getNestedDocuments(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/SkipUnavailableRule.java b/test/framework/src/main/java/org/elasticsearch/test/SkipUnavailableRule.java new file mode 100644 index 0000000000000..d5ce943b4d8fe --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/SkipUnavailableRule.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.test; + +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.Arrays; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Test rule to process skip_unavailable override annotations + */ +public class SkipUnavailableRule implements TestRule { + private final Map skipMap; + + public SkipUnavailableRule(String... clusterAliases) { + this.skipMap = Arrays.stream(clusterAliases).collect(Collectors.toMap(Function.identity(), alias -> true)); + } + + public Map getMap() { + return skipMap; + } + + @Override + public Statement apply(Statement base, Description description) { + // Check for annotation named "SkipOverride" and set the overrides accordingly + var aliases = description.getAnnotation(NotSkipped.class); + if (aliases != null) { + for (String alias : aliases.aliases()) { + skipMap.put(alias, false); + } + } + return base; + } + + /** + * Annotation to mark specific cluster in a test as not to be skipped when unavailable + */ + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.METHOD) + public @interface NotSkipped { + String[] aliases(); + } + +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/fixture/HttpHeaderParser.java b/test/framework/src/main/java/org/elasticsearch/test/fixture/HttpHeaderParser.java index 7018e5e259584..ec822c6bc42bf 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/fixture/HttpHeaderParser.java +++ b/test/framework/src/main/java/org/elasticsearch/test/fixture/HttpHeaderParser.java @@ -16,6 +16,7 @@ public enum HttpHeaderParser { ; private static final Pattern RANGE_HEADER_PATTERN = Pattern.compile("bytes=([0-9]+)-([0-9]+)"); + private static final Pattern CONTENT_RANGE_HEADER_PATTERN = Pattern.compile("bytes (?:(\\d+)-(\\d+)|\\*)/(?:(\\d+)|\\*)"); /** * Parse a "Range" header @@ -38,5 +39,75 @@ public static Range parseRangeHeader(String rangeHeaderValue) { return null; } - public record Range(long start, long end) {} + public record Range(long start, long end) { + + public String headerString() { + return "bytes=" + start + "-" + end; + } + } + + /** + * Parse a "Content-Range" header + * + * @see MDN: Content-Range header + * + * @param contentRangeHeaderValue The header value as a string + * @return a {@link ContentRange} instance representing the parsed value, or null if the header is malformed + */ + public static ContentRange parseContentRangeHeader(String contentRangeHeaderValue) { + final Matcher matcher = CONTENT_RANGE_HEADER_PATTERN.matcher(contentRangeHeaderValue); + if (matcher.matches()) { + try { + if (matcher.groupCount() == 3) { + final Long start = parseOptionalLongValue(matcher.group(1)); + final Long end = parseOptionalLongValue(matcher.group(2)); + final Long size = parseOptionalLongValue(matcher.group(3)); + return new ContentRange(start, end, size); + } + } catch (NumberFormatException e) { + return null; + } + } + return null; + } + + private static Long parseOptionalLongValue(String value) { + return value == null ? null : Long.parseLong(value); + } + + /** + * A HTTP "Content Range" + *

+ * This will always contain one of the following combinations of values: + *

    + *
  • start, end, size
    • + *
  • start, end
    • + *
  • size
    • + *
  • nothing
    • + *
+ * + * @param start The start of the range + * @param end The end of the range + * @param size The total size + */ + public record ContentRange(Long start, Long end, Long size) { + + public ContentRange { + assert (start == null) == (end == null) : "Must have either start and end or neither"; + } + + public boolean hasRange() { + return start != null && end != null; + } + + public boolean hasSize() { + return size != null; + } + + public String headerString() { + final String rangeString = hasRange() ? start + "-" + end : "*"; + final String sizeString = hasSize() ? String.valueOf(size) : "*"; + return "bytes " + rangeString + "/" + sizeString; + } + } } diff --git a/test/framework/src/test/java/org/elasticsearch/http/HttpHeaderParserTests.java b/test/framework/src/test/java/org/elasticsearch/http/HttpHeaderParserTests.java index e025e7770ea4c..5fb2c528482c2 100644 --- a/test/framework/src/test/java/org/elasticsearch/http/HttpHeaderParserTests.java +++ b/test/framework/src/test/java/org/elasticsearch/http/HttpHeaderParserTests.java @@ -50,4 +50,48 @@ public void testParseRangeHeaderEndlessRangeNotMatched() { public void testParseRangeHeaderSuffixLengthNotMatched() { assertNull(HttpHeaderParser.parseRangeHeader(Strings.format("bytes=-%d", randomLongBetween(0, Long.MAX_VALUE)))); } + + public void testRangeHeaderString() { + final long start = randomLongBetween(0, 10_000); + final long end = randomLongBetween(start, start + 10_000); + assertEquals("bytes=" + start + "-" + end, new HttpHeaderParser.Range(start, end).headerString()); + } + + public void testParseContentRangeHeaderFull() { + final long start = randomLongBetween(0, 10_000); + final long end = randomLongBetween(start, start + 10_000); + final long size = randomLongBetween(end, Long.MAX_VALUE); + assertEquals( + new HttpHeaderParser.ContentRange(start, end, size), + HttpHeaderParser.parseContentRangeHeader("bytes " + start + "-" + end + "/" + size) + ); + } + + public void testParseContentRangeHeaderNoSize() { + final long start = randomLongBetween(0, 10_000); + final long end = randomLongBetween(start, start + 10_000); + assertEquals( + new HttpHeaderParser.ContentRange(start, end, null), + HttpHeaderParser.parseContentRangeHeader("bytes " + start + "-" + end + "/*") + ); + } + + public void testParseContentRangeHeaderNoRange() { + final long size = randomNonNegativeLong(); + assertEquals(new HttpHeaderParser.ContentRange(null, null, size), HttpHeaderParser.parseContentRangeHeader("bytes */" + size)); + } + + public void testParseNoRangeOrSize() { + assertEquals(new HttpHeaderParser.ContentRange(null, null, null), HttpHeaderParser.parseContentRangeHeader("bytes */*")); + } + + public void testContentRangeHeaderString() { + final long start = randomLongBetween(0, 10_000); + final long end = randomLongBetween(start, start + 10_000); + final long size = randomLongBetween(end, Long.MAX_VALUE); + assertEquals("bytes */*", new HttpHeaderParser.ContentRange(null, null, null).headerString()); + assertEquals("bytes */" + size, new HttpHeaderParser.ContentRange(null, null, size).headerString()); + assertEquals("bytes " + start + "-" + end + "/*", new HttpHeaderParser.ContentRange(start, end, null).headerString()); + assertEquals("bytes " + start + "-" + end + "/" + size, new HttpHeaderParser.ContentRange(start, end, size).headerString()); + } } diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index 54602090050ab..15ebcf3d1feb7 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.IOUtils; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.test.ClasspathUtils; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.TestFeatureService; @@ -33,10 +32,8 @@ import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec; import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite; -import org.elasticsearch.test.rest.yaml.section.DoSection; import org.elasticsearch.test.rest.yaml.section.ExecutableSection; import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xcontent.ParseField; import org.junit.AfterClass; import org.junit.Before; @@ -58,7 +55,6 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * Runs a suite of yaml tests shared with all the official Elasticsearch @@ -217,28 +213,6 @@ public static void closeClient() throws IOException { } } - /** - * Create parameters for this parameterized test. - * Enables support for parsing the legacy version-based node_selector format. - */ - @Deprecated - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static Iterable createParametersWithLegacyNodeSelectorSupport() throws Exception { - var executableSectionRegistry = new NamedXContentRegistry( - Stream.concat( - ExecutableSection.DEFAULT_EXECUTABLE_CONTEXTS.stream().filter(entry -> entry.name.getPreferredName().equals("do") == false), - Stream.of( - new NamedXContentRegistry.Entry( - ExecutableSection.class, - new ParseField("do"), - DoSection::parseWithLegacyNodeSelectorSupport - ) - ) - ).toList() - ); - return createParameters(executableSectionRegistry, null); - } - /** * Create parameters for this parameterized test. Uses the * {@link ExecutableSection#XCONTENT_REGISTRY list} of executable sections diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java index 627554f6b261d..5a212e5b1ec58 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java @@ -19,7 +19,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.core.Tuple; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; @@ -84,16 +83,6 @@ */ public class DoSection implements ExecutableSection { public static DoSection parse(XContentParser parser) throws IOException { - return parse(parser, false); - } - - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - @Deprecated - public static DoSection parseWithLegacyNodeSelectorSupport(XContentParser parser) throws IOException { - return parse(parser, true); - } - - private static DoSection parse(XContentParser parser, boolean enableLegacyNodeSelectorSupport) throws IOException { String currentFieldName = null; XContentParser.Token token; @@ -183,7 +172,7 @@ private static DoSection parse(XContentParser parser, boolean enableLegacyNodeSe if (token == XContentParser.Token.FIELD_NAME) { selectorName = parser.currentName(); } else { - NodeSelector newSelector = buildNodeSelector(selectorName, parser, enableLegacyNodeSelectorSupport); + NodeSelector newSelector = buildNodeSelector(selectorName, parser); nodeSelector = nodeSelector == NodeSelector.ANY ? newSelector : new ComposeNodeSelector(nodeSelector, newSelector); @@ -604,11 +593,10 @@ private String formatStatusCodeMessage(ClientYamlTestResponse restTestResponse, ) ); - private static NodeSelector buildNodeSelector(String name, XContentParser parser, boolean enableLegacyVersionSupport) - throws IOException { + private static NodeSelector buildNodeSelector(String name, XContentParser parser) throws IOException { return switch (name) { case "attribute" -> parseAttributeValuesSelector(parser); - case "version" -> parseVersionSelector(parser, enableLegacyVersionSupport); + case "version" -> parseVersionSelector(parser); default -> throw new XContentParseException(parser.getTokenLocation(), "unknown node_selector [" + name + "]"); }; } @@ -673,7 +661,7 @@ private static boolean matchWithRange( } } - private static NodeSelector parseVersionSelector(XContentParser parser, boolean enableLegacyVersionSupport) throws IOException { + private static NodeSelector parseVersionSelector(XContentParser parser) throws IOException { if (false == parser.currentToken().isValue()) { throw new XContentParseException(parser.getTokenLocation(), "expected [version] to be a value"); } @@ -687,16 +675,10 @@ private static NodeSelector parseVersionSelector(XContentParser parser, boolean nodeMatcher = nodeVersion -> Build.current().version().equals(nodeVersion) == false; versionSelectorString = "version is not current (original)"; } else { - if (enableLegacyVersionSupport) { - var acceptedVersionRange = VersionRange.parseVersionRanges(parser.text()); - nodeMatcher = nodeVersion -> matchWithRange(nodeVersion, acceptedVersionRange, parser.getTokenLocation()); - versionSelectorString = "version ranges " + acceptedVersionRange; - } else { - throw new XContentParseException( - parser.getTokenLocation(), - "unknown version selector [" + parser.text() + "]. Only [current] and [original] are allowed." - ); - } + throw new XContentParseException( + parser.getTokenLocation(), + "unknown version selector [" + parser.text() + "]. Only [current] and [original] are allowed." + ); } return new NodeSelector() { diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSection.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSection.java index bbbf73d74e4ca..ef18a7852840b 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSection.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSection.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.CheckedFunction; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; import org.elasticsearch.test.rest.yaml.Features; import org.elasticsearch.xcontent.XContentLocation; @@ -305,7 +304,6 @@ static void parseSkipSection(XContentParser parser, PrerequisiteSectionBuilder b boolean valid = false; if (parser.currentToken().isValue()) { valid = switch (parser.currentName()) { - case "version" -> parseRestCompatVersion(parser, builder); case "reason" -> parseString(parser, builder::setSkipReason); case "features" -> parseString(parser, f -> parseFeatureField(f, builder)); case "os" -> parseString(parser, builder::skipIfOs); @@ -328,17 +326,6 @@ static void parseSkipSection(XContentParser parser, PrerequisiteSectionBuilder b parser.nextToken(); } - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - private static boolean parseRestCompatVersion(XContentParser parser, PrerequisiteSectionBuilder builder) throws IOException { - // allow skip version only for v7 REST compatibility tests, to be removed for V9 - if ("true".equals(System.getProperty("tests.restCompat"))) return parseString(parser, builder::skipIfVersion); - throw new IllegalArgumentException( - "Skipping by version is no longer supported, please skip based on cluster features. Please check the docs: \n" - + "https://github.com/elastic/elasticsearch/tree/main" - + "/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test#skipping-tests" - ); - } - private static void throwUnexpectedField(String section, XContentParser parser) throws IOException { throw new ParsingException( parser.getTokenLocation(), diff --git a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java index ee8d75b4c034b..465ff7c73e74b 100644 --- a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java +++ b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java @@ -16,9 +16,6 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.core.Strings; -import org.elasticsearch.core.UpdateForV9; -import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; -import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse; import org.elasticsearch.xcontent.XContentLocation; import org.elasticsearch.xcontent.XContentParseException; import org.elasticsearch.xcontent.XContentParser; @@ -31,11 +28,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.regex.Pattern; import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; import static org.hamcrest.CoreMatchers.equalTo; @@ -43,9 +38,6 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; public class DoSectionTests extends AbstractClientYamlTestFragmentParserTestCase { @@ -580,57 +572,6 @@ public void testParseDoSectionAllowedWarnings() throws Exception { assertThat(e.getMessage(), equalTo("the warning [foo] was both allowed and expected")); } - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // remove - public void testLegacyNodeSelectorByVersionRange() throws IOException { - parser = createParser(YamlXContent.yamlXContent, """ - node_selector: - version: 5.2.0-6.0.0 - indices.get_field_mapping: - index: test_index"""); - - DoSection doSection = DoSection.parseWithLegacyNodeSelectorSupport(parser); - assertNotSame(NodeSelector.ANY, doSection.getApiCallSection().getNodeSelector()); - Node v170 = nodeWithVersion("1.7.0"); - Node v521 = nodeWithVersion("5.2.1"); - Node v550 = nodeWithVersion("5.5.0"); - Node v612 = nodeWithVersion("6.1.2"); - List nodes = new ArrayList<>(); - nodes.add(v170); - nodes.add(v521); - nodes.add(v550); - nodes.add(v612); - doSection.getApiCallSection().getNodeSelector().select(nodes); - assertEquals(Arrays.asList(v521, v550), nodes); - ClientYamlTestExecutionContext context = mock(ClientYamlTestExecutionContext.class); - ClientYamlTestResponse mockResponse = mock(ClientYamlTestResponse.class); - when( - context.callApi( - "indices.get_field_mapping", - singletonMap("index", "test_index"), - emptyList(), - emptyMap(), - doSection.getApiCallSection().getNodeSelector() - ) - ).thenReturn(mockResponse); - when(context.nodesVersions()).thenReturn(Set.of(randomAlphaOfLength(10))); - when(mockResponse.getHeaders("X-elastic-product")).thenReturn(List.of("Elasticsearch")); - doSection.execute(context); - verify(context).callApi( - "indices.get_field_mapping", - singletonMap("index", "test_index"), - emptyList(), - emptyMap(), - doSection.getApiCallSection().getNodeSelector() - ); - - { - List badNodes = new ArrayList<>(); - badNodes.add(new Node(new HttpHost("dummy"))); - Exception e = expectThrows(IllegalStateException.class, () -> doSection.getApiCallSection().getNodeSelector().select(badNodes)); - assertEquals("expected [version] metadata to be set but got [host=http://dummy]", e.getMessage()); - } - } - public void testNodeSelectorByVersionRangeFails() throws IOException { parser = createParser(YamlXContent.yamlXContent, """ node_selector: diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java index 558f8e6f22ac1..f4a101eabd38a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java @@ -39,7 +39,6 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiPredicate; import java.util.function.Predicate; import java.util.function.Supplier; @@ -443,25 +442,27 @@ public IndicesAccessControl authorize( FieldPermissionsCache fieldPermissionsCache ) { // Short circuit if the indicesPermission allows all access to every index - if (Arrays.stream(groups).anyMatch(Group::isTotal)) { - return IndicesAccessControl.allowAll(); + for (Group group : groups) { + if (group.isTotal()) { + return IndicesAccessControl.allowAll(); + } } final Map resources = Maps.newMapWithExpectedSize(requestedIndicesOrAliases.size()); - final AtomicInteger totalResourceCountHolder = new AtomicInteger(0); + int totalResourceCount = 0; for (String indexOrAlias : requestedIndicesOrAliases) { final IndexResource resource = new IndexResource(indexOrAlias, lookup.get(indexOrAlias)); resources.put(resource.name, resource); - totalResourceCountHolder.getAndAdd(resource.size()); + totalResourceCount += resource.size(); } final boolean overallGranted = isActionGranted(action, resources); - + final int finalTotalResourceCount = totalResourceCount; final Supplier> indexPermissions = () -> buildIndicesAccessControl( action, resources, - totalResourceCountHolder.get(), + finalTotalResourceCount, fieldPermissionsCache ); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index d8b4b15307c47..ee9c8adc47adc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -225,11 +225,19 @@ static RoleDescriptor kibanaSystem(String name) { RoleDescriptor.IndicesPrivileges.builder().indices("logs-fleet_server*").privileges("read", "delete_index").build(), // Legacy "Alerts as data" used in Security Solution. // Kibana user creates these indices; reads / writes to them. - RoleDescriptor.IndicesPrivileges.builder().indices(ReservedRolesStore.ALERTS_LEGACY_INDEX).privileges("all").build(), + RoleDescriptor.IndicesPrivileges.builder() + .indices(ReservedRolesStore.ALERTS_LEGACY_INDEX, ReservedRolesStore.ALERTS_LEGACY_INDEX_REINDEXED_V8) + .privileges("all") + .build(), // Used in Security Solution for value lists. // Kibana user creates these indices; reads / writes to them. RoleDescriptor.IndicesPrivileges.builder() - .indices(ReservedRolesStore.LISTS_INDEX, ReservedRolesStore.LISTS_ITEMS_INDEX) + .indices( + ReservedRolesStore.LISTS_INDEX, + ReservedRolesStore.LISTS_ITEMS_INDEX, + ReservedRolesStore.LISTS_INDEX_REINDEXED_V8, + ReservedRolesStore.LISTS_ITEMS_INDEX_REINDEXED_V8 + ) .privileges("all") .build(), // "Alerts as data" internal backing indices used in Security Solution, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java index e43ae2d1b360b..3648d8a0c7daa 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java @@ -43,6 +43,7 @@ public class ReservedRolesStore implements BiConsumer, ActionListener> { /** "Security Solutions" only legacy signals index */ public static final String ALERTS_LEGACY_INDEX = ".siem-signals*"; + public static final String ALERTS_LEGACY_INDEX_REINDEXED_V8 = ".reindexed-v8-siem-signals*"; /** Alerts, Rules, Cases (RAC) index used by multiple solutions */ public static final String ALERTS_BACKING_INDEX = ".internal.alerts*"; @@ -60,9 +61,11 @@ public class ReservedRolesStore implements BiConsumer, ActionListene /** "Security Solutions" only lists index for value lists for detections */ public static final String LISTS_INDEX = ".lists-*"; + public static final String LISTS_INDEX_REINDEXED_V8 = ".reindexed-v8-lists-*"; /** "Security Solutions" only lists index for value list items for detections */ public static final String LISTS_ITEMS_INDEX = ".items-*"; + public static final String LISTS_ITEMS_INDEX_REINDEXED_V8 = ".reindexed-v8-items-*"; /** Index pattern for Universal Profiling */ public static final String UNIVERSAL_PROFILING_ALIASES = "profiling-*"; @@ -536,70 +539,6 @@ private static Map initializeReservedRoles() { + "and roles that grant access to Kibana." ) ), - // DEPRECATED: to be removed in 9.0.0 - entry( - "data_frame_transforms_admin", - new RoleDescriptor( - "data_frame_transforms_admin", - new String[] { "manage_data_frame_transforms" }, - new RoleDescriptor.IndicesPrivileges[] { - RoleDescriptor.IndicesPrivileges.builder() - .indices( - TransformInternalIndexConstants.AUDIT_INDEX_PATTERN, - TransformInternalIndexConstants.AUDIT_INDEX_PATTERN_DEPRECATED, - TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS - ) - .privileges("view_index_metadata", "read") - .build() }, - new RoleDescriptor.ApplicationResourcePrivileges[] { - RoleDescriptor.ApplicationResourcePrivileges.builder() - .application("kibana-*") - .resources("*") - .privileges("reserved_ml_user") - .build() }, - null, - null, - MetadataUtils.getDeprecatedReservedMetadata("Please use the [transform_admin] role instead"), - null, - null, - null, - null, - "Grants manage_data_frame_transforms cluster privileges, which enable you to manage transforms. " - + "This role also includes all Kibana privileges for the machine learning features." - ) - ), - // DEPRECATED: to be removed in 9.0.0 - entry( - "data_frame_transforms_user", - new RoleDescriptor( - "data_frame_transforms_user", - new String[] { "monitor_data_frame_transforms" }, - new RoleDescriptor.IndicesPrivileges[] { - RoleDescriptor.IndicesPrivileges.builder() - .indices( - TransformInternalIndexConstants.AUDIT_INDEX_PATTERN, - TransformInternalIndexConstants.AUDIT_INDEX_PATTERN_DEPRECATED, - TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS - ) - .privileges("view_index_metadata", "read") - .build() }, - new RoleDescriptor.ApplicationResourcePrivileges[] { - RoleDescriptor.ApplicationResourcePrivileges.builder() - .application("kibana-*") - .resources("*") - .privileges("reserved_ml_user") - .build() }, - null, - null, - MetadataUtils.getDeprecatedReservedMetadata("Please use the [transform_user] role instead"), - null, - null, - null, - null, - "Grants monitor_data_frame_transforms cluster privileges, which enable you to use transforms. " - + "This role also includes all Kibana privileges for the machine learning features. " - ) - ), entry( "transform_admin", new RoleDescriptor( @@ -829,7 +768,14 @@ private static RoleDescriptor buildViewerRoleDescriptor() { .build(), // Security RoleDescriptor.IndicesPrivileges.builder() - .indices(ReservedRolesStore.ALERTS_LEGACY_INDEX, ReservedRolesStore.LISTS_INDEX, ReservedRolesStore.LISTS_ITEMS_INDEX) + .indices( + ReservedRolesStore.ALERTS_LEGACY_INDEX, + ReservedRolesStore.LISTS_INDEX, + ReservedRolesStore.LISTS_ITEMS_INDEX, + ReservedRolesStore.ALERTS_LEGACY_INDEX_REINDEXED_V8, + ReservedRolesStore.LISTS_INDEX_REINDEXED_V8, + ReservedRolesStore.LISTS_ITEMS_INDEX_REINDEXED_V8 + ) .privileges("read", "view_index_metadata") .build(), // Alerts-as-data @@ -880,7 +826,14 @@ private static RoleDescriptor buildEditorRoleDescriptor() { .build(), // Security RoleDescriptor.IndicesPrivileges.builder() - .indices(ReservedRolesStore.ALERTS_LEGACY_INDEX, ReservedRolesStore.LISTS_INDEX, ReservedRolesStore.LISTS_ITEMS_INDEX) + .indices( + ReservedRolesStore.ALERTS_LEGACY_INDEX, + ReservedRolesStore.LISTS_INDEX, + ReservedRolesStore.LISTS_ITEMS_INDEX, + ReservedRolesStore.ALERTS_LEGACY_INDEX_REINDEXED_V8, + ReservedRolesStore.LISTS_INDEX_REINDEXED_V8, + ReservedRolesStore.LISTS_ITEMS_INDEX_REINDEXED_V8 + ) .privileges("read", "view_index_metadata", "write", "maintenance") .build(), // Alerts-as-data diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java index 1de584d5593f1..79a679441de3a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java @@ -27,12 +27,5 @@ public class TransformDeprecations { public static final String MAX_PAGE_SEARCH_SIZE_BREAKING_CHANGES_URL = "https://ela.st/es-deprecation-7-transform-max-page-search-size"; - public static final String DATA_FRAME_TRANSFORMS_ROLES_BREAKING_CHANGES_URL = - "https://ela.st/es-deprecation-9-data-frame-transforms-roles"; - - public static final String DATA_FRAME_TRANSFORMS_ROLES_IS_DEPRECATED = "This transform configuration uses one or more obsolete roles " - + "prefixed with [data_frame_transformers_] which will be unsupported after the next upgrade. Switch to a user with the equivalent " - + "roles prefixed with [transform_] and use [/_transform/_upgrade] to upgrade all transforms to the latest roles.";; - private TransformDeprecations() {} } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index 745da71539992..d84040aaf7a85 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -24,13 +24,11 @@ import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.common.time.TimeUtils; import org.elasticsearch.xpack.core.common.validation.SourceDestValidator; import org.elasticsearch.xpack.core.common.validation.SourceDestValidator.SourceDestValidation; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue.Level; -import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer; import org.elasticsearch.xpack.core.security.xcontent.XContentUtils; import org.elasticsearch.xpack.core.transform.TransformConfigVersion; import org.elasticsearch.xpack.core.transform.TransformDeprecations; @@ -43,7 +41,6 @@ import java.io.IOException; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Locale; @@ -52,7 +49,6 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; -import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AUTHENTICATION_KEY; /** * This class holds the configuration details of a data frame transform @@ -69,10 +65,6 @@ public final class TransformConfig implements SimpleDiffable, W public static final ParseField HEADERS = new ParseField("headers"); /** Version in which {@code FieldCapabilitiesRequest.runtime_fields} field was introduced. */ private static final TransportVersion FIELD_CAPS_RUNTIME_MAPPINGS_INTRODUCED_TRANSPORT_VERSION = TransportVersions.V_7_12_0; - private static final List DEPRECATED_DATA_FRAME_TRANSFORMS_ROLES = List.of( - "data_frame_transforms_admin", - "data_frame_transforms_user" - ); /** Specifies all the possible transform functions. */ public enum Function { @@ -413,37 +405,9 @@ public List checkForDeprecations(NamedXContentRegistry namedXC retentionPolicyConfig.checkForDeprecations(getId(), namedXContentRegistry, deprecations::add); } - var deprecatedTransformRoles = getRolesFromHeaders().stream().filter(DEPRECATED_DATA_FRAME_TRANSFORMS_ROLES::contains).toList(); - if (deprecatedTransformRoles.isEmpty() == false) { - deprecations.add( - new DeprecationIssue( - Level.CRITICAL, - "Transform [" + id + "] uses deprecated transform roles " + deprecatedTransformRoles, - TransformDeprecations.DATA_FRAME_TRANSFORMS_ROLES_BREAKING_CHANGES_URL, - TransformDeprecations.DATA_FRAME_TRANSFORMS_ROLES_IS_DEPRECATED, - false, - null - ) - ); - } - return deprecations; } - private List getRolesFromHeaders() throws IOException { - if (headers == null) { - return Collections.emptyList(); - } - - var encodedAuthenticationHeader = ClientHelper.filterSecurityHeaders(headers).getOrDefault(AUTHENTICATION_KEY, ""); - if (encodedAuthenticationHeader.isEmpty()) { - return Collections.emptyList(); - } - - var decodedAuthenticationHeader = AuthenticationContextSerializer.decode(encodedAuthenticationHeader); - return Arrays.asList(decodedAuthenticationHeader.getEffectiveSubject().getUser().roles()); - } - @Override public void writeTo(final StreamOutput out) throws IOException { out.writeString(id); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java index ceb375a97d1df..5369c95ad6fa7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java @@ -630,7 +630,7 @@ private void runTestOnIndices(int numberIndices, CheckedConsumer concreteFields) { List mappers = concreteFields.stream().map(MockFieldMapper::new).collect(Collectors.toList()); - return MappingLookup.fromMappers(Mapping.EMPTY, mappers, emptyList(), null); + return MappingLookup.fromMappers(Mapping.EMPTY, mappers, emptyList()); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index a96b26ddcb1eb..b08dd90ae9065 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -277,8 +277,6 @@ public void testIsReserved() { assertThat(ReservedRolesStore.isReserved("reporting_user"), is(true)); assertThat(ReservedRolesStore.isReserved("machine_learning_user"), is(true)); assertThat(ReservedRolesStore.isReserved("machine_learning_admin"), is(true)); - assertThat(ReservedRolesStore.isReserved("data_frame_transforms_user"), is(true)); - assertThat(ReservedRolesStore.isReserved("data_frame_transforms_admin"), is(true)); assertThat(ReservedRolesStore.isReserved("transform_user"), is(true)); assertThat(ReservedRolesStore.isReserved("transform_admin"), is(true)); assertThat(ReservedRolesStore.isReserved("watcher_user"), is(true)); @@ -613,6 +611,7 @@ public void testKibanaSystemRole() { ".apm-custom-link", ".apm-source-map", ReservedRolesStore.ALERTS_LEGACY_INDEX + randomAlphaOfLength(randomIntBetween(0, 13)), + ReservedRolesStore.ALERTS_LEGACY_INDEX_REINDEXED_V8 + randomAlphaOfLength(randomIntBetween(0, 13)), ReservedRolesStore.ALERTS_BACKING_INDEX + randomAlphaOfLength(randomIntBetween(0, 13)), ReservedRolesStore.ALERTS_BACKING_INDEX_REINDEXED + randomAlphaOfLength(randomIntBetween(0, 13)), ReservedRolesStore.ALERTS_INDEX_ALIAS + randomAlphaOfLength(randomIntBetween(0, 13)), @@ -620,7 +619,9 @@ public void testKibanaSystemRole() { ReservedRolesStore.PREVIEW_ALERTS_BACKING_INDEX + randomAlphaOfLength(randomIntBetween(0, 13)), ReservedRolesStore.PREVIEW_ALERTS_BACKING_INDEX_REINDEXED + randomAlphaOfLength(randomIntBetween(0, 13)), ReservedRolesStore.LISTS_INDEX + randomAlphaOfLength(randomIntBetween(0, 13)), + ReservedRolesStore.LISTS_INDEX_REINDEXED_V8 + randomAlphaOfLength(randomIntBetween(0, 13)), ReservedRolesStore.LISTS_ITEMS_INDEX + randomAlphaOfLength(randomIntBetween(0, 13)), + ReservedRolesStore.LISTS_ITEMS_INDEX_REINDEXED_V8 + randomAlphaOfLength(randomIntBetween(0, 13)), ".slo-observability." + randomAlphaOfLength(randomIntBetween(0, 13)) ).forEach(index -> assertAllIndicesAccessAllowed(kibanaRole, index)); @@ -3421,185 +3422,99 @@ public void testTransformAdminRole() { final TransportRequest request = mock(TransportRequest.class); final Authentication authentication = AuthenticationTestHelper.builder().build(); - RoleDescriptor[] roleDescriptors = { - ReservedRolesStore.roleDescriptor("data_frame_transforms_admin"), - ReservedRolesStore.roleDescriptor("transform_admin") }; + RoleDescriptor roleDescriptor = ReservedRolesStore.roleDescriptor("transform_admin"); - for (RoleDescriptor roleDescriptor : roleDescriptors) { - assertNotNull(roleDescriptor); - assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true)); - if (roleDescriptor.getName().equals("data_frame_transforms_admin")) { - assertThat(roleDescriptor.getMetadata(), hasEntry("_deprecated", true)); - } else { - assertThat(roleDescriptor.getMetadata(), not(hasEntry("_deprecated", true))); - } + assertNotNull(roleDescriptor); + assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true)); + assertThat(roleDescriptor.getMetadata(), not(hasEntry("_deprecated", true))); - final String allowedApplicationActionPattern = "example/custom/action/*"; - final String kibanaApplicationWithRandomIndex = "kibana-" + randomFrom(randomAlphaOfLengthBetween(8, 24), ".kibana"); - List lookup = roleDescriptor.getName().equals("data_frame_transforms_admin") - ? List.of( - new ApplicationPrivilegeDescriptor( - kibanaApplicationWithRandomIndex, - "reserved_ml_user", - Set.of(allowedApplicationActionPattern), - Map.of() - ) - ) - : List.of(); - Role role = Role.buildFromRoleDescriptor(roleDescriptor, new FieldPermissionsCache(Settings.EMPTY), RESTRICTED_INDICES, lookup); - assertThat(role.cluster().check(DeleteTransformAction.NAME, request, authentication), is(true)); - assertThat(role.cluster().check(GetTransformAction.NAME, request, authentication), is(true)); - assertThat(role.cluster().check(GetTransformStatsAction.NAME, request, authentication), is(true)); - assertThat(role.cluster().check(PreviewTransformAction.NAME, request, authentication), is(true)); - assertThat(role.cluster().check(PutTransformAction.NAME, request, authentication), is(true)); - assertThat(role.cluster().check(StartTransformAction.NAME, request, authentication), is(true)); - assertThat(role.cluster().check(StopTransformAction.NAME, request, authentication), is(true)); - assertThat(role.cluster().check(SetTransformUpgradeModeAction.NAME, request, authentication), is(true)); - assertThat(role.cluster().check(DelegatePkiAuthenticationAction.NAME, request, authentication), is(false)); - - assertThat(role.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(false)); - - assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS); - assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_PATTERN); - assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_PATTERN_DEPRECATED); - assertNoAccessAllowed(role, "foo"); - assertNoAccessAllowed(role, TransformInternalIndexConstants.LATEST_INDEX_NAME); // internal use only - - assertNoAccessAllowed(role, TestRestrictedIndices.SAMPLE_RESTRICTED_NAMES); - assertNoAccessAllowed(role, XPackPlugin.ASYNC_RESULTS_INDEX + randomAlphaOfLengthBetween(0, 2)); + final String allowedApplicationActionPattern = "example/custom/action/*"; + final String kibanaApplicationWithRandomIndex = "kibana-" + randomFrom(randomAlphaOfLengthBetween(8, 24), ".kibana"); + List lookup = List.of(); + Role role = Role.buildFromRoleDescriptor(roleDescriptor, new FieldPermissionsCache(Settings.EMPTY), RESTRICTED_INDICES, lookup); + assertThat(role.cluster().check(DeleteTransformAction.NAME, request, authentication), is(true)); + assertThat(role.cluster().check(GetTransformAction.NAME, request, authentication), is(true)); + assertThat(role.cluster().check(GetTransformStatsAction.NAME, request, authentication), is(true)); + assertThat(role.cluster().check(PreviewTransformAction.NAME, request, authentication), is(true)); + assertThat(role.cluster().check(PutTransformAction.NAME, request, authentication), is(true)); + assertThat(role.cluster().check(StartTransformAction.NAME, request, authentication), is(true)); + assertThat(role.cluster().check(StopTransformAction.NAME, request, authentication), is(true)); + assertThat(role.cluster().check(SetTransformUpgradeModeAction.NAME, request, authentication), is(true)); + assertThat(role.cluster().check(DelegatePkiAuthenticationAction.NAME, request, authentication), is(false)); - assertThat( - role.application() - .grants(ApplicationPrivilegeTests.createPrivilege(kibanaApplicationWithRandomIndex, "app-foo", "foo"), "*"), - is(false) - ); + assertThat(role.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(false)); - if (roleDescriptor.getName().equals("data_frame_transforms_admin")) { - assertThat( - role.application() - .grants( - ApplicationPrivilegeTests.createPrivilege( - kibanaApplicationWithRandomIndex, - "app-reserved_ml", - allowedApplicationActionPattern - ), - "*" - ), - is(true) - ); - } + assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS); + assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_PATTERN); + assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_PATTERN_DEPRECATED); + assertNoAccessAllowed(role, "foo"); + assertNoAccessAllowed(role, TransformInternalIndexConstants.LATEST_INDEX_NAME); // internal use only - final String otherApplication = "logstash-" + randomAlphaOfLengthBetween(8, 24); - assertThat( - role.application().grants(ApplicationPrivilegeTests.createPrivilege(otherApplication, "app-foo", "foo"), "*"), - is(false) - ); - if (roleDescriptor.getName().equals("data_frame_transforms_admin")) { - assertThat( - role.application() - .grants( - ApplicationPrivilegeTests.createPrivilege(otherApplication, "app-reserved_ml", allowedApplicationActionPattern), - "*" - ), - is(false) - ); - } - } + assertNoAccessAllowed(role, TestRestrictedIndices.SAMPLE_RESTRICTED_NAMES); + assertNoAccessAllowed(role, XPackPlugin.ASYNC_RESULTS_INDEX + randomAlphaOfLengthBetween(0, 2)); + + assertThat( + role.application().grants(ApplicationPrivilegeTests.createPrivilege(kibanaApplicationWithRandomIndex, "app-foo", "foo"), "*"), + is(false) + ); + + final String otherApplication = "logstash-" + randomAlphaOfLengthBetween(8, 24); + assertThat( + role.application().grants(ApplicationPrivilegeTests.createPrivilege(otherApplication, "app-foo", "foo"), "*"), + is(false) + ); } public void testTransformUserRole() { final TransportRequest request = mock(TransportRequest.class); final Authentication authentication = AuthenticationTestHelper.builder().build(); - RoleDescriptor[] roleDescriptors = { - ReservedRolesStore.roleDescriptor("data_frame_transforms_user"), - ReservedRolesStore.roleDescriptor("transform_user") }; + RoleDescriptor roleDescriptor = ReservedRolesStore.roleDescriptor("transform_user"); - for (RoleDescriptor roleDescriptor : roleDescriptors) { - assertNotNull(roleDescriptor); - assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true)); - if (roleDescriptor.getName().equals("data_frame_transforms_user")) { - assertThat(roleDescriptor.getMetadata(), hasEntry("_deprecated", true)); - } else { - assertThat(roleDescriptor.getMetadata(), not(hasEntry("_deprecated", true))); - } + assertNotNull(roleDescriptor); + assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true)); + assertThat(roleDescriptor.getMetadata(), not(hasEntry("_deprecated", true))); - final String allowedApplicationActionPattern = "example/custom/action/*"; - final String kibanaApplicationWithRandomIndex = "kibana-" + randomFrom(randomAlphaOfLengthBetween(8, 24), ".kibana"); - List lookup = roleDescriptor.getName().equals("data_frame_transforms_user") - ? List.of( - new ApplicationPrivilegeDescriptor( - kibanaApplicationWithRandomIndex, - "reserved_ml_user", - Set.of(allowedApplicationActionPattern), - Map.of() - ) - ) - : List.of(); - Role role = Role.buildFromRoleDescriptor(roleDescriptor, new FieldPermissionsCache(Settings.EMPTY), RESTRICTED_INDICES, lookup); - assertThat(role.cluster().check(DeleteTransformAction.NAME, request, authentication), is(false)); - assertThat(role.cluster().check(GetTransformAction.NAME, request, authentication), is(true)); - assertThat(role.cluster().check(GetTransformStatsAction.NAME, request, authentication), is(true)); - assertThat(role.cluster().check(PreviewTransformAction.NAME, request, authentication), is(false)); - assertThat(role.cluster().check(PutTransformAction.NAME, request, authentication), is(false)); - assertThat(role.cluster().check(StartTransformAction.NAME, request, authentication), is(false)); - assertThat(role.cluster().check(StopTransformAction.NAME, request, authentication), is(false)); - assertThat(role.cluster().check(SetTransformUpgradeModeAction.NAME, request, authentication), is(false)); - assertThat(role.cluster().check(DelegatePkiAuthenticationAction.NAME, request, authentication), is(false)); - assertThat(role.cluster().check(ActivateProfileAction.NAME, request, authentication), is(false)); - assertThat(role.cluster().check(SuggestProfilesAction.NAME, request, authentication), is(false)); - assertThat(role.cluster().check(UpdateProfileDataAction.NAME, request, authentication), is(false)); - assertThat(role.cluster().check(GetProfilesAction.NAME, request, authentication), is(false)); - assertThat(role.cluster().check(ProfileHasPrivilegesAction.NAME, request, authentication), is(false)); - - assertThat(role.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(false)); - - assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS); - assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_PATTERN); - assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_PATTERN_DEPRECATED); - assertNoAccessAllowed(role, "foo"); - assertNoAccessAllowed(role, TransformInternalIndexConstants.LATEST_INDEX_NAME); - - assertNoAccessAllowed(role, TestRestrictedIndices.SAMPLE_RESTRICTED_NAMES); - assertNoAccessAllowed(role, XPackPlugin.ASYNC_RESULTS_INDEX + randomAlphaOfLengthBetween(0, 2)); + final String allowedApplicationActionPattern = "example/custom/action/*"; + final String kibanaApplicationWithRandomIndex = "kibana-" + randomFrom(randomAlphaOfLengthBetween(8, 24), ".kibana"); + List lookup = List.of(); + Role role = Role.buildFromRoleDescriptor(roleDescriptor, new FieldPermissionsCache(Settings.EMPTY), RESTRICTED_INDICES, lookup); + assertThat(role.cluster().check(DeleteTransformAction.NAME, request, authentication), is(false)); + assertThat(role.cluster().check(GetTransformAction.NAME, request, authentication), is(true)); + assertThat(role.cluster().check(GetTransformStatsAction.NAME, request, authentication), is(true)); + assertThat(role.cluster().check(PreviewTransformAction.NAME, request, authentication), is(false)); + assertThat(role.cluster().check(PutTransformAction.NAME, request, authentication), is(false)); + assertThat(role.cluster().check(StartTransformAction.NAME, request, authentication), is(false)); + assertThat(role.cluster().check(StopTransformAction.NAME, request, authentication), is(false)); + assertThat(role.cluster().check(SetTransformUpgradeModeAction.NAME, request, authentication), is(false)); + assertThat(role.cluster().check(DelegatePkiAuthenticationAction.NAME, request, authentication), is(false)); + assertThat(role.cluster().check(ActivateProfileAction.NAME, request, authentication), is(false)); + assertThat(role.cluster().check(SuggestProfilesAction.NAME, request, authentication), is(false)); + assertThat(role.cluster().check(UpdateProfileDataAction.NAME, request, authentication), is(false)); + assertThat(role.cluster().check(GetProfilesAction.NAME, request, authentication), is(false)); + assertThat(role.cluster().check(ProfileHasPrivilegesAction.NAME, request, authentication), is(false)); - assertThat( - role.application() - .grants(ApplicationPrivilegeTests.createPrivilege(kibanaApplicationWithRandomIndex, "app-foo", "foo"), "*"), - is(false) - ); + assertThat(role.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(false)); - if (roleDescriptor.getName().equals("data_frame_transforms_user")) { - assertThat( - role.application() - .grants( - ApplicationPrivilegeTests.createPrivilege( - kibanaApplicationWithRandomIndex, - "app-reserved_ml", - allowedApplicationActionPattern - ), - "*" - ), - is(true) - ); - } + assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS); + assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_PATTERN); + assertOnlyReadAllowed(role, TransformInternalIndexConstants.AUDIT_INDEX_PATTERN_DEPRECATED); + assertNoAccessAllowed(role, "foo"); + assertNoAccessAllowed(role, TransformInternalIndexConstants.LATEST_INDEX_NAME); - final String otherApplication = "logstash-" + randomAlphaOfLengthBetween(8, 24); - assertThat( - role.application().grants(ApplicationPrivilegeTests.createPrivilege(otherApplication, "app-foo", "foo"), "*"), - is(false) - ); - if (roleDescriptor.getName().equals("data_frame_transforms_user")) { - assertThat( - role.application() - .grants( - ApplicationPrivilegeTests.createPrivilege(otherApplication, "app-reserved_ml", allowedApplicationActionPattern), - "*" - ), - is(false) - ); - } - } + assertNoAccessAllowed(role, TestRestrictedIndices.SAMPLE_RESTRICTED_NAMES); + assertNoAccessAllowed(role, XPackPlugin.ASYNC_RESULTS_INDEX + randomAlphaOfLengthBetween(0, 2)); + + assertThat( + role.application().grants(ApplicationPrivilegeTests.createPrivilege(kibanaApplicationWithRandomIndex, "app-foo", "foo"), "*"), + is(false) + ); + + final String otherApplication = "logstash-" + randomAlphaOfLengthBetween(8, 24); + assertThat( + role.application().grants(ApplicationPrivilegeTests.createPrivilege(otherApplication, "app-foo", "foo"), "*"), + is(false) + ); } public void testWatcherAdminRole() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java index 2e7e5293c835f..a9b4fa984ea1e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java @@ -27,8 +27,6 @@ import org.elasticsearch.xpack.core.common.validation.SourceDestValidator.SourceDestValidation; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue.Level; -import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; -import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import org.elasticsearch.xpack.core.transform.TransformConfigVersion; import org.elasticsearch.xpack.core.transform.TransformDeprecations; @@ -46,7 +44,6 @@ import java.util.Map; import static org.elasticsearch.test.TestMatchers.matchesPattern; -import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AUTHENTICATION_KEY; import static org.elasticsearch.xpack.core.transform.transforms.DestConfigTests.randomDestConfig; import static org.elasticsearch.xpack.core.transform.transforms.SourceConfigTests.randomInvalidSourceConfig; import static org.elasticsearch.xpack.core.transform.transforms.SourceConfigTests.randomSourceConfig; @@ -61,8 +58,6 @@ public class TransformConfigTests extends AbstractSerializingTransformTestCase roles) throws IOException { - var authentication = AuthenticationTestHelper.builder() - .realm() - .user(new User(randomAlphaOfLength(10), roles.toArray(String[]::new))) - .build(); - Map headers = Map.of(AUTHENTICATION_KEY, authentication.encode()); - TransformConfig deprecatedConfig = randomTransformConfigWithHeaders(headers); - - // important: checkForDeprecations does _not_ create new deprecation warnings - assertThat( - deprecatedConfig.checkForDeprecations(xContentRegistry()), - equalTo( - List.of( - new DeprecationIssue( - Level.CRITICAL, - "Transform [" + deprecatedConfig.getId() + "] uses deprecated transform roles " + roles, - TransformDeprecations.DATA_FRAME_TRANSFORMS_ROLES_BREAKING_CHANGES_URL, - TransformDeprecations.DATA_FRAME_TRANSFORMS_ROLES_IS_DEPRECATED, - false, - null - ) - ) - ) - ); - } - public void testSerializingMetadataPreservesOrder() throws IOException { String json = Strings.format(""" { diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java index 452f40baa34a8..6e43d40a3005a 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java @@ -12,6 +12,7 @@ import org.apache.http.HttpHost; import org.elasticsearch.Version; import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -37,9 +38,11 @@ import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.elasticsearch.xpack.esql.ccq.Clusters.REMOTE_CLUSTER_NAME; import static org.hamcrest.Matchers.any; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasKey; @ThreadLeakFilters(filters = TestClustersThreadFilter.class) public class MultiClustersIT extends ESRestTestCase { @@ -395,6 +398,40 @@ public void testIndexPattern() throws Exception { } } + @SuppressWarnings("unchecked") + public void testStats() throws IOException { + assumeTrue("capabilities endpoint is not available", capabilitiesEndpointAvailable()); + + Request caps = new Request("GET", "_capabilities?method=GET&path=_cluster/stats&capabilities=esql-stats"); + Response capsResponse = client().performRequest(caps); + Map capsResult = entityAsMap(capsResponse.getEntity()); + assumeTrue("esql stats capability missing", capsResult.get("supported").equals(true)); + + run("FROM test-local-index,*:test-remote-index | STATS total = SUM(data) BY color | SORT color", includeCCSMetadata()); + Request stats = new Request("GET", "_cluster/stats"); + Response statsResponse = client().performRequest(stats); + Map result = entityAsMap(statsResponse.getEntity()); + assertThat(result, hasKey("ccs")); + Map ccs = (Map) result.get("ccs"); + assertThat(ccs, hasKey("_esql")); + Map esql = (Map) ccs.get("_esql"); + assertThat(esql, hasKey("total")); + assertThat(esql, hasKey("success")); + assertThat(esql, hasKey("took")); + assertThat(esql, hasKey("remotes_per_search_max")); + assertThat(esql, hasKey("remotes_per_search_avg")); + assertThat(esql, hasKey("failure_reasons")); + assertThat(esql, hasKey("features")); + assertThat(esql, hasKey("clusters")); + Map clusters = (Map) esql.get("clusters"); + assertThat(clusters, hasKey(REMOTE_CLUSTER_NAME)); + assertThat(clusters, hasKey("(local)")); + Map clusterData = (Map) clusters.get(REMOTE_CLUSTER_NAME); + assertThat(clusterData, hasKey("total")); + assertThat(clusterData, hasKey("skipped")); + assertThat(clusterData, hasKey("took")); + } + private RestClient remoteClusterClient() throws IOException { var clusterHosts = parseClusterHosts(remoteCluster.getHttpAddresses()); return buildClient(restClientSettings(), clusterHosts.toArray(new HttpHost[0])); @@ -404,6 +441,10 @@ private static boolean ccsMetadataAvailable() { return Clusters.localClusterVersion().onOrAfter(Version.V_8_16_0); } + private static boolean capabilitiesEndpointAvailable() { + return Clusters.localClusterVersion().onOrAfter(Version.V_8_15_0); + } + private static boolean includeCCSMetadata() { return ccsMetadataAvailable() && randomBoolean(); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java index f0bdf089f69d1..cbab6d0acfef7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java @@ -551,11 +551,11 @@ public static Type asType(ElementType elementType, Type actualType) { } private static Type bytesRefBlockType(Type actualType) { - if (actualType == GEO_POINT || actualType == CARTESIAN_POINT || actualType == GEO_SHAPE || actualType == CARTESIAN_SHAPE) { - return actualType; - } else { - return KEYWORD; - } + return switch (actualType) { + case NULL -> NULL; + case GEO_POINT, CARTESIAN_POINT, GEO_SHAPE, CARTESIAN_SHAPE -> actualType; + default -> KEYWORD; + }; } Object convert(String value) { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 8c1ff650b4f52..fbd4f9feca78d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.logging.LogConfigurator; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Nullable; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.test.rest.ESRestTestCase; @@ -50,71 +51,72 @@ public class CsvTestsDataLoader { private static final int BULK_DATA_SIZE = 100_000; - private static final TestsDataset EMPLOYEES = new TestsDataset("employees", "mapping-default.json", "employees.csv").noSubfields(); - private static final TestsDataset EMPLOYEES_INCOMPATIBLE = new TestsDataset( + private static final TestDataset EMPLOYEES = new TestDataset("employees", "mapping-default.json", "employees.csv").noSubfields(); + private static final TestDataset EMPLOYEES_INCOMPATIBLE = new TestDataset( "employees_incompatible", "mapping-default-incompatible.json", "employees_incompatible.csv" ).noSubfields(); - private static final TestsDataset HOSTS = new TestsDataset("hosts"); - private static final TestsDataset APPS = new TestsDataset("apps"); - private static final TestsDataset APPS_SHORT = APPS.withIndex("apps_short").withTypeMapping(Map.of("id", "short")); - private static final TestsDataset LANGUAGES = new TestsDataset("languages"); - private static final TestsDataset LANGUAGES_LOOKUP = LANGUAGES.withIndex("languages_lookup") + private static final TestDataset HOSTS = new TestDataset("hosts"); + private static final TestDataset APPS = new TestDataset("apps"); + private static final TestDataset APPS_SHORT = APPS.withIndex("apps_short").withTypeMapping(Map.of("id", "short")); + private static final TestDataset LANGUAGES = new TestDataset("languages"); + private static final TestDataset LANGUAGES_LOOKUP = LANGUAGES.withIndex("languages_lookup") .withSetting("languages_lookup-settings.json"); - private static final TestsDataset LANGUAGES_LOOKUP_NON_UNIQUE_KEY = LANGUAGES_LOOKUP.withIndex("languages_lookup_non_unique_key") + private static final TestDataset LANGUAGES_LOOKUP_NON_UNIQUE_KEY = LANGUAGES_LOOKUP.withIndex("languages_lookup_non_unique_key") .withData("languages_non_unique_key.csv"); - private static final TestsDataset LANGUAGES_NESTED_FIELDS = new TestsDataset( + private static final TestDataset LANGUAGES_NESTED_FIELDS = new TestDataset( "languages_nested_fields", "mapping-languages_nested_fields.json", "languages_nested_fields.csv" ).withSetting("languages_lookup-settings.json"); - private static final TestsDataset ALERTS = new TestsDataset("alerts"); - private static final TestsDataset UL_LOGS = new TestsDataset("ul_logs"); - private static final TestsDataset SAMPLE_DATA = new TestsDataset("sample_data"); - private static final TestsDataset MV_SAMPLE_DATA = new TestsDataset("mv_sample_data"); - private static final TestsDataset SAMPLE_DATA_STR = SAMPLE_DATA.withIndex("sample_data_str") + private static final TestDataset ALERTS = new TestDataset("alerts"); + private static final TestDataset UL_LOGS = new TestDataset("ul_logs"); + private static final TestDataset SAMPLE_DATA = new TestDataset("sample_data"); + private static final TestDataset MV_SAMPLE_DATA = new TestDataset("mv_sample_data"); + private static final TestDataset SAMPLE_DATA_STR = SAMPLE_DATA.withIndex("sample_data_str") .withTypeMapping(Map.of("client_ip", "keyword")); - private static final TestsDataset SAMPLE_DATA_TS_LONG = SAMPLE_DATA.withIndex("sample_data_ts_long") + private static final TestDataset SAMPLE_DATA_TS_LONG = SAMPLE_DATA.withIndex("sample_data_ts_long") .withData("sample_data_ts_long.csv") .withTypeMapping(Map.of("@timestamp", "long")); - private static final TestsDataset SAMPLE_DATA_TS_NANOS = SAMPLE_DATA.withIndex("sample_data_ts_nanos") + private static final TestDataset SAMPLE_DATA_TS_NANOS = SAMPLE_DATA.withIndex("sample_data_ts_nanos") .withData("sample_data_ts_nanos.csv") .withTypeMapping(Map.of("@timestamp", "date_nanos")); - private static final TestsDataset MISSING_IP_SAMPLE_DATA = new TestsDataset("missing_ip_sample_data"); - private static final TestsDataset CLIENT_IPS = new TestsDataset("clientips"); - private static final TestsDataset CLIENT_IPS_LOOKUP = CLIENT_IPS.withIndex("clientips_lookup") + private static final TestDataset MISSING_IP_SAMPLE_DATA = new TestDataset("missing_ip_sample_data"); + private static final TestDataset CLIENT_IPS = new TestDataset("clientips"); + private static final TestDataset CLIENT_IPS_LOOKUP = CLIENT_IPS.withIndex("clientips_lookup") .withSetting("clientips_lookup-settings.json"); - private static final TestsDataset MESSAGE_TYPES = new TestsDataset("message_types"); - private static final TestsDataset MESSAGE_TYPES_LOOKUP = MESSAGE_TYPES.withIndex("message_types_lookup") + private static final TestDataset MESSAGE_TYPES = new TestDataset("message_types"); + private static final TestDataset MESSAGE_TYPES_LOOKUP = MESSAGE_TYPES.withIndex("message_types_lookup") .withSetting("message_types_lookup-settings.json"); - private static final TestsDataset CLIENT_CIDR = new TestsDataset("client_cidr"); - private static final TestsDataset AGES = new TestsDataset("ages"); - private static final TestsDataset HEIGHTS = new TestsDataset("heights"); - private static final TestsDataset DECADES = new TestsDataset("decades"); - private static final TestsDataset AIRPORTS = new TestsDataset("airports"); - private static final TestsDataset AIRPORTS_MP = AIRPORTS.withIndex("airports_mp").withData("airports_mp.csv"); - private static final TestsDataset AIRPORTS_NO_DOC_VALUES = new TestsDataset("airports_no_doc_values").withData("airports.csv"); - private static final TestsDataset AIRPORTS_NOT_INDEXED = new TestsDataset("airports_not_indexed").withData("airports.csv"); - private static final TestsDataset AIRPORTS_NOT_INDEXED_NOR_DOC_VALUES = new TestsDataset("airports_not_indexed_nor_doc_values") - .withData("airports.csv"); - private static final TestsDataset AIRPORTS_WEB = new TestsDataset("airports_web"); - private static final TestsDataset DATE_NANOS = new TestsDataset("date_nanos"); - private static final TestsDataset COUNTRIES_BBOX = new TestsDataset("countries_bbox"); - private static final TestsDataset COUNTRIES_BBOX_WEB = new TestsDataset("countries_bbox_web"); - private static final TestsDataset AIRPORT_CITY_BOUNDARIES = new TestsDataset("airport_city_boundaries"); - private static final TestsDataset CARTESIAN_MULTIPOLYGONS = new TestsDataset("cartesian_multipolygons"); - private static final TestsDataset CARTESIAN_MULTIPOLYGONS_NO_DOC_VALUES = new TestsDataset("cartesian_multipolygons_no_doc_values") + private static final TestDataset CLIENT_CIDR = new TestDataset("client_cidr"); + private static final TestDataset AGES = new TestDataset("ages"); + private static final TestDataset HEIGHTS = new TestDataset("heights"); + private static final TestDataset DECADES = new TestDataset("decades"); + private static final TestDataset AIRPORTS = new TestDataset("airports"); + private static final TestDataset AIRPORTS_MP = AIRPORTS.withIndex("airports_mp").withData("airports_mp.csv"); + private static final TestDataset AIRPORTS_NO_DOC_VALUES = new TestDataset("airports_no_doc_values").withData("airports.csv"); + private static final TestDataset AIRPORTS_NOT_INDEXED = new TestDataset("airports_not_indexed").withData("airports.csv"); + private static final TestDataset AIRPORTS_NOT_INDEXED_NOR_DOC_VALUES = new TestDataset("airports_not_indexed_nor_doc_values").withData( + "airports.csv" + ); + private static final TestDataset AIRPORTS_WEB = new TestDataset("airports_web"); + private static final TestDataset DATE_NANOS = new TestDataset("date_nanos"); + private static final TestDataset COUNTRIES_BBOX = new TestDataset("countries_bbox"); + private static final TestDataset COUNTRIES_BBOX_WEB = new TestDataset("countries_bbox_web"); + private static final TestDataset AIRPORT_CITY_BOUNDARIES = new TestDataset("airport_city_boundaries"); + private static final TestDataset CARTESIAN_MULTIPOLYGONS = new TestDataset("cartesian_multipolygons"); + private static final TestDataset CARTESIAN_MULTIPOLYGONS_NO_DOC_VALUES = new TestDataset("cartesian_multipolygons_no_doc_values") .withData("cartesian_multipolygons.csv"); - private static final TestsDataset MULTIVALUE_GEOMETRIES = new TestsDataset("multivalue_geometries"); - private static final TestsDataset MULTIVALUE_POINTS = new TestsDataset("multivalue_points"); - private static final TestsDataset DISTANCES = new TestsDataset("distances"); - private static final TestsDataset K8S = new TestsDataset("k8s", "k8s-mappings.json", "k8s.csv").withSetting("k8s-settings.json"); - private static final TestsDataset ADDRESSES = new TestsDataset("addresses"); - private static final TestsDataset BOOKS = new TestsDataset("books").withSetting("books-settings.json"); - private static final TestsDataset SEMANTIC_TEXT = new TestsDataset("semantic_text").withInferenceEndpoint(true); - - public static final Map CSV_DATASET_MAP = Map.ofEntries( + private static final TestDataset MULTIVALUE_GEOMETRIES = new TestDataset("multivalue_geometries"); + private static final TestDataset MULTIVALUE_POINTS = new TestDataset("multivalue_points"); + private static final TestDataset DISTANCES = new TestDataset("distances"); + private static final TestDataset K8S = new TestDataset("k8s", "k8s-mappings.json", "k8s.csv").withSetting("k8s-settings.json"); + private static final TestDataset ADDRESSES = new TestDataset("addresses"); + private static final TestDataset BOOKS = new TestDataset("books").withSetting("books-settings.json"); + private static final TestDataset SEMANTIC_TEXT = new TestDataset("semantic_text").withInferenceEndpoint(true); + + public static final Map CSV_DATASET_MAP = Map.ofEntries( Map.entry(EMPLOYEES.indexName, EMPLOYEES), Map.entry(EMPLOYEES_INCOMPATIBLE.indexName, EMPLOYEES_INCOMPATIBLE), Map.entry(HOSTS.indexName, HOSTS), @@ -265,12 +267,12 @@ public static void main(String[] args) throws IOException { } } - public static Set availableDatasetsForEs(RestClient client, boolean supportsIndexModeLookup) throws IOException { + public static Set availableDatasetsForEs(RestClient client, boolean supportsIndexModeLookup) throws IOException { boolean inferenceEnabled = clusterHasInferenceEndpoint(client); - Set testDataSets = new HashSet<>(); + Set testDataSets = new HashSet<>(); - for (TestsDataset dataset : CSV_DATASET_MAP.values()) { + for (TestDataset dataset : CSV_DATASET_MAP.values()) { if ((inferenceEnabled || dataset.requiresInferenceEndpoint == false) && (supportsIndexModeLookup || isLookupDataset(dataset) == false)) { testDataSets.add(dataset); @@ -280,7 +282,7 @@ public static Set availableDatasetsForEs(RestClient client, boolea return testDataSets; } - public static boolean isLookupDataset(TestsDataset dataset) throws IOException { + public static boolean isLookupDataset(TestDataset dataset) throws IOException { Settings settings = dataset.readSettingsFile(); String mode = settings.get("index.mode"); return (mode != null && mode.equalsIgnoreCase("lookup")); @@ -362,7 +364,7 @@ private static void loadEnrichPolicy(RestClient client, String policyName, Strin client.performRequest(request); } - private static void load(RestClient client, TestsDataset dataset, Logger logger, IndexCreator indexCreator) throws IOException { + private static void load(RestClient client, TestDataset dataset, Logger logger, IndexCreator indexCreator) throws IOException { final String mappingName = "/" + dataset.mappingFileName; URL mapping = CsvTestsDataLoader.class.getResource(mappingName); if (mapping == null) { @@ -603,25 +605,32 @@ private static void forceMerge(RestClient client, Set indices, Logger lo } } - public record TestsDataset( + public record MultiIndexTestDataset(String indexPattern, List datasets) { + public static MultiIndexTestDataset of(TestDataset testsDataset) { + return new MultiIndexTestDataset(testsDataset.indexName, List.of(testsDataset)); + } + + } + + public record TestDataset( String indexName, String mappingFileName, String dataFileName, String settingFileName, boolean allowSubFields, - Map typeMapping, + @Nullable Map typeMapping, boolean requiresInferenceEndpoint ) { - public TestsDataset(String indexName, String mappingFileName, String dataFileName) { + public TestDataset(String indexName, String mappingFileName, String dataFileName) { this(indexName, mappingFileName, dataFileName, null, true, null, false); } - public TestsDataset(String indexName) { + public TestDataset(String indexName) { this(indexName, "mapping-" + indexName + ".json", indexName + ".csv", null, true, null, false); } - public TestsDataset withIndex(String indexName) { - return new TestsDataset( + public TestDataset withIndex(String indexName) { + return new TestDataset( indexName, mappingFileName, dataFileName, @@ -632,8 +641,8 @@ public TestsDataset withIndex(String indexName) { ); } - public TestsDataset withData(String dataFileName) { - return new TestsDataset( + public TestDataset withData(String dataFileName) { + return new TestDataset( indexName, mappingFileName, dataFileName, @@ -644,8 +653,8 @@ public TestsDataset withData(String dataFileName) { ); } - public TestsDataset withSetting(String settingFileName) { - return new TestsDataset( + public TestDataset withSetting(String settingFileName) { + return new TestDataset( indexName, mappingFileName, dataFileName, @@ -656,8 +665,8 @@ public TestsDataset withSetting(String settingFileName) { ); } - public TestsDataset noSubfields() { - return new TestsDataset( + public TestDataset noSubfields() { + return new TestDataset( indexName, mappingFileName, dataFileName, @@ -668,8 +677,8 @@ public TestsDataset noSubfields() { ); } - public TestsDataset withTypeMapping(Map typeMapping) { - return new TestsDataset( + public TestDataset withTypeMapping(Map typeMapping) { + return new TestDataset( indexName, mappingFileName, dataFileName, @@ -680,8 +689,8 @@ public TestsDataset withTypeMapping(Map typeMapping) { ); } - public TestsDataset withInferenceEndpoint(boolean needsInference) { - return new TestsDataset(indexName, mappingFileName, dataFileName, settingFileName, allowSubFields, typeMapping, needsInference); + public TestDataset withInferenceEndpoint(boolean needsInference) { + return new TestDataset(indexName, mappingFileName, dataFileName, settingFileName, allowSubFields, typeMapping, needsInference); } private Settings readSettingsFile() throws IOException { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 9b1356438141c..95119cae95590 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -419,6 +419,7 @@ FROM employees | EVAL language_code = emp_no % 10 | LOOKUP JOIN languages_lookup_non_unique_key ON language_code | SORT emp_no +| EVAL language_name = MV_SORT(language_name) | KEEP emp_no, language_code, language_name ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index bf6e2f8ae0893..81164382c0541 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -445,6 +445,7 @@ count:long | message:keyword multiIndexMissingIpToString required_capability: union_types +required_capability: metadata_fields required_capability: union_types_missing_field FROM sample_data, sample_data_str, missing_ip_sample_data METADATA _index @@ -479,6 +480,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450 multiIndexMissingIpToIp required_capability: union_types +required_capability: metadata_fields required_capability: union_types_missing_field FROM sample_data, sample_data_str, missing_ip_sample_data METADATA _index @@ -1373,9 +1375,6 @@ client_ip:ip | event_duration:long | message:keyword | @timestamp:keywo # Once INLINESTATS supports expressions in agg functions and groups, convert the group in the inlinestats multiIndexIndirectUseOfUnionTypesInSort -// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: -// make the csv tests work with multiple indices. -required_capability: union_types FROM sample_data, sample_data_ts_long | SORT client_ip ASC | LIMIT 1 @@ -1386,8 +1385,6 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInEval -// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: -// make the csv tests work with multiple indices. required_capability: union_types FROM sample_data, sample_data_ts_long | EVAL foo = event_duration > 1232381 @@ -1400,9 +1397,6 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInRename -// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: -// make the csv tests work with multiple indices. -required_capability: union_types required_capability: union_types_fix_rename_resolution FROM sample_data, sample_data_ts_long | RENAME message AS event_message @@ -1415,9 +1409,6 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInKeep -// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: -// make the csv tests work with multiple indices. -required_capability: union_types FROM sample_data, sample_data_ts_long | KEEP client_ip, event_duration, message | SORT client_ip ASC @@ -1429,9 +1420,6 @@ client_ip:ip | event_duration:long | message:keyword ; multiIndexIndirectUseOfUnionTypesInWildcardKeep -// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: -// make the csv tests work with multiple indices. -required_capability: union_types required_capability: union_types_fix_rename_resolution FROM sample_data, sample_data_ts_long | KEEP * @@ -1444,9 +1432,6 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInWildcardKeep2 -// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: -// make the csv tests work with multiple indices. -required_capability: union_types required_capability: union_types_fix_rename_resolution FROM sample_data, sample_data_ts_long | KEEP *e* @@ -1460,9 +1445,6 @@ FROM sample_data, sample_data_ts_long multiIndexUseOfUnionTypesInKeep -// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: -// make the csv tests work with multiple indices. -required_capability: union_types required_capability: union_types_fix_rename_resolution FROM sample_data, sample_data_ts_long | KEEP @timestamp @@ -1474,9 +1456,6 @@ null ; multiIndexUseOfUnionTypesInDrop -// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: -// make the csv tests work with multiple indices. -required_capability: union_types required_capability: union_types_fix_rename_resolution FROM sample_data, sample_data_ts_long | DROP @timestamp @@ -1489,9 +1468,6 @@ client_ip:ip | event_duration:long | message:keyword ; multiIndexIndirectUseOfUnionTypesInWildcardDrop -// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: -// make the csv tests work with multiple indices. -required_capability: union_types required_capability: union_types_fix_rename_resolution FROM sample_data, sample_data_ts_long | DROP *time* @@ -1504,9 +1480,6 @@ client_ip:ip | event_duration:long | message:keyword ; multiIndexIndirectUseOfUnionTypesInWhere -// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: -// make the csv tests work with multiple indices. -required_capability: union_types FROM sample_data, sample_data_ts_long | WHERE message == "Disconnected" ; @@ -1517,9 +1490,6 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInDissect -// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: -// make the csv tests work with multiple indices. -required_capability: union_types FROM sample_data, sample_data_ts_long | DISSECT message "%{foo}" | SORT client_ip ASC @@ -1531,9 +1501,6 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInGrok -// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: -// make the csv tests work with multiple indices. -required_capability: union_types FROM sample_data, sample_data_ts_long | GROK message "%{WORD:foo}" | SORT client_ip ASC @@ -1545,9 +1512,6 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInEnrich -// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: -// make the csv tests work with multiple indices. -required_capability: union_types required_capability: enrich_load FROM sample_data, sample_data_ts_long | EVAL client_ip = client_ip::keyword @@ -1561,9 +1525,6 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInStats -// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: -// make the csv tests work with multiple indices. -required_capability: union_types FROM sample_data, sample_data_ts_long | STATS foo = max(event_duration) BY client_ip | SORT client_ip ASC @@ -1577,9 +1538,6 @@ foo:long | client_ip:ip ; multiIndexIndirectUseOfUnionTypesInInlineStats-Ignore -// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: -// make the csv tests work with multiple indices. -required_capability: union_types required_capability: inlinestats FROM sample_data, sample_data_ts_long | INLINESTATS foo = max(event_duration) @@ -1592,9 +1550,6 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInLookup-Ignore -// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: -// make the csv tests work with multiple indices. -required_capability: union_types required_capability: lookup_v4 FROM sample_data, sample_data_ts_long | SORT client_ip ASC @@ -1608,9 +1563,6 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInMvExpand -// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: -// make the csv tests work with multiple indices. -required_capability: union_types FROM sample_data, sample_data_ts_long | EVAL foo = MV_APPEND(message, message) | SORT client_ip ASC diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractCrossClustersUsageTelemetryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractCrossClustersUsageTelemetryIT.java new file mode 100644 index 0000000000000..ffbddd52b2551 --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractCrossClustersUsageTelemetryIT.java @@ -0,0 +1,205 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.cluster.stats.CCSTelemetrySnapshot; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.AbstractMultiClustersTestCase; +import org.elasticsearch.test.SkipUnavailableRule; +import org.elasticsearch.usage.UsageService; +import org.elasticsearch.xpack.core.async.GetAsyncResultRequest; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.elasticsearch.core.TimeValue.timeValueMillis; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; + +public class AbstractCrossClustersUsageTelemetryIT extends AbstractMultiClustersTestCase { + private static final Logger LOGGER = LogManager.getLogger(AbstractCrossClustersUsageTelemetryIT.class); + protected static final String REMOTE1 = "cluster-a"; + protected static final String REMOTE2 = "cluster-b"; + protected static final String LOCAL_INDEX = "logs-1"; + protected static final String REMOTE_INDEX = "logs-2"; + // We want to send search to a specific node (we don't care which one) so that we could + // collect the CCS telemetry from it later + protected String queryNode; + + @Before + public void setupQueryNode() { + // The tests are set up in a way that all queries within a single test are sent to the same node, + // thus enabling incremental collection of telemetry data, but the node is random for each test. + queryNode = cluster(LOCAL_CLUSTER).getRandomNodeName(); + } + + protected CCSTelemetrySnapshot getTelemetryFromQuery(String query, String client) throws ExecutionException, InterruptedException { + EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest(); + request.query(query); + request.pragmas(AbstractEsqlIntegTestCase.randomPragmas()); + request.columnar(randomBoolean()); + request.includeCCSMetadata(randomBoolean()); + return getTelemetryFromQuery(request, client); + } + + protected CCSTelemetrySnapshot getTelemetryFromQuery(EsqlQueryRequest request, String client) throws ExecutionException, + InterruptedException { + // We don't care here too much about the response, we just want to trigger the telemetry collection. + // So we check it's not null and leave the rest to other tests. + if (client != null) { + assertResponse( + cluster(LOCAL_CLUSTER).client(queryNode) + .filterWithHeader(Map.of(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER, client)) + .execute(EsqlQueryAction.INSTANCE, request), + Assert::assertNotNull + ); + + } else { + assertResponse(cluster(LOCAL_CLUSTER).client(queryNode).execute(EsqlQueryAction.INSTANCE, request), Assert::assertNotNull); + } + return getTelemetrySnapshot(queryNode); + } + + protected CCSTelemetrySnapshot getTelemetryFromAsyncQuery(String query) throws Exception { + EsqlQueryRequest request = EsqlQueryRequest.asyncEsqlQueryRequest(); + request.query(query); + request.pragmas(AbstractEsqlIntegTestCase.randomPragmas()); + request.columnar(randomBoolean()); + request.includeCCSMetadata(randomBoolean()); + request.waitForCompletionTimeout(TimeValue.timeValueMillis(100)); + request.keepOnCompletion(false); + return getTelemetryFromAsyncQuery(request); + } + + protected CCSTelemetrySnapshot getTelemetryFromAsyncQuery(EsqlQueryRequest request) throws Exception { + AtomicReference asyncExecutionId = new AtomicReference<>(); + assertResponse(cluster(LOCAL_CLUSTER).client(queryNode).execute(EsqlQueryAction.INSTANCE, request), resp -> { + if (resp.isRunning()) { + assertNotNull("async execution id is null", resp.asyncExecutionId()); + asyncExecutionId.set(resp.asyncExecutionId().get()); + } + }); + if (asyncExecutionId.get() != null) { + assertBusy(() -> { + var getResultsRequest = new GetAsyncResultRequest(asyncExecutionId.get()).setWaitForCompletionTimeout(timeValueMillis(1)); + try ( + var resp = cluster(LOCAL_CLUSTER).client(queryNode) + .execute(EsqlAsyncGetResultAction.INSTANCE, getResultsRequest) + .actionGet(30, TimeUnit.SECONDS) + ) { + assertFalse(resp.isRunning()); + } + }); + } + return getTelemetrySnapshot(queryNode); + } + + protected CCSTelemetrySnapshot getTelemetryFromFailedQuery(String query) throws Exception { + EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest(); + request.query(query); + request.pragmas(AbstractEsqlIntegTestCase.randomPragmas()); + request.columnar(randomBoolean()); + request.includeCCSMetadata(randomBoolean()); + + ExecutionException ee = expectThrows( + ExecutionException.class, + cluster(LOCAL_CLUSTER).client(queryNode).execute(EsqlQueryAction.INSTANCE, request)::get + ); + assertNotNull(ee.getCause()); + + return getTelemetrySnapshot(queryNode); + } + + private CCSTelemetrySnapshot getTelemetrySnapshot(String nodeName) { + var usage = cluster(LOCAL_CLUSTER).getInstance(UsageService.class, nodeName); + return usage.getEsqlUsageHolder().getCCSTelemetrySnapshot(); + } + + @Override + protected boolean reuseClusters() { + return false; + } + + @Override + protected List remoteClusterAlias() { + return List.of(REMOTE1, REMOTE2); + } + + @Rule + public SkipUnavailableRule skipOverride = new SkipUnavailableRule(REMOTE1, REMOTE2); + + protected Map setupClusters() { + int numShardsLocal = randomIntBetween(1, 5); + populateLocalIndices(LOCAL_INDEX, numShardsLocal); + + int numShardsRemote = randomIntBetween(1, 5); + populateRemoteIndices(REMOTE1, REMOTE_INDEX, numShardsRemote); + + Map clusterInfo = new HashMap<>(); + clusterInfo.put("local.num_shards", numShardsLocal); + clusterInfo.put("local.index", LOCAL_INDEX); + clusterInfo.put("remote.num_shards", numShardsRemote); + clusterInfo.put("remote.index", REMOTE_INDEX); + + int numShardsRemote2 = randomIntBetween(1, 5); + populateRemoteIndices(REMOTE2, REMOTE_INDEX, numShardsRemote2); + clusterInfo.put("remote2.index", REMOTE_INDEX); + clusterInfo.put("remote2.num_shards", numShardsRemote2); + + return clusterInfo; + } + + void populateLocalIndices(String indexName, int numShards) { + Client localClient = client(LOCAL_CLUSTER); + assertAcked( + localClient.admin() + .indices() + .prepareCreate(indexName) + .setSettings(Settings.builder().put("index.number_of_shards", numShards)) + .setMapping("id", "type=keyword", "tag", "type=keyword", "v", "type=long") + ); + for (int i = 0; i < 10; i++) { + localClient.prepareIndex(indexName).setSource("id", "local-" + i, "tag", "local", "v", i).get(); + } + localClient.admin().indices().prepareRefresh(indexName).get(); + } + + void populateRemoteIndices(String clusterAlias, String indexName, int numShards) { + Client remoteClient = client(clusterAlias); + assertAcked( + remoteClient.admin() + .indices() + .prepareCreate(indexName) + .setSettings(Settings.builder().put("index.number_of_shards", numShards)) + .setMapping("id", "type=keyword", "tag", "type=keyword", "v", "type=long") + ); + for (int i = 0; i < 10; i++) { + remoteClient.prepareIndex(indexName).setSource("id", "remote-" + i, "tag", "remote", "v", i * i).get(); + } + remoteClient.admin().indices().prepareRefresh(indexName).get(); + } + + @Override + protected Map skipUnavailableForRemoteClusters() { + var map = skipOverride.getMap(); + LOGGER.info("Using skip_unavailable map: [{}]", map); + return map; + } +} diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersUsageTelemetryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersUsageTelemetryIT.java new file mode 100644 index 0000000000000..33d868e7a69eb --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersUsageTelemetryIT.java @@ -0,0 +1,231 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.action; + +import org.elasticsearch.action.admin.cluster.stats.CCSTelemetrySnapshot; +import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.SkipUnavailableRule; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.ASYNC_FEATURE; +import static org.hamcrest.Matchers.equalTo; + +public class CrossClustersUsageTelemetryIT extends AbstractCrossClustersUsageTelemetryIT { + + @Override + protected Collection> nodePlugins(String clusterAlias) { + List> plugins = new ArrayList<>(super.nodePlugins(clusterAlias)); + plugins.add(EsqlPluginWithEnterpriseOrTrialLicense.class); + plugins.add(CrossClustersQueryIT.InternalExchangePlugin.class); + return plugins; + } + + public void assertPerClusterCount(CCSTelemetrySnapshot.PerClusterCCSTelemetry perCluster, long count) { + assertThat(perCluster.getCount(), equalTo(count)); + assertThat(perCluster.getSkippedCount(), equalTo(0L)); + assertThat(perCluster.getTook().count(), equalTo(count)); + } + + public void testLocalRemote() throws Exception { + setupClusters(); + var telemetry = getTelemetryFromQuery("from logs-*,c*:logs-* | stats sum (v)", "kibana"); + + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + assertThat(telemetry.getFailureReasons().size(), equalTo(0)); + assertThat(telemetry.getTook().count(), equalTo(1L)); + assertThat(telemetry.getTookMrtFalse().count(), equalTo(0L)); + assertThat(telemetry.getTookMrtTrue().count(), equalTo(0L)); + assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0)); + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L)); + assertThat(telemetry.getClientCounts().size(), equalTo(1)); + assertThat(telemetry.getClientCounts().get("kibana"), equalTo(1L)); + assertThat(telemetry.getFeatureCounts().get(ASYNC_FEATURE), equalTo(null)); + + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(3)); + for (String clusterAlias : remoteClusterAlias()) { + assertPerClusterCount(perCluster.get(clusterAlias), 1L); + } + assertPerClusterCount(perCluster.get(LOCAL_CLUSTER), 1L); + + telemetry = getTelemetryFromQuery("from logs-*,c*:logs-* | stats sum (v)", "kibana"); + assertThat(telemetry.getTotalCount(), equalTo(2L)); + assertThat(telemetry.getClientCounts().get("kibana"), equalTo(2L)); + perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(3)); + for (String clusterAlias : remoteClusterAlias()) { + assertPerClusterCount(perCluster.get(clusterAlias), 2L); + } + assertPerClusterCount(perCluster.get(LOCAL_CLUSTER), 2L); + } + + public void testLocalOnly() throws Exception { + setupClusters(); + // Should not produce any usage info since it's a local search + var telemetry = getTelemetryFromQuery("from logs-* | stats sum (v)", "kibana"); + + assertThat(telemetry.getTotalCount(), equalTo(0L)); + assertThat(telemetry.getSuccessCount(), equalTo(0L)); + assertThat(telemetry.getByRemoteCluster().size(), equalTo(0)); + } + + @SkipUnavailableRule.NotSkipped(aliases = REMOTE1) + public void testFailed() throws Exception { + setupClusters(); + // Should not produce any usage info since it's a local search + var telemetry = getTelemetryFromFailedQuery("from no_such_index | stats sum (v)"); + + assertThat(telemetry.getTotalCount(), equalTo(0L)); + assertThat(telemetry.getSuccessCount(), equalTo(0L)); + assertThat(telemetry.getByRemoteCluster().size(), equalTo(0)); + + // One remote is skipped, one is not + telemetry = getTelemetryFromFailedQuery("from logs-*,c*:no_such_index | stats sum (v)"); + + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(0L)); + assertThat(telemetry.getByRemoteCluster().size(), equalTo(1)); + assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0)); + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L)); + Map expectedFailure = Map.of(CCSUsageTelemetry.Result.NOT_FOUND.getName(), 1L); + assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure)); + // cluster-b should be skipped + assertThat(telemetry.getByRemoteCluster().get(REMOTE2).getCount(), equalTo(0L)); + assertThat(telemetry.getByRemoteCluster().get(REMOTE2).getSkippedCount(), equalTo(1L)); + + // this is only for cluster-a so no skipped remotes + telemetry = getTelemetryFromFailedQuery("from logs-*,cluster-a:no_such_index | stats sum (v)"); + assertThat(telemetry.getTotalCount(), equalTo(2L)); + assertThat(telemetry.getSuccessCount(), equalTo(0L)); + assertThat(telemetry.getByRemoteCluster().size(), equalTo(1)); + assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0)); + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L)); + expectedFailure = Map.of(CCSUsageTelemetry.Result.NOT_FOUND.getName(), 2L); + assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure)); + assertThat(telemetry.getByRemoteCluster().size(), equalTo(1)); + } + + // TODO: enable when skip-up patch is merged + // public void testSkipAllRemotes() throws Exception { + // var telemetry = getTelemetryFromQuery("from logs-*,c*:no_such_index | stats sum (v)", "unknown"); + // + // assertThat(telemetry.getTotalCount(), equalTo(1L)); + // assertThat(telemetry.getSuccessCount(), equalTo(1L)); + // assertThat(telemetry.getFailureReasons().size(), equalTo(0)); + // assertThat(telemetry.getTook().count(), equalTo(1L)); + // assertThat(telemetry.getTookMrtFalse().count(), equalTo(0L)); + // assertThat(telemetry.getTookMrtTrue().count(), equalTo(0L)); + // assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0)); + // assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); + // assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L)); + // assertThat(telemetry.getClientCounts().size(), equalTo(0)); + // + // var perCluster = telemetry.getByRemoteCluster(); + // assertThat(perCluster.size(), equalTo(3)); + // for (String clusterAlias : remoteClusterAlias()) { + // var clusterData = perCluster.get(clusterAlias); + // assertThat(clusterData.getCount(), equalTo(0L)); + // assertThat(clusterData.getSkippedCount(), equalTo(1L)); + // assertThat(clusterData.getTook().count(), equalTo(0L)); + // } + // assertPerClusterCount(perCluster.get(LOCAL_CLUSTER), 1L); + // } + + public void testRemoteOnly() throws Exception { + setupClusters(); + var telemetry = getTelemetryFromQuery("from c*:logs-* | stats sum (v)", "kibana"); + + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + assertThat(telemetry.getFailureReasons().size(), equalTo(0)); + assertThat(telemetry.getTook().count(), equalTo(1L)); + assertThat(telemetry.getTookMrtFalse().count(), equalTo(0L)); + assertThat(telemetry.getTookMrtTrue().count(), equalTo(0L)); + assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0)); + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L)); + assertThat(telemetry.getClientCounts().size(), equalTo(1)); + assertThat(telemetry.getClientCounts().get("kibana"), equalTo(1L)); + assertThat(telemetry.getFeatureCounts().get(ASYNC_FEATURE), equalTo(null)); + + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(2)); + for (String clusterAlias : remoteClusterAlias()) { + assertPerClusterCount(perCluster.get(clusterAlias), 1L); + } + assertThat(telemetry.getByRemoteCluster().size(), equalTo(2)); + } + + public void testAsync() throws Exception { + setupClusters(); + var telemetry = getTelemetryFromAsyncQuery("from logs-*,c*:logs-* | stats sum (v)"); + + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + assertThat(telemetry.getFailureReasons().size(), equalTo(0)); + assertThat(telemetry.getTook().count(), equalTo(1L)); + assertThat(telemetry.getTookMrtFalse().count(), equalTo(0L)); + assertThat(telemetry.getTookMrtTrue().count(), equalTo(0L)); + assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0)); + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L)); + assertThat(telemetry.getClientCounts().size(), equalTo(0)); + assertThat(telemetry.getFeatureCounts().get(ASYNC_FEATURE), equalTo(1L)); + + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(3)); + for (String clusterAlias : remoteClusterAlias()) { + assertPerClusterCount(perCluster.get(clusterAlias), 1L); + } + assertPerClusterCount(perCluster.get(LOCAL_CLUSTER), 1L); + + // do it again + telemetry = getTelemetryFromAsyncQuery("from logs-*,c*:logs-* | stats sum (v)"); + assertThat(telemetry.getTotalCount(), equalTo(2L)); + assertThat(telemetry.getFeatureCounts().get(ASYNC_FEATURE), equalTo(2L)); + perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(3)); + for (String clusterAlias : remoteClusterAlias()) { + assertPerClusterCount(perCluster.get(clusterAlias), 2L); + } + assertPerClusterCount(perCluster.get(LOCAL_CLUSTER), 2L); + } + + public void testNoSuchCluster() throws Exception { + setupClusters(); + // This is not recognized as a cross-cluster search + var telemetry = getTelemetryFromFailedQuery("from c*:logs*, nocluster:nomatch | stats sum (v)"); + + assertThat(telemetry.getTotalCount(), equalTo(0L)); + assertThat(telemetry.getSuccessCount(), equalTo(0L)); + assertThat(telemetry.getByRemoteCluster().size(), equalTo(0)); + } + + @SkipUnavailableRule.NotSkipped(aliases = REMOTE1) + public void testDisconnect() throws Exception { + setupClusters(); + // Disconnect remote1 + cluster(REMOTE1).close(); + var telemetry = getTelemetryFromFailedQuery("from logs-*,cluster-a:logs-* | stats sum (v)"); + + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(0L)); + Map expectedFailure = Map.of(CCSUsageTelemetry.Result.REMOTES_UNAVAILABLE.getName(), 1L); + assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure)); + } + +} diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersUsageTelemetryNoLicenseIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersUsageTelemetryNoLicenseIT.java new file mode 100644 index 0000000000000..2b993e9474062 --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersUsageTelemetryNoLicenseIT.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.action; + +import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry; +import org.elasticsearch.plugins.Plugin; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class CrossClustersUsageTelemetryNoLicenseIT extends AbstractCrossClustersUsageTelemetryIT { + + @Override + protected Collection> nodePlugins(String clusterAlias) { + List> plugins = new ArrayList<>(super.nodePlugins(clusterAlias)); + plugins.add(EsqlPluginWithNonEnterpriseOrExpiredLicense.class); + plugins.add(CrossClustersQueryIT.InternalExchangePlugin.class); + return plugins; + } + + public void testLicenseFailure() throws Exception { + setupClusters(); + var telemetry = getTelemetryFromFailedQuery("from logs-*,c*:logs-* | stats sum (v)"); + + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(0L)); + assertThat(telemetry.getTook().count(), equalTo(0L)); + assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0)); + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); + Map expectedFailure = Map.of(CCSUsageTelemetry.Result.LICENSE.getName(), 1L); + assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure)); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java index 9b21efc069e9f..c1afa728bc37b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java @@ -206,6 +206,10 @@ public Cluster getCluster(String clusterAlias) { return clusterInfo.get(clusterAlias); } + public Map getClusters() { + return clusterInfo; + } + /** * Utility to swap a Cluster object. Guidelines for the remapping function: *
    diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java index dad63d25046d9..974f029eab2ef 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java @@ -80,7 +80,8 @@ public void esql( ); QueryMetric clientId = QueryMetric.fromString("rest"); metrics.total(clientId); - session.execute(request, executionInfo, planRunner, wrap(x -> { + + ActionListener executeListener = wrap(x -> { planningMetricsManager.publish(planningMetrics, true); listener.onResponse(x); }, ex -> { @@ -88,7 +89,10 @@ public void esql( metrics.failed(clientId); planningMetricsManager.publish(planningMetrics, false); listener.onFailure(ex); - })); + }); + // Wrap it in a listener so that if we have any exceptions during execution, the listener picks it up + // and all the metrics are properly updated + ActionListener.run(executeListener, l -> session.execute(request, executionInfo, planRunner, l)); } public IndexResolver indexResolver() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java index f6c23304c189b..f596d589cdde2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java @@ -11,6 +11,8 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.Vector; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -50,6 +52,52 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; import static org.elasticsearch.xpack.esql.core.util.StringUtils.ordinal; +/** + * The {@code IN} operator. + *

    + * This function has quite "unique" null handling rules around {@code null} and multivalued + * fields. The {@code null} rules are inspired by PostgreSQL, and, presumably, every other + * SQL implementation. The multivalue rules are pretty much an extension of the "multivalued + * fields are like null in scalars" rule. Here's some examples: + *

    + *
      + *
    • {@code 'x' IN ('a', 'b', 'c')} => @{code false}
      • + *
    • {@code 'x' IN ('a', 'x', 'c')} => @{code true}
      • + *
    • {@code null IN ('a', 'b', 'c')} => @{code null}
      • + *
    • {@code ['x', 'y'] IN ('a', 'b', 'c')} => @{code null} and a warning
      • + *
    • {@code 'x' IN ('a', null, 'c')} => @{code null}
      • + *
    • {@code 'x' IN ('x', null, 'c')} => @{code true}
      • + *
    • {@code 'x' IN ('x', ['a', 'b'], 'c')} => @{code true} and a warning
      • + *
    • {@code 'x' IN ('a', ['a', 'b'], 'c')} => @{code false} and a warning
      • + *
    + *

    + * And here's the decision tree for {@code WHERE x IN (a, b, c)}: + *

    + *
      + *
    1. {@code x IS NULL} => return {@code null}
      2. + *
    2. {@code MV_COUNT(x) > 1} => emit a warning and return {@code null}
      3. + *
    3. {@code a IS NULL AND b IS NULL AND c IS NULL} => return {@code null}
      4. + *
    4. {@code MV_COUNT(a) > 1 OR MV_COUNT(b) > 1 OR MV_COUNT(c) > 1} => emit a warning and continue
      5. + *
    5. {@code MV_COUNT(a) > 1 AND MV_COUNT(b) > 1 AND MV_COUNT(c) > 1} => return {@code null}
      6. + *
    6. {@code x == a OR x == b OR x == c} => return {@code true}
      7. + *
    7. {@code a IS NULL OR b IS NULL OR c IS NULL} => return {@code null}
      8. + *
    8. {@code else} => {@code false}
      9. + *
    + *

    + * I believe the first five entries are *mostly* optimizations and making the + * Three-valued logic of SQL + * explicit and integrated with our multivalue field rules. And make all that work with the + * actual evaluator code. You could probably shorten this to the last three points, but lots + * of folks aren't familiar with SQL's three-valued logic anyway, so let's be explicit. + *

    + *

    + * Because of this chain of logic we don't use the standard evaluator generators. They'd just + * require too many special cases and nothing else quite works like this. I mean, everything + * works just like this in that "three-valued logic" sort of way, but not in the "java code" + * sort of way. So! Instead of using the standard evaluator generators we use the + * String Template generators that we use for things like {@link Block} and {@link Vector}. + *

    + */ public class In extends EsqlScalarFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "In", In::new); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java index d3fc9e15e2e04..1edab8ce0e4a6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java @@ -15,15 +15,16 @@ import java.io.IOException; import java.util.Map; -import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; -public class EsIndex implements Writeable { +import static java.util.stream.Collectors.toMap; - private final String name; - private final Map mapping; - private final Map indexNameWithModes; +public record EsIndex(String name, Map mapping, Map indexNameWithModes) implements Writeable { + + public EsIndex { + assert name != null; + assert mapping != null; + } /** * Intended for tests. Returns an index with an empty index mode map. @@ -32,56 +33,32 @@ public EsIndex(String name, Map mapping) { this(name, mapping, Map.of()); } - public EsIndex(String name, Map mapping, Map indexNameWithModes) { - assert name != null; - assert mapping != null; - this.name = name; - this.mapping = mapping; - this.indexNameWithModes = indexNameWithModes; - } - - public EsIndex(StreamInput in) throws IOException { - this(in.readString(), in.readImmutableMap(StreamInput::readString, EsField::readFrom), readIndexNameWithModes(in)); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(name()); - out.writeMap(mapping(), (o, x) -> x.writeTo(out)); - writeIndexNameWithModes(indexNameWithModes, out); - } - - @SuppressWarnings("unchecked") - private static Map readIndexNameWithModes(StreamInput in) throws IOException { + public static EsIndex readFrom(StreamInput in) throws IOException { + String name = in.readString(); + Map mapping = in.readImmutableMap(StreamInput::readString, EsField::readFrom); + Map indexNameWithModes; if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { - return in.readMap(IndexMode::readFrom); + indexNameWithModes = in.readMap(IndexMode::readFrom); } else { + @SuppressWarnings("unchecked") Set indices = (Set) in.readGenericValue(); assert indices != null; - return indices.stream().collect(Collectors.toMap(e -> e, e -> IndexMode.STANDARD)); + indexNameWithModes = indices.stream().collect(toMap(e -> e, e -> IndexMode.STANDARD)); } + return new EsIndex(name, mapping, indexNameWithModes); } - private static void writeIndexNameWithModes(Map concreteIndices, StreamOutput out) throws IOException { + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name()); + out.writeMap(mapping(), (o, x) -> x.writeTo(out)); if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { - out.writeMap(concreteIndices, (o, v) -> IndexMode.writeTo(v, out)); + out.writeMap(indexNameWithModes, (o, v) -> IndexMode.writeTo(v, out)); } else { - out.writeGenericValue(concreteIndices.keySet()); + out.writeGenericValue(indexNameWithModes.keySet()); } } - public String name() { - return name; - } - - public Map mapping() { - return mapping; - } - - public Map indexNameWithModes() { - return indexNameWithModes; - } - public Set concreteIndices() { return indexNameWithModes.keySet(); } @@ -90,25 +67,4 @@ public Set concreteIndices() { public String toString() { return name; } - - @Override - public int hashCode() { - return Objects.hash(name, mapping, indexNameWithModes); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - - if (obj == null || getClass() != obj.getClass()) { - return false; - } - - EsIndex other = (EsIndex) obj; - return Objects.equals(name, other.name) - && Objects.equals(mapping, other.mapping) - && Objects.equals(indexNameWithModes, other.indexNameWithModes); - } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java index fcb5d9dbd61cf..b46a563867367 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java @@ -116,7 +116,7 @@ private static Enrich readFrom(StreamInput in) throws IOException { if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) { concreteIndices = in.readMap(StreamInput::readString, StreamInput::readString); } else { - EsIndex esIndex = new EsIndex(in); + EsIndex esIndex = EsIndex.readFrom(in); if (esIndex.concreteIndices().size() > 1) { throw new IllegalStateException("expected a single enrich index; got " + esIndex); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java index 794a52b8e3f89..df0b258679d4c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java @@ -58,7 +58,7 @@ public EsRelation(Source source, EsIndex index, List attributes, Inde private static EsRelation readFrom(StreamInput in) throws IOException { Source source = Source.readFrom((PlanStreamInput) in); - EsIndex esIndex = new EsIndex(in); + EsIndex esIndex = EsIndex.readFrom(in); List attributes = in.readNamedWriteableCollectionAsList(Attribute.class); if (supportingEsSourceOptions(in.getTransportVersion())) { // We don't do anything with these strings diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EnrichExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EnrichExec.java index a14332ebef7c3..42cf3528f2ae6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EnrichExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EnrichExec.java @@ -87,7 +87,7 @@ private static EnrichExec readFrom(StreamInput in) throws IOException { concreteIndices = in.readMap(StreamInput::readString, StreamInput::readString); } else { mode = Enrich.Mode.ANY; - EsIndex esIndex = new EsIndex(in); + EsIndex esIndex = EsIndex.readFrom(in); if (esIndex.concreteIndices().size() != 1) { throw new IllegalStateException("expected a single concrete enrich index; got " + esIndex.concreteIndices()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java index 267b9e613abef..ab533899aaff6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java @@ -138,7 +138,7 @@ public EsQueryExec( */ public static EsQueryExec deserialize(StreamInput in) throws IOException { var source = Source.readFrom((PlanStreamInput) in); - var index = new EsIndex(in); + var index = EsIndex.readFrom(in); var indexMode = EsRelation.readIndexMode(in); var attrs = in.readNamedWriteableCollectionAsList(Attribute.class); var query = in.readOptionalNamedWriteable(QueryBuilder.class); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java index cd167b4683493..eeeafc52f158b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java @@ -51,7 +51,7 @@ public EsSourceExec(Source source, EsIndex index, List attributes, Qu private EsSourceExec(StreamInput in) throws IOException { this( Source.readFrom((PlanStreamInput) in), - new EsIndex(in), + EsIndex.readFrom(in), in.readNamedWriteableCollectionAsList(Attribute.class), in.readOptionalNamedWriteable(QueryBuilder.class), EsRelation.readIndexMode(in) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EstimatesRowSize.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EstimatesRowSize.java index cfb6cce2579a2..f4ab546fa7493 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EstimatesRowSize.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EstimatesRowSize.java @@ -103,9 +103,6 @@ public String toString() { static int estimateSize(DataType dataType) { ElementType elementType = PlannerUtils.toElementType(dataType); - if (elementType == ElementType.DOC) { - throw new EsqlIllegalArgumentException("can't load a [doc] with field extraction"); - } if (elementType == ElementType.UNKNOWN) { throw new EsqlIllegalArgumentException("[unknown] can't be the result of field extraction"); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/TopNExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/TopNExec.java index 61e40b3fa4693..bbf0c681bec7d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/TopNExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/TopNExec.java @@ -10,9 +10,11 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.Order; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; @@ -95,6 +97,9 @@ public Integer estimatedRowSize() { @Override public PhysicalPlan estimateRowSize(State state) { + final List output = output(); + final boolean needsSortedDocIds = output.stream().anyMatch(a -> a.dataType() == DataType.DOC_DATA_TYPE); + state.add(needsSortedDocIds, output); int size = state.consumeAllFields(true); return Objects.equals(this.estimatedRowSize, size) ? this : new TopNExec(source(), child(), order, limit, size); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 6d9cf38d34517..b1fe0e7a7cf54 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -13,22 +13,16 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; -import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.common.breaker.NoopCircuitBreaker; import org.elasticsearch.common.logging.HeaderWarning; -import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.compute.aggregation.GroupingAggregator; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.ElementType; -import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.lucene.LuceneCountOperator; import org.elasticsearch.compute.lucene.LuceneOperator; import org.elasticsearch.compute.lucene.LuceneSourceOperator; import org.elasticsearch.compute.lucene.LuceneTopNSourceOperator; import org.elasticsearch.compute.lucene.TimeSeriesSortedSourceOperatorFactory; import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator; -import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.Operator; import org.elasticsearch.compute.operator.OrdinalsGroupingOperator; import org.elasticsearch.compute.operator.SourceOperator; @@ -380,29 +374,13 @@ public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { } } - static class TypeConvertingBlockLoader implements BlockLoader { - protected final BlockLoader delegate; - private final EvalOperator.ExpressionEvaluator convertEvaluator; + private static class TypeConvertingBlockLoader implements BlockLoader { + private final BlockLoader delegate; + private final TypeConverter typeConverter; protected TypeConvertingBlockLoader(BlockLoader delegate, AbstractConvertFunction convertFunction) { this.delegate = delegate; - DriverContext driverContext1 = new DriverContext( - BigArrays.NON_RECYCLING_INSTANCE, - new org.elasticsearch.compute.data.BlockFactory( - new NoopCircuitBreaker(CircuitBreaker.REQUEST), - BigArrays.NON_RECYCLING_INSTANCE - ) - ); - this.convertEvaluator = convertFunction.toEvaluator(e -> driverContext -> new EvalOperator.ExpressionEvaluator() { - @Override - public org.elasticsearch.compute.data.Block eval(Page page) { - // This is a pass-through evaluator, since it sits directly on the source loading (no prior expressions) - return page.getBlock(0); - } - - @Override - public void close() {} - }).get(driverContext1); + this.typeConverter = TypeConverter.fromConvertFunction(convertFunction); } @Override @@ -413,8 +391,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { @Override public Block convert(Block block) { - Page page = new Page((org.elasticsearch.compute.data.Block) block); - return convertEvaluator.eval(page); + return typeConverter.convert((org.elasticsearch.compute.data.Block) block); } @Override @@ -427,9 +404,7 @@ public ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) throws @Override public Block read(BlockFactory factory, Docs docs) throws IOException { Block block = reader.read(factory, docs); - Page page = new Page((org.elasticsearch.compute.data.Block) block); - org.elasticsearch.compute.data.Block converted = convertEvaluator.eval(page); - return converted; + return typeConverter.convert((org.elasticsearch.compute.data.Block) block); } @Override @@ -469,7 +444,7 @@ public SortedSetDocValues ordinals(LeafReaderContext context) { @Override public final String toString() { - return "TypeConvertingBlockLoader[delegate=" + delegate + ", convertEvaluator=" + convertEvaluator + "]"; + return "TypeConvertingBlockLoader[delegate=" + delegate + ", typeConverter=" + typeConverter + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index c40263baa6566..af38551c1ad06 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -345,6 +345,8 @@ private PhysicalOperation planExchangeSource(ExchangeSourceExec exchangeSource, } private PhysicalOperation planTopN(TopNExec topNExec, LocalExecutionPlannerContext context) { + final Integer rowSize = topNExec.estimatedRowSize(); + assert rowSize != null && rowSize > 0 : "estimated row size [" + rowSize + "] wasn't set"; PhysicalOperation source = plan(topNExec.child(), context); ElementType[] elementTypes = new ElementType[source.layout.numberOfChannels()]; @@ -385,24 +387,8 @@ private PhysicalOperation planTopN(TopNExec topNExec, LocalExecutionPlannerConte } else { throw new EsqlIllegalArgumentException("limit only supported with literal values"); } - - // TODO Replace page size with passing estimatedRowSize down - /* - * The 2000 below is a hack to account for incoming size and to make - * sure the estimated row size is never 0 which'd cause a divide by 0. - * But we should replace this with passing the estimate into the real - * topn and letting it actually measure the size of rows it produces. - * That'll be more accurate. And we don't have a path for estimating - * incoming rows. And we don't need one because we can estimate. - */ return source.with( - new TopNOperatorFactory( - limit, - asList(elementTypes), - asList(encoders), - orders, - context.pageSize(2000 + topNExec.estimatedRowSize()) - ), + new TopNOperatorFactory(limit, asList(elementTypes), asList(encoders), orders, context.pageSize(rowSize)), source.layout ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TypeConverter.java new file mode 100644 index 0000000000000..334875927eb96 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TypeConverter.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.planner; + +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.breaker.NoopCircuitBreaker; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +class TypeConverter { + private final String evaluatorName; + private final ExpressionEvaluator convertEvaluator; + + private TypeConverter(String evaluatorName, ExpressionEvaluator convertEvaluator) { + this.evaluatorName = evaluatorName; + this.convertEvaluator = convertEvaluator; + } + + public static TypeConverter fromConvertFunction(AbstractConvertFunction convertFunction) { + DriverContext driverContext1 = new DriverContext( + BigArrays.NON_RECYCLING_INSTANCE, + new org.elasticsearch.compute.data.BlockFactory( + new NoopCircuitBreaker(CircuitBreaker.REQUEST), + BigArrays.NON_RECYCLING_INSTANCE + ) + ); + return new TypeConverter( + convertFunction.functionName(), + convertFunction.toEvaluator(e -> driverContext -> new ExpressionEvaluator() { + @Override + public org.elasticsearch.compute.data.Block eval(Page page) { + // This is a pass-through evaluator, since it sits directly on the source loading (no prior expressions) + return page.getBlock(0); + } + + @Override + public void close() {} + }).get(driverContext1) + ); + } + + public Block convert(Block block) { + return convertEvaluator.eval(new Page(block)); + } + + @Override + public String toString() { + return evaluatorName; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java index 50d5819688e46..b44e249e38006 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java @@ -9,6 +9,8 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; +import org.elasticsearch.action.admin.cluster.stats.CCSUsage; +import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.internal.Client; @@ -20,16 +22,20 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.operator.exchange.ExchangeService; +import org.elasticsearch.core.Nullable; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.search.SearchService; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.transport.RemoteClusterService; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.usage.UsageService; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.async.AsyncExecutionId; +import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.ColumnInfoImpl; import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo; import org.elasticsearch.xpack.esql.action.EsqlQueryAction; @@ -52,6 +58,7 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; @@ -71,6 +78,7 @@ public class TransportEsqlQueryAction extends HandledTransportAction asyncTaskManagementService; private final RemoteClusterService remoteClusterService; private final QueryBuilderResolver queryBuilderResolver; + private final UsageService usageService; @Inject @SuppressWarnings("this-escape") @@ -86,8 +94,8 @@ public TransportEsqlQueryAction( BlockFactory blockFactory, Client client, NamedWriteableRegistry registry, - IndexNameExpressionResolver indexNameExpressionResolver - + IndexNameExpressionResolver indexNameExpressionResolver, + UsageService usageService ) { // TODO replace SAME when removing workaround for https://github.com/elastic/elasticsearch/issues/97916 super(EsqlQueryAction.NAME, transportService, actionFilters, EsqlQueryRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); @@ -126,6 +134,7 @@ public TransportEsqlQueryAction( ); this.remoteClusterService = transportService.getRemoteClusterService(); this.queryBuilderResolver = new QueryBuilderResolver(searchService, clusterService, transportService, indexNameExpressionResolver); + this.usageService = usageService; } @Override @@ -197,8 +206,65 @@ private void innerExecute(Task task, EsqlQueryRequest request, ActionListener toResponse(task, request, configuration, result)) + ActionListener.wrap(result -> { + recordCCSTelemetry(task, executionInfo, request, null); + listener.onResponse(toResponse(task, request, configuration, result)); + }, ex -> { + recordCCSTelemetry(task, executionInfo, request, ex); + listener.onFailure(ex); + }) ); + + } + + private void recordCCSTelemetry(Task task, EsqlExecutionInfo executionInfo, EsqlQueryRequest request, @Nullable Exception exception) { + if (executionInfo.isCrossClusterSearch() == false) { + return; + } + + CCSUsage.Builder usageBuilder = new CCSUsage.Builder(); + usageBuilder.setClientFromTask(task); + if (exception != null) { + if (exception instanceof VerificationException ve) { + CCSUsageTelemetry.Result failureType = classifyVerificationException(ve); + if (failureType != CCSUsageTelemetry.Result.UNKNOWN) { + usageBuilder.setFailure(failureType); + } else { + usageBuilder.setFailure(exception); + } + } else { + usageBuilder.setFailure(exception); + } + } + var took = executionInfo.overallTook(); + if (took != null) { + usageBuilder.took(took.getMillis()); + } + if (request.async()) { + usageBuilder.setFeature(CCSUsageTelemetry.ASYNC_FEATURE); + } + + AtomicInteger remotesCount = new AtomicInteger(); + executionInfo.getClusters().forEach((clusterAlias, cluster) -> { + if (cluster.getStatus() == EsqlExecutionInfo.Cluster.Status.SKIPPED) { + usageBuilder.skippedRemote(clusterAlias); + } else { + usageBuilder.perClusterUsage(clusterAlias, cluster.getTook()); + } + if (clusterAlias.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) == false) { + remotesCount.getAndIncrement(); + } + }); + assert remotesCount.get() > 0 : "Got cross-cluster search telemetry without any remote clusters"; + usageBuilder.setRemotesCount(remotesCount.get()); + usageService.getEsqlUsageHolder().updateUsage(usageBuilder.build()); + } + + private CCSUsageTelemetry.Result classifyVerificationException(VerificationException exception) { + if (exception.getDetailedMessage().contains("Unknown index")) { + return CCSUsageTelemetry.Result.NOT_FOUND; + } + return CCSUsageTelemetry.Result.UNKNOWN; } private EsqlExecutionInfo getOrCreateExecutionInfo(Task task, EsqlQueryRequest request) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index bd3b3bdb3483c..eb5e8206e9e6f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -312,7 +312,7 @@ public void analyzedPlan( .collect(Collectors.toSet()); final List indices = preAnalysis.indices; - EsqlSessionCCSUtils.checkForCcsLicense(indices, indicesExpressionGrouper, verifier.licenseState()); + EsqlSessionCCSUtils.checkForCcsLicense(executionInfo, indices, indicesExpressionGrouper, verifier.licenseState()); final Set targetClusters = enrichPolicyResolver.groupIndicesPerCluster( indices.stream().flatMap(t -> Arrays.stream(Strings.commaDelimitedListToStringArray(t.id().index()))).toArray(String[]::new) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java index 662572c466511..95f7a37ce4d62 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java @@ -308,6 +308,7 @@ static void updateExecutionInfoAtEndOfPlanning(EsqlExecutionInfo execInfo) { * @throws org.elasticsearch.ElasticsearchStatusException if the license is not valid (or present) for ES|QL CCS search. */ public static void checkForCcsLicense( + EsqlExecutionInfo executionInfo, List indices, IndicesExpressionGrouper indicesGrouper, XPackLicenseState licenseState @@ -326,6 +327,17 @@ public static void checkForCcsLicense( // check if it is a cross-cluster query if (groupedIndices.size() > 1 || groupedIndices.containsKey(RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY) == false) { if (EsqlLicenseChecker.isCcsAllowed(licenseState) == false) { + // initialize the cluster entries in EsqlExecutionInfo before throwing the invalid license error + // so that the CCS telemetry handler can recognize that this error is CCS-related + for (Map.Entry entry : groupedIndices.entrySet()) { + executionInfo.swapCluster( + entry.getKey(), + (k, v) -> new EsqlExecutionInfo.Cluster( + entry.getKey(), + Strings.arrayToCommaDelimitedString(entry.getValue().indices()) + ) + ); + } throw EsqlLicenseChecker.invalidLicenseForCcsException(licenseState); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index e78d42db11d25..fe9a5e569669d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -57,6 +57,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; import org.elasticsearch.xpack.esql.enrich.EnrichLookupService; import org.elasticsearch.xpack.esql.enrich.LookupFromIndexService; import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; @@ -97,11 +98,15 @@ import java.net.URL; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.elasticsearch.xpack.esql.CsvSpecReader.specParser; import static org.elasticsearch.xpack.esql.CsvTestUtils.ExpectedResults; @@ -245,10 +250,6 @@ public final void test() throws Throwable { testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.MATCH_OPERATOR_COLON.capabilityName()) ); assumeFalse("can't load metrics in csv tests", testCase.requiredCapabilities.contains(cap(EsqlFeatures.METRICS_SYNTAX))); - assumeFalse( - "multiple indices aren't supported", - testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.UNION_TYPES.capabilityName()) - ); assumeFalse( "can't use QSTR function in csv tests", testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.QSTR_FUNCTION.capabilityName()) @@ -317,7 +318,13 @@ private void doTest() throws Exception { } finally { Releasables.close(() -> Iterators.map(actualResults.pages().iterator(), p -> p::releaseBlocks)); // Give the breaker service some time to clear in case we got results before the rest of the driver had cleaned up - assertBusy(() -> assertThat(bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L))); + assertBusy( + () -> assertThat( + "Not all circuits were cleaned up", + bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST).getUsed(), + equalTo(0L) + ) + ); } } @@ -332,29 +339,71 @@ protected void assertResults(ExpectedResults expected, ActualResults actual, boo CsvAssert.assertResults(expected, actual, ignoreOrder, logger); } - private static IndexResolution loadIndexResolution(String mappingName, String indexName, Map typeMapping) { - var mapping = new TreeMap<>(loadMapping(mappingName)); - if ((typeMapping == null || typeMapping.isEmpty()) == false) { - for (var entry : typeMapping.entrySet()) { - if (mapping.containsKey(entry.getKey())) { - DataType dataType = DataType.fromTypeName(entry.getValue()); - EsField field = mapping.get(entry.getKey()); - EsField editedField = new EsField(field.getName(), dataType, field.getProperties(), field.isAggregatable()); - mapping.put(entry.getKey(), editedField); - } + private static IndexResolution loadIndexResolution(CsvTestsDataLoader.MultiIndexTestDataset datasets) { + var indexNames = datasets.datasets().stream().map(CsvTestsDataLoader.TestDataset::indexName); + Map indexModes = indexNames.collect(Collectors.toMap(x -> x, x -> IndexMode.STANDARD)); + List mappings = datasets.datasets() + .stream() + .map(ds -> new MappingPerIndex(ds.indexName(), createMappingForIndex(ds))) + .toList(); + return IndexResolution.valid(new EsIndex(datasets.indexPattern(), mergeMappings(mappings), indexModes)); + } + + private static Map createMappingForIndex(CsvTestsDataLoader.TestDataset dataset) { + var mapping = new TreeMap<>(loadMapping(dataset.mappingFileName())); + if (dataset.typeMapping() == null) { + return mapping; + } + for (var entry : dataset.typeMapping().entrySet()) { + if (mapping.containsKey(entry.getKey())) { + DataType dataType = DataType.fromTypeName(entry.getValue()); + EsField field = mapping.get(entry.getKey()); + EsField editedField = new EsField(field.getName(), dataType, field.getProperties(), field.isAggregatable()); + mapping.put(entry.getKey(), editedField); + } + } + return mapping; + } + + record MappingPerIndex(String index, Map mapping) {} + + private static Map mergeMappings(List mappingsPerIndex) { + Map> columnNamesToFieldByIndices = new HashMap<>(); + for (var mappingPerIndex : mappingsPerIndex) { + for (var entry : mappingPerIndex.mapping().entrySet()) { + String columnName = entry.getKey(); + EsField field = entry.getValue(); + columnNamesToFieldByIndices.computeIfAbsent(columnName, k -> new HashMap<>()).put(mappingPerIndex.index(), field); } } - return IndexResolution.valid(new EsIndex(indexName, mapping, Map.of(indexName, IndexMode.STANDARD))); + + return columnNamesToFieldByIndices.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> mergeFields(e.getKey(), e.getValue()))); + } + + private static EsField mergeFields(String index, Map columnNameToField) { + var indexFields = columnNameToField.values(); + if (indexFields.stream().distinct().count() > 1) { + var typesToIndices = new HashMap>(); + for (var typeToIndex : columnNameToField.entrySet()) { + typesToIndices.computeIfAbsent(typeToIndex.getValue().getDataType().typeName(), k -> new HashSet<>()) + .add(typeToIndex.getKey()); + } + return new InvalidMappedField(index, typesToIndices); + } else { + return indexFields.iterator().next(); + } } private static EnrichResolution loadEnrichPolicies() { EnrichResolution enrichResolution = new EnrichResolution(); for (CsvTestsDataLoader.EnrichConfig policyConfig : CsvTestsDataLoader.ENRICH_POLICIES) { EnrichPolicy policy = loadEnrichPolicyMapping(policyConfig.policyFileName()); - CsvTestsDataLoader.TestsDataset sourceIndex = CSV_DATASET_MAP.get(policy.getIndices().get(0)); + CsvTestsDataLoader.TestDataset sourceIndex = CSV_DATASET_MAP.get(policy.getIndices().get(0)); // this could practically work, but it's wrong: // EnrichPolicyResolution should contain the policy (system) index, not the source index - EsIndex esIndex = loadIndexResolution(sourceIndex.mappingFileName(), sourceIndex.indexName(), null).get(); + EsIndex esIndex = loadIndexResolution(CsvTestsDataLoader.MultiIndexTestDataset.of(sourceIndex.withTypeMapping(Map.of()))).get(); var concreteIndices = Map.of(RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY, Iterables.get(esIndex.concreteIndices(), 0)); enrichResolution.addResolvedPolicy( policyConfig.policyName(), @@ -382,8 +431,8 @@ private static EnrichPolicy loadEnrichPolicyMapping(String policyFileName) { } } - private LogicalPlan analyzedPlan(LogicalPlan parsed, CsvTestsDataLoader.TestsDataset dataset) { - var indexResolution = loadIndexResolution(dataset.mappingFileName(), dataset.indexName(), dataset.typeMapping()); + private LogicalPlan analyzedPlan(LogicalPlan parsed, CsvTestsDataLoader.MultiIndexTestDataset datasets) { + var indexResolution = loadIndexResolution(datasets); var enrichPolicies = loadEnrichPolicies(); var analyzer = new Analyzer(new AnalyzerContext(configuration, functionRegistry, indexResolution, enrichPolicies), TEST_VERIFIER); LogicalPlan plan = analyzer.analyze(parsed); @@ -392,7 +441,7 @@ private LogicalPlan analyzedPlan(LogicalPlan parsed, CsvTestsDataLoader.TestsDat return plan; } - private static CsvTestsDataLoader.TestsDataset testsDataset(LogicalPlan parsed) { + private static CsvTestsDataLoader.MultiIndexTestDataset testDatasets(LogicalPlan parsed) { var preAnalysis = new PreAnalyzer().preAnalyze(parsed); var indices = preAnalysis.indices; if (indices.isEmpty()) { @@ -400,13 +449,13 @@ private static CsvTestsDataLoader.TestsDataset testsDataset(LogicalPlan parsed) * If the data set doesn't matter we'll just grab one we know works. * Employees is fine. */ - return CSV_DATASET_MAP.get("employees"); + return CsvTestsDataLoader.MultiIndexTestDataset.of(CSV_DATASET_MAP.get("employees")); } else if (preAnalysis.indices.size() > 1) { throw new IllegalArgumentException("unexpected index resolution to multiple entries [" + preAnalysis.indices.size() + "]"); } String indexName = indices.get(0).id().index(); - List datasets = new ArrayList<>(); + List datasets = new ArrayList<>(); if (indexName.endsWith("*")) { String indexPrefix = indexName.substring(0, indexName.length() - 1); for (var entry : CSV_DATASET_MAP.entrySet()) { @@ -415,25 +464,35 @@ private static CsvTestsDataLoader.TestsDataset testsDataset(LogicalPlan parsed) } } } else { - var dataset = CSV_DATASET_MAP.get(indexName); - datasets.add(dataset); + for (String index : indexName.split(",")) { + var dataset = CSV_DATASET_MAP.get(index); + if (dataset == null) { + throw new IllegalArgumentException("unknown CSV dataset for table [" + index + "]"); + } + datasets.add(dataset); + } } if (datasets.isEmpty()) { throw new IllegalArgumentException("unknown CSV dataset for table [" + indexName + "]"); } - // TODO: Support multiple datasets - return datasets.get(0); + return new CsvTestsDataLoader.MultiIndexTestDataset(indexName, datasets); } - private static TestPhysicalOperationProviders testOperationProviders(CsvTestsDataLoader.TestsDataset dataset) throws Exception { - var testData = loadPageFromCsv(CsvTests.class.getResource("/data/" + dataset.dataFileName()), dataset.typeMapping()); - return new TestPhysicalOperationProviders(testData.v1(), testData.v2()); + private static TestPhysicalOperationProviders testOperationProviders(CsvTestsDataLoader.MultiIndexTestDataset datasets) + throws Exception { + var indexResolution = loadIndexResolution(datasets); + var indexPages = new ArrayList(); + for (CsvTestsDataLoader.TestDataset dataset : datasets.datasets()) { + var testData = loadPageFromCsv(CsvTests.class.getResource("/data/" + dataset.dataFileName()), dataset.typeMapping()); + indexPages.add(new TestPhysicalOperationProviders.IndexPage(dataset.indexName(), testData.v1(), testData.v2())); + } + return TestPhysicalOperationProviders.create(indexPages); } private ActualResults executePlan(BigArrays bigArrays) throws Exception { LogicalPlan parsed = parser.createStatement(testCase.query); - var testDataset = testsDataset(parsed); - LogicalPlan analyzed = analyzedPlan(parsed, testDataset); + var testDatasets = testDatasets(parsed); + LogicalPlan analyzed = analyzedPlan(parsed, testDatasets); EsqlSession session = new EsqlSession( getTestName(), @@ -449,7 +508,7 @@ private ActualResults executePlan(BigArrays bigArrays) throws Exception { null, EsqlTestUtils.MOCK_QUERY_BUILDER_RESOLVER ); - TestPhysicalOperationProviders physicalOperationProviders = testOperationProviders(testDataset); + TestPhysicalOperationProviders physicalOperationProviders = testOperationProviders(testDatasets); PlainActionFuture listener = new PlainActionFuture<>(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java index 82dd5a88ffaf1..8f846edf2b41c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java @@ -56,7 +56,7 @@ private static Map randomConcreteIndices() { @Override protected Writeable.Reader instanceReader() { - return a -> new EsIndex(new PlanStreamInput(a, a.namedWriteableRegistry(), null)); + return a -> EsIndex.readFrom(new PlanStreamInput(a, a.namedWriteableRegistry(), null)); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 591ceff7120e8..ff710a90e8154 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -143,6 +143,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -920,8 +921,8 @@ public void testQueryWithNull() { var optimized = optimizedPlan(plan); var topN = as(optimized, TopNExec.class); - // no fields are added after the top n - so 0 here - assertThat(topN.estimatedRowSize(), equalTo(0)); + // all fields + nullsum are loaded in the final TopN + assertThat(topN.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES)); var exchange = asRemoteExchange(topN.child()); var project = as(exchange.child(), ProjectExec.class); @@ -929,7 +930,7 @@ public void testQueryWithNull() { var eval = as(extract.child(), EvalExec.class); var source = source(eval.child()); // All fields loaded - assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + 3 * Integer.BYTES + Long.BYTES)); + assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + 3 * Integer.BYTES + 2 * Integer.BYTES)); } public void testPushAndInequalitiesFilter() { @@ -1141,8 +1142,8 @@ public void testExtractorForEvalWithoutProject() throws Exception { var project = as(exchange.child(), ProjectExec.class); var extract = as(project.child(), FieldExtractExec.class); var topNLocal = as(extract.child(), TopNExec.class); - // two extra ints for forwards and backwards map - assertThat(topNLocal.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES * 2)); + // all fields plus nullsum and shards, segments, docs and two extra ints for forwards and backwards map + assertThat(topNLocal.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES + Integer.BYTES * 2 + Integer.BYTES * 3)); var eval = as(topNLocal.child(), EvalExec.class); var source = source(eval.child()); @@ -7650,6 +7651,31 @@ public void testScoreTopN() { assertTrue(esRelation.output().stream().anyMatch(a -> a.name().equals(MetadataAttribute.SCORE) && a instanceof MetadataAttribute)); } + public void testReductionPlanForTopN() { + int limit = between(1, 100); + var plan = physicalPlan(String.format(Locale.ROOT, """ + FROM test + | sort emp_no + | LIMIT %d + """, limit)); + Tuple plans = PlannerUtils.breakPlanBetweenCoordinatorAndDataNode(plan, config); + PhysicalPlan reduction = PlannerUtils.reductionPlan(plans.v2()); + TopNExec reductionTopN = as(reduction, TopNExec.class); + assertThat(reductionTopN.estimatedRowSize(), equalTo(allFieldRowSize)); + assertThat(reductionTopN.limit().fold(), equalTo(limit)); + } + + public void testReductionPlanForAggs() { + var plan = physicalPlan(""" + FROM test + | stats x = sum(salary) BY first_name + """); + Tuple plans = PlannerUtils.breakPlanBetweenCoordinatorAndDataNode(plan, config); + PhysicalPlan reduction = PlannerUtils.reductionPlan(plans.v2()); + AggregateExec reductionAggs = as(reduction, AggregateExec.class); + assertThat(reductionAggs.estimatedRowSize(), equalTo(58)); // double and keyword + } + @SuppressWarnings("SameParameterValue") private static void assertFilterCondition( Filter filter, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java index 78512636b57e9..01dd4db123ee2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java @@ -30,16 +30,22 @@ import org.elasticsearch.compute.operator.OrdinalsGroupingOperator; import org.elasticsearch.compute.operator.SourceOperator; import org.elasticsearch.compute.operator.SourceOperator.SourceOperatorFactory; +import org.elasticsearch.core.Nullable; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; -import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.plugins.scanners.StablePluginsRegistry; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.TestBlockFactory; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes; +import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec; import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec; @@ -48,8 +54,12 @@ import org.elasticsearch.xpack.ml.MachineLearning; import java.io.IOException; +import java.util.ArrayList; import java.util.List; +import java.util.OptionalInt; import java.util.Random; +import java.util.function.BiFunction; +import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.IntStream; @@ -57,26 +67,33 @@ import static com.carrotsearch.randomizedtesting.generators.RandomNumbers.randomIntBetween; import static java.util.stream.Collectors.joining; import static org.apache.lucene.tests.util.LuceneTestCase.createTempDir; -import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.DOC_VALUES; -import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.NONE; public class TestPhysicalOperationProviders extends AbstractPhysicalOperationProviders { + private final List indexPages; - private final Page testData; - private final List columnNames; + private TestPhysicalOperationProviders(List indexPages, AnalysisRegistry analysisRegistry) { + super(analysisRegistry); + this.indexPages = indexPages; + } - public TestPhysicalOperationProviders(Page testData, List columnNames) throws IOException { - super( - new AnalysisModule( - TestEnvironment.newEnvironment( - Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build() - ), - List.of(new MachineLearning(Settings.EMPTY), new CommonAnalysisPlugin()), - new StablePluginsRegistry() - ).getAnalysisRegistry() - ); - this.testData = testData; - this.columnNames = columnNames; + public static TestPhysicalOperationProviders create(List indexPages) throws IOException { + return new TestPhysicalOperationProviders(indexPages, createAnalysisRegistry()); + } + + public record IndexPage(String index, Page page, List columnNames) { + OptionalInt columnIndex(String columnName) { + return IntStream.range(0, columnNames.size()).filter(i -> columnNames.get(i).equals(columnName)).findFirst(); + } + } + + private static AnalysisRegistry createAnalysisRegistry() throws IOException { + return new AnalysisModule( + TestEnvironment.newEnvironment( + Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build() + ), + List.of(new MachineLearning(Settings.EMPTY), new CommonAnalysisPlugin()), + new StablePluginsRegistry() + ).getAnalysisRegistry(); } @Override @@ -118,13 +135,12 @@ public Operator.OperatorFactory ordinalGroupingOperatorFactory( aggregatorFactories, groupElementType, context.bigArrays(), - attrSource.name() + attrSource ); } private class TestSourceOperator extends SourceOperator { - - boolean finished = false; + private int index = 0; private final DriverContext driverContext; TestSourceOperator(DriverContext driverContext) { @@ -133,28 +149,29 @@ private class TestSourceOperator extends SourceOperator { @Override public Page getOutput() { - if (finished == false) { - finish(); - } - + var pageIndex = indexPages.get(index); + var page = pageIndex.page; BlockFactory blockFactory = driverContext.blockFactory(); DocVector docVector = new DocVector( - blockFactory.newConstantIntVector(0, testData.getPositionCount()), - blockFactory.newConstantIntVector(0, testData.getPositionCount()), - blockFactory.newIntArrayVector(IntStream.range(0, testData.getPositionCount()).toArray(), testData.getPositionCount()), + // The shard ID is used to encode the index ID. + blockFactory.newConstantIntVector(index, page.getPositionCount()), + blockFactory.newConstantIntVector(0, page.getPositionCount()), + blockFactory.newIntArrayVector(IntStream.range(0, page.getPositionCount()).toArray(), page.getPositionCount()), true ); - return new Page(docVector.asBlock()); + var block = docVector.asBlock(); + index++; + return new Page(block); } @Override public boolean isFinished() { - return finished; + return index == indexPages.size(); } @Override public void finish() { - finished = true; + index = indexPages.size(); } @Override @@ -177,24 +194,19 @@ public String describe() { } private class TestFieldExtractOperator implements Operator { - + private final Attribute attribute; private Page lastPage; boolean finished; - String columnName; - private final DataType dataType; - private final MappedFieldType.FieldExtractPreference extractPreference; - - TestFieldExtractOperator(String columnName, DataType dataType, MappedFieldType.FieldExtractPreference extractPreference) { - assert columnNames.contains(columnName); - this.columnName = columnName; - this.dataType = dataType; + private final FieldExtractPreference extractPreference; + + TestFieldExtractOperator(Attribute attr, FieldExtractPreference extractPreference) { + this.attribute = attr; this.extractPreference = extractPreference; } @Override public void addInput(Page page) { - Block block = extractBlockForColumn(page, columnName, dataType, extractPreference); - lastPage = page.appendBlock(block); + lastPage = page.appendBlock(getBlock(page.getBlock(0), attribute, extractPreference)); } @Override @@ -226,12 +238,12 @@ public void close() { } private class TestFieldExtractOperatorFactory implements Operator.OperatorFactory { - final Operator op; - private String columnName; + private final Operator op; + private final Attribute attribute; - TestFieldExtractOperatorFactory(Attribute attr, MappedFieldType.FieldExtractPreference extractPreference) { - this.op = new TestFieldExtractOperator(attr.name(), attr.dataType(), extractPreference); - this.columnName = attr.name(); + TestFieldExtractOperatorFactory(Attribute attr, FieldExtractPreference extractPreference) { + this.op = new TestFieldExtractOperator(attr, extractPreference); + this.attribute = attr; } @Override @@ -241,27 +253,88 @@ public Operator get(DriverContext driverContext) { @Override public String describe() { - return "TestFieldExtractOperator(" + columnName + ")"; + return "TestFieldExtractOperator(" + attribute.name() + ")"; + } + } + + private Block getBlock(DocBlock docBlock, Attribute attribute, FieldExtractPreference extractPreference) { + if (attribute instanceof UnsupportedAttribute) { + return docBlock.blockFactory().newConstantNullBlock(docBlock.getPositionCount()); + } + return extractBlockForColumn( + docBlock, + attribute.dataType(), + extractPreference, + attribute instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField multiTypeEsField + ? (indexDoc, blockCopier) -> getBlockForMultiType(indexDoc, multiTypeEsField, blockCopier) + : (indexDoc, blockCopier) -> extractBlockForSingleDoc(indexDoc, attribute.name(), blockCopier) + ); + } + + private Block getBlockForMultiType(DocBlock indexDoc, MultiTypeEsField multiTypeEsField, TestBlockCopier blockCopier) { + var indexId = indexDoc.asVector().shards().getInt(0); + var indexPage = indexPages.get(indexId); + var conversion = (AbstractConvertFunction) multiTypeEsField.getConversionExpressionForIndex(indexPage.index); + Supplier nulls = () -> indexDoc.blockFactory().newConstantNullBlock(indexDoc.getPositionCount()); + if (conversion == null) { + return nulls.get(); + } + var field = (FieldAttribute) conversion.field(); + return indexPage.columnIndex(field.fieldName()).isEmpty() + ? nulls.get() + : TypeConverter.fromConvertFunction(conversion).convert(extractBlockForSingleDoc(indexDoc, field.fieldName(), blockCopier)); + } + + private Block extractBlockForSingleDoc(DocBlock docBlock, String columnName, TestBlockCopier blockCopier) { + var indexId = docBlock.asVector().shards().getInt(0); + var indexPage = indexPages.get(indexId); + int columnIndex = indexPage.columnIndex(columnName) + .orElseThrow(() -> new EsqlIllegalArgumentException("Cannot find column named [{}] in {}", columnName, indexPage.columnNames)); + var originalData = indexPage.page.getBlock(columnIndex); + return blockCopier.copyBlock(originalData); + } + + private static void foreachIndexDoc(DocBlock docBlock, Consumer indexDocConsumer) { + var currentIndex = -1; + List currentList = null; + DocVector vector = docBlock.asVector(); + for (int i = 0; i < docBlock.getPositionCount(); i++) { + int indexId = vector.shards().getInt(i); + if (indexId != currentIndex) { + consumeIndexDoc(indexDocConsumer, vector, currentList); + currentList = new ArrayList<>(); + currentIndex = indexId; + } + currentList.add(i); + } + consumeIndexDoc(indexDocConsumer, vector, currentList); + } + + private static void consumeIndexDoc(Consumer indexDocConsumer, DocVector vector, @Nullable List currentList) { + if (currentList != null) { + try (DocVector indexDocVector = vector.filter(currentList.stream().mapToInt(Integer::intValue).toArray())) { + indexDocConsumer.accept(indexDocVector.asBlock()); + } } } private class TestHashAggregationOperator extends HashAggregationOperator { - private final String columnName; + private final Attribute attribute; TestHashAggregationOperator( List aggregators, Supplier blockHash, - String columnName, + Attribute attribute, DriverContext driverContext ) { super(aggregators, blockHash, driverContext); - this.columnName = columnName; + this.attribute = attribute; } @Override protected Page wrapPage(Page page) { - return page.appendBlock(extractBlockForColumn(page, columnName, null, NONE)); + return page.appendBlock(getBlock(page.getBlock(0), attribute, FieldExtractPreference.NONE)); } } @@ -270,24 +343,24 @@ protected Page wrapPage(Page page) { * {@link HashAggregationOperator}. */ private class TestOrdinalsGroupingAggregationOperatorFactory implements Operator.OperatorFactory { - private int groupByChannel; - private List aggregators; - private ElementType groupElementType; - private BigArrays bigArrays; - private String columnName; + private final int groupByChannel; + private final List aggregators; + private final ElementType groupElementType; + private final BigArrays bigArrays; + private final Attribute attribute; TestOrdinalsGroupingAggregationOperatorFactory( int channelIndex, List aggregatorFactories, ElementType groupElementType, BigArrays bigArrays, - String name + Attribute attribute ) { this.groupByChannel = channelIndex; this.aggregators = aggregatorFactories; this.groupElementType = groupElementType; this.bigArrays = bigArrays; - this.columnName = name; + this.attribute = attribute; } @Override @@ -302,7 +375,7 @@ public Operator get(DriverContext driverContext) { pageSize, false ), - columnName, + attribute, driverContext ); } @@ -318,32 +391,34 @@ public String describe() { } private Block extractBlockForColumn( - Page page, - String columnName, + DocBlock docBlock, DataType dataType, - MappedFieldType.FieldExtractPreference extractPreference + FieldExtractPreference extractPreference, + BiFunction extractBlock ) { - var columnIndex = -1; - // locate the block index corresponding to "columnName" - for (int i = 0, size = columnNames.size(); i < size && columnIndex < 0; i++) { - if (columnNames.get(i).equals(columnName)) { - columnIndex = i; - } - } - if (columnIndex < 0) { - throw new EsqlIllegalArgumentException("Cannot find column named [{}] in {}", columnName, columnNames); + BlockFactory blockFactory = docBlock.blockFactory(); + boolean mapToDocValues = shouldMapToDocValues(dataType, extractPreference); + try ( + Block.Builder blockBuilder = mapToDocValues + ? blockFactory.newLongBlockBuilder(docBlock.getPositionCount()) + : blockBuilder(dataType, docBlock.getPositionCount(), TestBlockFactory.getNonBreakingInstance()) + ) { + foreachIndexDoc(docBlock, indexDoc -> { + TestBlockCopier blockCopier = mapToDocValues + ? TestSpatialPointStatsBlockCopier.create(indexDoc.asVector().docs(), dataType) + : new TestBlockCopier(indexDoc.asVector().docs()); + Block blockForIndex = extractBlock.apply(indexDoc, blockCopier); + blockBuilder.copyFrom(blockForIndex, 0, blockForIndex.getPositionCount()); + }); + var result = blockBuilder.build(); + assert result.getPositionCount() == docBlock.getPositionCount() + : "Expected " + docBlock.getPositionCount() + " rows, got " + result.getPositionCount(); + return result; } - DocBlock docBlock = page.getBlock(0); - IntVector docIndices = docBlock.asVector().docs(); - Block originalData = testData.getBlock(columnIndex); - var blockCopier = shouldMapToDocValues(dataType, extractPreference) - ? TestSpatialPointStatsBlockCopier.create(docIndices, dataType) - : new TestBlockCopier(docIndices); - return blockCopier.copyBlock(originalData); } - private boolean shouldMapToDocValues(DataType dataType, MappedFieldType.FieldExtractPreference extractPreference) { - return extractPreference == DOC_VALUES && DataType.isSpatialPoint(dataType); + private boolean shouldMapToDocValues(DataType dataType, FieldExtractPreference extractPreference) { + return extractPreference == FieldExtractPreference.DOC_VALUES && DataType.isSpatialPoint(dataType); } private static class TestBlockCopier { @@ -409,9 +484,9 @@ protected Block copyBlock(Block originalData) { } private static TestSpatialPointStatsBlockCopier create(IntVector docIndices, DataType dataType) { - Function encoder = switch (dataType.esType()) { - case "geo_point" -> SpatialCoordinateTypes.GEO::wkbAsLong; - case "cartesian_point" -> SpatialCoordinateTypes.CARTESIAN::wkbAsLong; + Function encoder = switch (dataType) { + case GEO_POINT -> SpatialCoordinateTypes.GEO::wkbAsLong; + case CARTESIAN_POINT -> SpatialCoordinateTypes.CARTESIAN::wkbAsLong; default -> throw new IllegalArgumentException("Unsupported spatial data type: " + dataType); }; return new TestSpatialPointStatsBlockCopier(docIndices) { @@ -422,4 +497,13 @@ protected long encode(BytesRef wkb) { }; } } + + private static Block.Builder blockBuilder(DataType dataType, int estimatedSize, BlockFactory blockFactory) { + ElementType elementType = switch (dataType) { + case SHORT -> ElementType.INT; + case FLOAT, HALF_FLOAT, SCALED_FLOAT -> ElementType.DOUBLE; + default -> PlannerUtils.toElementType(dataType); + }; + return elementType.newBlockBuilder(estimatedSize, blockFactory); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtilsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtilsTests.java index 1000c05282fdb..6b01010ffa5f4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtilsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtilsTests.java @@ -644,6 +644,7 @@ public void testMissingIndicesIsFatal() { public void testCheckForCcsLicense() { final TestIndicesExpressionGrouper indicesGrouper = new TestIndicesExpressionGrouper(); + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); // this seems to be used only for tracking usage of features, not for checking if a license is expired final LongSupplier currTime = () -> System.currentTimeMillis(); @@ -671,22 +672,22 @@ public void testCheckForCcsLicense() { List indices = new ArrayList<>(); indices.add(new TableInfo(new TableIdentifier(EMPTY, null, randomFrom("idx", "idx1,idx2*")))); - checkForCcsLicense(indices, indicesGrouper, enterpriseLicenseValid); - checkForCcsLicense(indices, indicesGrouper, platinumLicenseValid); - checkForCcsLicense(indices, indicesGrouper, goldLicenseValid); - checkForCcsLicense(indices, indicesGrouper, trialLicenseValid); - checkForCcsLicense(indices, indicesGrouper, basicLicenseValid); - checkForCcsLicense(indices, indicesGrouper, standardLicenseValid); - checkForCcsLicense(indices, indicesGrouper, missingLicense); - checkForCcsLicense(indices, indicesGrouper, nullLicense); - - checkForCcsLicense(indices, indicesGrouper, enterpriseLicenseInactive); - checkForCcsLicense(indices, indicesGrouper, platinumLicenseInactive); - checkForCcsLicense(indices, indicesGrouper, goldLicenseInactive); - checkForCcsLicense(indices, indicesGrouper, trialLicenseInactive); - checkForCcsLicense(indices, indicesGrouper, basicLicenseInactive); - checkForCcsLicense(indices, indicesGrouper, standardLicenseInactive); - checkForCcsLicense(indices, indicesGrouper, missingLicenseInactive); + checkForCcsLicense(executionInfo, indices, indicesGrouper, enterpriseLicenseValid); + checkForCcsLicense(executionInfo, indices, indicesGrouper, platinumLicenseValid); + checkForCcsLicense(executionInfo, indices, indicesGrouper, goldLicenseValid); + checkForCcsLicense(executionInfo, indices, indicesGrouper, trialLicenseValid); + checkForCcsLicense(executionInfo, indices, indicesGrouper, basicLicenseValid); + checkForCcsLicense(executionInfo, indices, indicesGrouper, standardLicenseValid); + checkForCcsLicense(executionInfo, indices, indicesGrouper, missingLicense); + checkForCcsLicense(executionInfo, indices, indicesGrouper, nullLicense); + + checkForCcsLicense(executionInfo, indices, indicesGrouper, enterpriseLicenseInactive); + checkForCcsLicense(executionInfo, indices, indicesGrouper, platinumLicenseInactive); + checkForCcsLicense(executionInfo, indices, indicesGrouper, goldLicenseInactive); + checkForCcsLicense(executionInfo, indices, indicesGrouper, trialLicenseInactive); + checkForCcsLicense(executionInfo, indices, indicesGrouper, basicLicenseInactive); + checkForCcsLicense(executionInfo, indices, indicesGrouper, standardLicenseInactive); + checkForCcsLicense(executionInfo, indices, indicesGrouper, missingLicenseInactive); } // cross-cluster search requires a valid (active, non-expired) enterprise license OR a valid trial license @@ -701,8 +702,8 @@ public void testCheckForCcsLicense() { } // licenses that work - checkForCcsLicense(indices, indicesGrouper, enterpriseLicenseValid); - checkForCcsLicense(indices, indicesGrouper, trialLicenseValid); + checkForCcsLicense(executionInfo, indices, indicesGrouper, enterpriseLicenseValid); + checkForCcsLicense(executionInfo, indices, indicesGrouper, trialLicenseValid); // all others fail --- @@ -739,9 +740,10 @@ private void assertLicenseCheckFails( XPackLicenseState licenseState, String expectedErrorMessageSuffix ) { + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); ElasticsearchStatusException e = expectThrows( ElasticsearchStatusException.class, - () -> checkForCcsLicense(indices, indicesGrouper, licenseState) + () -> checkForCcsLicense(executionInfo, indices, indicesGrouper, licenseState) ); assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST)); assertThat( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java index f01a125bc3c23..c3f7080ebdd92 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.math.Pow; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat; import org.elasticsearch.xpack.esql.expression.predicate.fulltext.FullTextPredicate; +import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Grok; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; @@ -87,6 +88,7 @@ import static java.util.Collections.emptyList; import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration; import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; +import static org.elasticsearch.xpack.esql.index.EsIndexSerializationTests.randomEsIndex; import static org.mockito.Mockito.mock; /** @@ -491,6 +493,9 @@ public void accept(Page page) { if (argClass == Configuration.class) { return randomConfiguration(); } + if (argClass == EsIndex.class) { + return randomEsIndex(); + } if (argClass == JoinConfig.class) { return new JoinConfig( JoinTypes.LEFT, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java index 876ff01812064..62c302e97815d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java @@ -9,6 +9,7 @@ import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsMapper; import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper; import org.elasticsearch.xpack.inference.queries.SemanticQueryBuilder; import org.elasticsearch.xpack.inference.rank.random.RandomRankRetrieverBuilder; @@ -48,7 +49,8 @@ public Set getTestFeatures() { SemanticTextFieldMapper.SEMANTIC_TEXT_ALWAYS_EMIT_INFERENCE_ID_FIX, SEMANTIC_TEXT_HIGHLIGHTER, SEMANTIC_MATCH_QUERY_REWRITE_INTERCEPTION_SUPPORTED, - SEMANTIC_SPARSE_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED + SEMANTIC_SPARSE_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED, + SemanticInferenceMetadataFieldsMapper.EXPLICIT_NULL_FIXES ); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java index 22d6157b335ca..f4aa49bad1648 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java @@ -39,6 +39,7 @@ import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; +import org.elasticsearch.xcontent.XContent; import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceError; import org.elasticsearch.xpack.inference.mapper.SemanticTextField; import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper; @@ -50,6 +51,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -67,6 +69,8 @@ */ public class ShardBulkInferenceActionFilter implements MappedActionFilter { protected static final int DEFAULT_BATCH_SIZE = 512; + private static final Object EXPLICIT_NULL = new Object(); + private static final ChunkedInference EMPTY_CHUNKED_INFERENCE = new EmptyChunkedInference(); private final ClusterService clusterService; private final InferenceServiceRegistry inferenceServiceRegistry; @@ -393,11 +397,22 @@ private void applyInferenceResponses(BulkItemRequest item, FieldInferenceRespons for (var entry : response.responses.entrySet()) { var fieldName = entry.getKey(); var responses = entry.getValue(); - var model = responses.get(0).model(); + Model model = null; + + InferenceFieldMetadata inferenceFieldMetadata = fieldInferenceMap.get(fieldName); + if (inferenceFieldMetadata == null) { + throw new IllegalStateException("No inference field metadata for field [" + fieldName + "]"); + } + // ensure that the order in the original field is consistent in case of multiple inputs Collections.sort(responses, Comparator.comparingInt(FieldInferenceResponse::inputOrder)); Map> chunkMap = new LinkedHashMap<>(); for (var resp : responses) { + // Get the first non-null model from the response list + if (model == null) { + model = resp.model; + } + var lst = chunkMap.computeIfAbsent(resp.sourceField, k -> new ArrayList<>()); lst.addAll( SemanticTextField.toSemanticTextFieldChunks( @@ -409,21 +424,26 @@ private void applyInferenceResponses(BulkItemRequest item, FieldInferenceRespons ) ); } + List inputs = responses.stream() .filter(r -> r.sourceField().equals(fieldName)) .map(r -> r.input) .collect(Collectors.toList()); + + // The model can be null if we are only processing update requests that clear inference results. This is ok because we will + // merge in the field's existing model settings on the data node. var result = new SemanticTextField( useLegacyFormat, fieldName, useLegacyFormat ? inputs : null, new SemanticTextField.InferenceResult( - model.getInferenceEntityId(), - new SemanticTextField.ModelSettings(model), + inferenceFieldMetadata.getInferenceId(), + model != null ? new SemanticTextField.ModelSettings(model) : null, chunkMap ), indexRequest.getContentType() ); + if (useLegacyFormat) { SemanticTextUtils.insertValue(fieldName, newDocMap, result); } else { @@ -490,7 +510,8 @@ private Map> createFieldInferenceRequests(Bu } else { var inferenceMetadataFieldsValue = XContentMapValues.extractValue( InferenceMetadataFieldsMapper.NAME + "." + field, - docMap + docMap, + EXPLICIT_NULL ); if (inferenceMetadataFieldsValue != null) { // Inference has already been computed @@ -500,9 +521,22 @@ private Map> createFieldInferenceRequests(Bu int order = 0; for (var sourceField : entry.getSourceFields()) { - // TODO: Detect when the field is provided with an explicit null value - var valueObj = XContentMapValues.extractValue(sourceField, docMap); - if (valueObj == null) { + var valueObj = XContentMapValues.extractValue(sourceField, docMap, EXPLICIT_NULL); + if (useLegacyFormat == false && isUpdateRequest && valueObj == EXPLICIT_NULL) { + /** + * It's an update request, and the source field is explicitly set to null, + * so we need to propagate this information to the inference fields metadata + * to overwrite any inference previously computed on the field. + * This ensures that the field is treated as intentionally cleared, + * preventing any unintended carryover of prior inference results. + */ + var slot = ensureResponseAccumulatorSlot(itemIndex); + slot.addOrUpdateResponse( + new FieldInferenceResponse(field, sourceField, null, order++, 0, null, EMPTY_CHUNKED_INFERENCE) + ); + continue; + } + if (valueObj == null || valueObj == EXPLICIT_NULL) { if (isUpdateRequest && useLegacyFormat) { addInferenceResponseFailure( item.id(), @@ -552,4 +586,11 @@ static IndexRequest getIndexRequestOrNull(DocWriteRequest docWriteRequest) { return null; } } + + private static class EmptyChunkedInference implements ChunkedInference { + @Override + public Iterator chunksAsMatchedTextAndByteReference(XContent xcontent) { + return Collections.emptyIterator(); + } + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapper.java index 7a1a9b056d0a1..3f49973d6e35f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapper.java @@ -12,6 +12,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.join.BitSetProducer; import org.elasticsearch.common.xcontent.XContentParserUtils; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; @@ -38,6 +39,8 @@ public class SemanticInferenceMetadataFieldsMapper extends InferenceMetadataFieldsMapper { private static final SemanticInferenceMetadataFieldsMapper INSTANCE = new SemanticInferenceMetadataFieldsMapper(); + public static final NodeFeature EXPLICIT_NULL_FIXES = new NodeFeature("semantic_text.inference_metadata_fields.explicit_null_fixes"); + public static final TypeParser PARSER = new FixedTypeParser( c -> InferenceMetadataFieldsMapper.isEnabled(c.getSettings()) ? INSTANCE : null ); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java index cfd05cb29ca03..fddff17dab4cf 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java @@ -338,16 +338,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws static { SEMANTIC_TEXT_FIELD_PARSER.declareStringArray(optionalConstructorArg(), new ParseField(TEXT_FIELD)); - SEMANTIC_TEXT_FIELD_PARSER.declareObject( - constructorArg(), - (p, c) -> INFERENCE_RESULT_PARSER.parse(p, c), - new ParseField(INFERENCE_FIELD) - ); + SEMANTIC_TEXT_FIELD_PARSER.declareObject(constructorArg(), INFERENCE_RESULT_PARSER, new ParseField(INFERENCE_FIELD)); INFERENCE_RESULT_PARSER.declareString(constructorArg(), new ParseField(INFERENCE_ID_FIELD)); - INFERENCE_RESULT_PARSER.declareObject( + INFERENCE_RESULT_PARSER.declareObjectOrNull( constructorArg(), (p, c) -> MODEL_SETTINGS_PARSER.parse(p, null), + null, new ParseField(MODEL_SETTINGS_FIELD) ); INFERENCE_RESULT_PARSER.declareField(constructorArg(), (p, c) -> { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index b47c55c302273..690a136c566e0 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -384,6 +384,17 @@ void parseCreateFieldFromContext(DocumentParserContext context, SemanticTextFiel mapper = this; } + if (mapper.fieldType().getModelSettings() == null) { + for (var chunkList : field.inference().chunks().values()) { + if (chunkList.isEmpty() == false) { + throw new DocumentParsingException( + xContentLocation, + "[" + MODEL_SETTINGS_FIELD + "] must be set for field [" + fullFieldName + "] when chunks are provided" + ); + } + } + } + var chunksField = mapper.fieldType().getChunksField(); var embeddingsField = mapper.fieldType().getEmbeddingsField(); var offsetsField = mapper.fieldType().getOffsetsField(); @@ -895,7 +906,7 @@ private static boolean canMergeModelSettings( if (Objects.equals(previous, current)) { return true; } - if (previous == null) { + if (previous == null || current == null) { return true; } conflicts.addConflict("model_settings", ""); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java index 478c81f7c5a32..0432a2ff3fc9e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.ActionFilterChain; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -67,6 +68,8 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.awaitLatch; import static org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilter.DEFAULT_BATCH_SIZE; import static org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilter.getIndexRequestOrNull; +import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getChunksFieldName; +import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getOriginalTextFieldName; import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomChunkedInferenceEmbeddingSparse; import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomSemanticText; import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomSemanticTextInput; @@ -75,12 +78,15 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class ShardBulkInferenceActionFilterTests extends ESTestCase { + private static final Object EXPLICIT_NULL = new Object(); + private final boolean useLegacyFormat; private ThreadPool threadPool; @@ -205,6 +211,11 @@ public void testItemFailures() throws Exception { XContentMapValues.extractValue(useLegacyFormat ? "field1.text" : "field1", actualRequest.sourceAsMap()), equalTo("I am a success") ); + if (useLegacyFormat == false) { + assertNotNull( + XContentMapValues.extractValue(InferenceMetadataFieldsMapper.NAME + ".field1", actualRequest.sourceAsMap()) + ); + } // item 2 is a failure assertNotNull(bulkShardRequest.items()[2].getPrimaryResponse()); @@ -232,6 +243,79 @@ public void testItemFailures() throws Exception { awaitLatch(chainExecuted, 10, TimeUnit.SECONDS); } + @SuppressWarnings({ "unchecked", "rawtypes" }) + public void testExplicitNull() throws Exception { + StaticModel model = StaticModel.createRandomInstance(); + model.putResult("I am a failure", new ChunkedInferenceError(new IllegalArgumentException("boom"))); + model.putResult("I am a success", randomChunkedInferenceEmbeddingSparse(List.of("I am a success"))); + + ShardBulkInferenceActionFilter filter = createFilter( + threadPool, + Map.of(model.getInferenceEntityId(), model), + randomIntBetween(1, 10), + useLegacyFormat + ); + + CountDownLatch chainExecuted = new CountDownLatch(1); + ActionFilterChain actionFilterChain = (task, action, request, listener) -> { + try { + BulkShardRequest bulkShardRequest = (BulkShardRequest) request; + assertNull(bulkShardRequest.getInferenceFieldMap()); + assertThat(bulkShardRequest.items().length, equalTo(5)); + + // item 0 + assertNull(bulkShardRequest.items()[0].getPrimaryResponse()); + IndexRequest actualRequest = getIndexRequestOrNull(bulkShardRequest.items()[0].request()); + assertThat(XContentMapValues.extractValue("obj.field1", actualRequest.sourceAsMap(), EXPLICIT_NULL), is(EXPLICIT_NULL)); + assertNull(XContentMapValues.extractValue(InferenceMetadataFieldsMapper.NAME, actualRequest.sourceAsMap(), EXPLICIT_NULL)); + + // item 1 is a success + assertNull(bulkShardRequest.items()[1].getPrimaryResponse()); + actualRequest = getIndexRequestOrNull(bulkShardRequest.items()[1].request()); + assertInferenceResults(useLegacyFormat, actualRequest, "obj.field1", "I am a success", 1); + + // item 2 is a failure + assertNotNull(bulkShardRequest.items()[2].getPrimaryResponse()); + assertTrue(bulkShardRequest.items()[2].getPrimaryResponse().isFailed()); + var failure = bulkShardRequest.items()[2].getPrimaryResponse().getFailure(); + assertThat(failure.getCause().getCause().getMessage(), containsString("boom")); + + // item 3 + assertNull(bulkShardRequest.items()[3].getPrimaryResponse()); + actualRequest = getIndexRequestOrNull(bulkShardRequest.items()[3].request()); + assertInferenceResults(useLegacyFormat, actualRequest, "obj.field1", EXPLICIT_NULL, 0); + + // item 4 + assertNull(bulkShardRequest.items()[4].getPrimaryResponse()); + actualRequest = getIndexRequestOrNull(bulkShardRequest.items()[4].request()); + assertNull(XContentMapValues.extractValue("obj.field1", actualRequest.sourceAsMap(), EXPLICIT_NULL)); + assertNull(XContentMapValues.extractValue(InferenceMetadataFieldsMapper.NAME, actualRequest.sourceAsMap(), EXPLICIT_NULL)); + } finally { + chainExecuted.countDown(); + } + }; + ActionListener actionListener = mock(ActionListener.class); + Task task = mock(Task.class); + + Map inferenceFieldMap = Map.of( + "obj.field1", + new InferenceFieldMetadata("obj.field1", model.getInferenceEntityId(), new String[] { "obj.field1" }) + ); + Map sourceWithNull = new HashMap<>(); + sourceWithNull.put("field1", null); + + BulkItemRequest[] items = new BulkItemRequest[5]; + items[0] = new BulkItemRequest(0, new IndexRequest("index").source(Map.of("obj", sourceWithNull))); + items[1] = new BulkItemRequest(1, new IndexRequest("index").source("obj.field1", "I am a success")); + items[2] = new BulkItemRequest(2, new IndexRequest("index").source("obj.field1", "I am a failure")); + items[3] = new BulkItemRequest(3, new UpdateRequest().doc(new IndexRequest("index").source(Map.of("obj", sourceWithNull)))); + items[4] = new BulkItemRequest(4, new UpdateRequest().doc(new IndexRequest("index").source(Map.of("field2", "value")))); + BulkShardRequest request = new BulkShardRequest(new ShardId("test", "test", 0), WriteRequest.RefreshPolicy.NONE, items); + request.setInferenceFieldMap(inferenceFieldMap); + filter.apply(task, TransportShardBulkAction.ACTION_NAME, request, actionListener, actionFilterChain); + awaitLatch(chainExecuted, 10, TimeUnit.SECONDS); + } + @SuppressWarnings({ "unchecked", "rawtypes" }) public void testManyRandomDocs() throws Exception { Map inferenceModelMap = new HashMap<>(); @@ -435,6 +519,53 @@ private static BulkItemRequest[] randomBulkItemRequest( new BulkItemRequest(requestId, new IndexRequest("index").source(expectedDocMap, requestContentType)) }; } + @SuppressWarnings({ "unchecked" }) + private static void assertInferenceResults( + boolean useLegacyFormat, + IndexRequest request, + String fieldName, + Object expectedOriginalValue, + int expectedChunkCount + ) { + final Map requestMap = request.sourceAsMap(); + if (useLegacyFormat) { + assertThat( + XContentMapValues.extractValue(getOriginalTextFieldName(fieldName), requestMap, EXPLICIT_NULL), + equalTo(expectedOriginalValue) + ); + + List chunks = (List) XContentMapValues.extractValue(getChunksFieldName(fieldName), requestMap); + if (expectedChunkCount > 0) { + assertNotNull(chunks); + assertThat(chunks.size(), equalTo(expectedChunkCount)); + } else { + // If the expected chunk count is 0, we expect that no inference has been performed. In this case, the source should not be + // transformed, and thus the semantic text field structure should not be created. + assertNull(chunks); + } + } else { + assertThat(XContentMapValues.extractValue(fieldName, requestMap, EXPLICIT_NULL), equalTo(expectedOriginalValue)); + + Map inferenceMetadataFields = (Map) XContentMapValues.extractValue( + InferenceMetadataFieldsMapper.NAME, + requestMap, + EXPLICIT_NULL + ); + assertNotNull(inferenceMetadataFields); + + // When using the inference metadata fields format, chunks are mapped by source field. We handle clearing inference results for + // a field by emitting an empty chunk list for it. This is done to prevent the clear operation from clearing inference results + // for other source fields. + List chunks = (List) XContentMapValues.extractValue( + getChunksFieldName(fieldName) + "." + fieldName, + inferenceMetadataFields, + EXPLICIT_NULL + ); + assertNotNull(chunks); + assertThat(chunks.size(), equalTo(expectedChunkCount)); + } + } + private static class StaticModel extends TestModel { private final Map resultMap; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldMapperTests.java index 6504ccc4dd39f..8fcc0df0093ce 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldMapperTests.java @@ -9,10 +9,15 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xpack.inference.InferencePlugin; import java.util.Collection; @@ -24,6 +29,32 @@ protected Collection getPlugins() { return Collections.singletonList(new InferencePlugin(Settings.EMPTY)); } + public void testIsEnabled() { + var settings = Settings.builder() + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), getRandomCompatibleIndexVersion(true)) + .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), true) + .build(); + assertFalse(InferenceMetadataFieldsMapper.isEnabled(settings)); + + settings = Settings.builder() + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), getRandomCompatibleIndexVersion(true)) + .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), false) + .build(); + assertFalse(InferenceMetadataFieldsMapper.isEnabled(settings)); + + settings = Settings.builder() + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), getRandomCompatibleIndexVersion(false)) + .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), true) + .build(); + assertFalse(InferenceMetadataFieldsMapper.isEnabled(settings)); + + settings = Settings.builder() + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), getRandomCompatibleIndexVersion(false)) + .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), false) + .build(); + assertTrue(InferenceMetadataFieldsMapper.isEnabled(settings)); + } + @Override public void testFieldHasValue() { assertTrue( @@ -42,4 +73,26 @@ public void testFieldHasValueWithEmptyFieldInfos() { public MappedFieldType getMappedFieldType() { return new SemanticInferenceMetadataFieldsMapper.FieldType(); } + + static IndexVersion getRandomCompatibleIndexVersion(boolean useLegacyFormat) { + if (useLegacyFormat) { + if (randomBoolean()) { + return IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.UPGRADE_TO_LUCENE_10_0_0, + IndexVersionUtils.getPreviousVersion(IndexVersions.INFERENCE_METADATA_FIELDS) + ); + } + return IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT); + } else { + if (randomBoolean()) { + return IndexVersionUtils.randomVersionBetween(random(), IndexVersions.INFERENCE_METADATA_FIELDS, IndexVersion.current()); + } + return IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, + IndexVersionUtils.getPreviousVersion(IndexVersions.UPGRADE_TO_LUCENE_10_0_0) + ); + } + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 11362c3cedd06..e6d68c8343d8b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -24,6 +24,7 @@ import org.apache.lucene.search.join.QueryBitSetProducer; import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.common.Strings; @@ -112,6 +113,10 @@ protected Collection getPlugins() { private MapperService createMapperService(XContentBuilder mappings, boolean useLegacyFormat) throws IOException { var settings = Settings.builder() + .put( + IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), + SemanticInferenceMetadataFieldMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat) + ) .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat) .build(); return createMapperService(settings, mappings); @@ -770,6 +775,35 @@ public void testDenseVectorElementType() throws IOException { assertMapperService.accept(byteMapperService, DenseVectorFieldMapper.ElementType.BYTE); } + public void testModelSettingsRequiredWithChunks() throws IOException { + // Create inference results where model settings are set to null and chunks are provided + Model model = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); + SemanticTextField randomSemanticText = randomSemanticText(useLegacyFormat, "field", model, List.of("a"), XContentType.JSON); + SemanticTextField inferenceResults = new SemanticTextField( + randomSemanticText.useLegacyFormat(), + randomSemanticText.fieldName(), + randomSemanticText.originalValues(), + new SemanticTextField.InferenceResult( + randomSemanticText.inference().inferenceId(), + null, + randomSemanticText.inference().chunks() + ), + randomSemanticText.contentType() + ); + + MapperService mapperService = createMapperService( + mapping(b -> addSemanticTextMapping(b, "field", model.getInferenceEntityId(), null)), + useLegacyFormat + ); + SourceToParse source = source(b -> addSemanticTextInferenceResults(useLegacyFormat, b, List.of(inferenceResults))); + DocumentParsingException ex = expectThrows( + DocumentParsingException.class, + DocumentParsingException.class, + () -> mapperService.documentMapper().parse(source) + ); + assertThat(ex.getMessage(), containsString("[model_settings] must be set for field [field] when chunks are provided")); + } + private MapperService mapperServiceForFieldWithModelSettings( String fieldName, String inferenceId, diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update.yml index 660d3e37f4242..27c405f6c23bf 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update.yml @@ -819,84 +819,210 @@ setup: - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 } - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 30 } -# TODO: Uncomment this test once we implement a fix -#--- -#"Bypass inference on bulk update operation": -# # Update as upsert -# - do: -# bulk: -# body: -# - '{"update": {"_index": "test-index", "_id": "doc_1"}}' -# - '{"doc": { "sparse_field": "inference test", "dense_field": "another inference test", "non_inference_field": "non inference test" }, "doc_as_upsert": true}' -# -# - match: { errors: false } -# - match: { items.0.update.result: "created" } -# -# - do: -# bulk: -# body: -# - '{"update": {"_index": "test-index", "_id": "doc_1"}}' -# - '{"doc": { "non_inference_field": "another value" }, "doc_as_upsert": true}' -# refresh: true -# -# - match: { errors: false } -# - match: { items.0.update.result: "updated" } -# -# - do: -# search: -# index: test-index -# body: -# fields: [ _inference_fields ] -# query: -# match_all: { } -# -# - match: { hits.total.value: 1 } -# - match: { hits.total.relation: eq } -# -# - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 } -# - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 } -# - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings -# - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 } -# - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 14 } -# -# - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 } -# - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 1 } -# - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings -# - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 } -# - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 22 } -# -# - match: { hits.hits.0._source.sparse_field: "inference test" } -# - match: { hits.hits.0._source.dense_field: "another inference test" } -# - match: { hits.hits.0._source.non_inference_field: "another value" } -# -# - do: -# bulk: -# body: -# - '{"update": {"_index": "test-index", "_id": "doc_1"}}' -# - '{"doc": { "sparse_field": null, "dense_field": null, "non_inference_field": "updated value" }, "doc_as_upsert": true}' -# refresh: true -# -# - match: { errors: false } -# - match: { items.0.update.result: "updated" } -# -# - do: -# search: -# index: test-index -# body: -# fields: [ _inference_fields ] -# query: -# match_all: { } -# -# - match: { hits.total.value: 1 } -# - match: { hits.total.relation: eq } -# -# # TODO: BUG! Setting sparse_field & dense_field to null does not clear _inference_fields -# - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 } -# - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 0 } -# -# - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 } -# - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 0 } -# -# - not_exists: hits.hits.0._source.sparse_field -# - not_exists: hits.hits.0._source.dense_field -# - match: { hits.hits.0._source.non_inference_field: "updated value" } +--- +"Bypass inference on bulk update operation": + # Update as upsert + - do: + bulk: + body: + - '{"update": {"_index": "test-index", "_id": "doc_1"}}' + - '{"doc": { "sparse_field": "inference test", "dense_field": "another inference test", "non_inference_field": "non inference test" }, "doc_as_upsert": true}' + + - match: { errors: false } + - match: { items.0.update.result: "created" } + + - do: + bulk: + body: + - '{"update": {"_index": "test-index", "_id": "doc_1"}}' + - '{"doc": { "non_inference_field": "another value" }, "doc_as_upsert": true}' + refresh: true + + - match: { errors: false } + - match: { items.0.update.result: "updated" } + + - do: + search: + index: test-index + body: + fields: [ _inference_fields ] + query: + match_all: { } + + - match: { hits.total.value: 1 } + - match: { hits.total.relation: eq } + + - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 } + - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 } + - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings + - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 } + - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 14 } + + - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 } + - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 1 } + - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings + - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 } + - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 22 } + + - match: { hits.hits.0._source.sparse_field: "inference test" } + - match: { hits.hits.0._source.dense_field: "another inference test" } + - match: { hits.hits.0._source.non_inference_field: "another value" } + +--- +"Explicit nulls clear inference results on bulk update operation": + - requires: + cluster_features: "semantic_text.inference_metadata_fields.explicit_null_fixes" + reason: Fixes explicit null handling when using the _inference_fields metafield + + - skip: + features: [ "headers" ] + + - do: + indices.create: + index: test-copy-to-index + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: false + mappings: + properties: + sparse_field: + type: semantic_text + inference_id: sparse-inference-id + sparse_source_field: + type: text + copy_to: sparse_field + dense_field: + type: semantic_text + inference_id: dense-inference-id + dense_source_field: + type: text + copy_to: dense_field + non_inference_field: + type: text + + - do: + index: + index: test-copy-to-index + id: doc_1 + body: + sparse_field: "inference test" + sparse_source_field: "sparse source test" + dense_field: "another inference test" + dense_source_field: "dense source test" + non_inference_field: "non inference test" + refresh: true + + - do: + headers: + # Force JSON content type so that we use a parser that interprets the embeddings as doubles + Content-Type: application/json + search: + index: test-copy-to-index + body: + fields: [ _inference_fields ] + query: + match_all: { } + + - match: { hits.total.value: 1 } + - match: { hits.total.relation: eq } + + - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 2 } + - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field: 1 } + - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.embeddings + - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.start_offset: 0 } + - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_field.0.end_offset: 14 } + - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field: 1 } + - exists: hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.embeddings + - set: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.embeddings: sparse_source_field_embeddings } + - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.start_offset: 0 } + - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.end_offset: 18 } + + - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 2 } + - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field: 1 } + - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.embeddings + - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.start_offset: 0 } + - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_field.0.end_offset: 22 } + - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field: 1 } + - exists: hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.embeddings + - set: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.embeddings: dense_source_field_embeddings } + - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.start_offset: 0 } + - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.end_offset: 17 } + + - match: { hits.hits.0._source.sparse_field: "inference test" } + - match: { hits.hits.0._source.sparse_source_field: "sparse source test" } + - match: { hits.hits.0._source.dense_field: "another inference test" } + - match: { hits.hits.0._source.dense_source_field: "dense source test" } + - match: { hits.hits.0._source.non_inference_field: "non inference test" } + + - do: + bulk: + body: + - '{"update": {"_index": "test-copy-to-index", "_id": "doc_1"}}' + - '{"doc": { "sparse_field": null, "dense_field": null, "non_inference_field": "updated value" }, "doc_as_upsert": true}' + refresh: true + + - match: { errors: false } + - match: { items.0.update.result: "updated" } + + - do: + headers: + # Force JSON content type so that we use a parser that interprets the embeddings as doubles + Content-Type: application/json + search: + index: test-copy-to-index + body: + fields: [ _inference_fields ] + query: + match_all: { } + + - match: { hits.total.value: 1 } + - match: { hits.total.relation: eq } + + - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks: 1 } + - length: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field: 1 } + - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.embeddings: $sparse_source_field_embeddings } + - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.start_offset: 0 } + - match: { hits.hits.0._source._inference_fields.sparse_field.inference.chunks.sparse_source_field.0.end_offset: 18 } + + - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks: 1 } + - length: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field: 1 } + - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.embeddings: $dense_source_field_embeddings } + - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.start_offset: 0 } + - match: { hits.hits.0._source._inference_fields.dense_field.inference.chunks.dense_source_field.0.end_offset: 17 } + + - not_exists: hits.hits.0._source.sparse_field + - match: { hits.hits.0._source.sparse_source_field: "sparse source test" } + - not_exists: hits.hits.0._source.dense_field + - match: { hits.hits.0._source.dense_source_field: "dense source test" } + - match: { hits.hits.0._source.non_inference_field: "updated value" } + + - do: + bulk: + body: + - '{"update": {"_index": "test-copy-to-index", "_id": "doc_1"}}' + - '{"doc": { "sparse_source_field": null, "dense_source_field": null, "non_inference_field": "another value" }, "doc_as_upsert": true}' + refresh: true + + - match: { errors: false } + - match: { items.0.update.result: "updated" } + + - do: + search: + index: test-copy-to-index + body: + fields: [ _inference_fields ] + query: + match_all: { } + + - match: { hits.total.value: 1 } + - match: { hits.total.relation: eq } + + - not_exists: hits.hits.0._source._inference_fields + - not_exists: hits.hits.0._source.sparse_field + - not_exists: hits.hits.0._source.sparse_source_field + - not_exists: hits.hits.0._source.dense_field + - not_exists: hits.hits.0._source.dense_source_field + - match: { hits.hits.0._source.non_inference_field: "another value" } diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update_bwc.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update_bwc.yml index 6b494d531b2d1..912cdb5a85d35 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update_bwc.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update_bwc.yml @@ -632,6 +632,31 @@ setup: - match: { _source.dense_field.inference.chunks.0.text: "another inference test" } - match: { _source.non_inference_field: "another value" } +--- +"Explicit nulls clear inference results on bulk update operation": + # Update as upsert + - do: + bulk: + body: + - '{"update": {"_index": "test-index", "_id": "doc_1"}}' + - '{"doc": { "sparse_field": "inference test", "dense_field": "another inference test", "non_inference_field": "non inference test" }, "doc_as_upsert": true}' + + - match: { errors: false } + - match: { items.0.update.result: "created" } + + - do: + get: + index: test-index + id: doc_1 + + - match: { _source.sparse_field.text: "inference test" } + - exists: _source.sparse_field.inference.chunks.0.embeddings + - match: { _source.sparse_field.inference.chunks.0.text: "inference test" } + - match: { _source.dense_field.text: "another inference test" } + - exists: _source.dense_field.inference.chunks.0.embeddings + - match: { _source.dense_field.inference.chunks.0.text: "another inference test" } + - match: { _source.non_inference_field: "non inference test" } + - do: bulk: body: diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeDetector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeDetector.java index e771fb3b94568..cf97ed629cfc5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeDetector.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeDetector.java @@ -189,7 +189,7 @@ private TestStats testTrendVs(TestStats H0, double[] values, double[] weights) { private TestStats testStepChangeVs(TestStats H0, double[] values, double[] weights, int[] candidateChangePoints) { double vStep = Double.MAX_VALUE; - int changePoint = -1; + int changePoint = ChangeType.NO_CHANGE_POINT; // Initialize running stats so that they are only missing the individual changepoint values RunningStats lowerRange = new RunningStats(); @@ -226,7 +226,7 @@ private TestStats testStepChangeVs(TestStats H0, double[] values, double[] weigh private TestStats testTrendChangeVs(TestStats H0, double[] values, double[] weights, int[] candidateChangePoints) { double vChange = Double.MAX_VALUE; - int changePoint = -1; + int changePoint = ChangeType.NO_CHANGE_POINT; // Initialize running stats so that they are only missing the individual changepoint values RunningStats lowerRange = new RunningStats(); @@ -349,7 +349,7 @@ private TestStats testDistributionChange( ) { double maxDiff = 0.0; - int changePoint = -1; + int changePoint = ChangeType.NO_CHANGE_POINT; // Initialize running stats so that they are only missing the individual changepoint values RunningStats lowerRange = new RunningStats(); @@ -378,10 +378,12 @@ private TestStats testDistributionChange( // before we run the tests. SampleData sampleData = sample(values, weights, discoveredChangePoints); final double[] sampleValues = sampleData.values(); - final double[] sampleWeights = sampleData.weights(); double pValue = 1; for (int cp : sampleData.changePoints()) { + if (cp == ChangeType.NO_CHANGE_POINT) { + continue; + } double[] x = Arrays.copyOfRange(sampleValues, 0, cp); double[] y = Arrays.copyOfRange(sampleValues, cp, sampleValues.length); double statistic = KOLMOGOROV_SMIRNOV_TEST.kolmogorovSmirnovStatistic(x, y); @@ -451,7 +453,7 @@ private record TestStats(Type type, double pValue, double var, double nParams, i } TestStats(Type type, double pValue, double var, double nParams, DataStats dataStats) { - this(type, pValue, var, nParams, -1, dataStats); + this(type, pValue, var, nParams, ChangeType.NO_CHANGE_POINT, dataStats); } boolean accept(double pValueThreshold) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregator.java index d643a937180a1..ce622df184617 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregator.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregator.java @@ -47,7 +47,7 @@ public InternalAggregation doReduce(InternalAggregations aggregations, Aggregati ChangeType change = ChangePointDetector.getChangeType(bucketValues); ChangePointBucket changePointBucket = null; - if (change.changePoint() >= 0) { + if (change.changePoint() != ChangeType.NO_CHANGE_POINT) { changePointBucket = extractBucket(bucketsPaths()[0], aggregations, change.changePoint()).map( b -> new ChangePointBucket(b.getKey(), b.getDocCount(), b.getAggregations()) ).orElse(null); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeType.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeType.java index c62355dc47451..7df542b59107b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeType.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeType.java @@ -21,8 +21,10 @@ */ public interface ChangeType extends NamedWriteable, NamedXContentObject { + int NO_CHANGE_POINT = -1; + default int changePoint() { - return -1; + return NO_CHANGE_POINT; } default double pValue() { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeDetectorTests.java index 75f668a96e77e..36076bbb0ec25 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeDetectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangeDetectorTests.java @@ -17,6 +17,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.DoubleStream; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThan; @@ -243,4 +244,14 @@ public void testProblemDistributionChange() { ChangeType type = new ChangeDetector(bucketValues).detect(0.05); assertThat(type, instanceOf(ChangeType.DistributionChange.class)); } + + public void testUncertainNonStationary() { + MlAggsHelper.DoubleBucketValues bucketValues = new MlAggsHelper.DoubleBucketValues( + null, + new double[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 700, 735, 715 } + ); + ChangeType type = new ChangeDetector(bucketValues).detect(0.01); + assertThat(type, instanceOf(ChangeType.NonStationary.class)); + assertThat(((ChangeType.NonStationary) type).getTrend(), equalTo("increasing")); + } } diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml index 5f4dcbd416720..9f19e2e04d2ca 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml @@ -39,6 +39,8 @@ template: log.level: type: alias path: severity_text + event_name: + type: keyword body: type: object properties: diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml index 95a42b137df52..635ba386f739c 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml @@ -105,6 +105,7 @@ Event body: service.name: my-service attributes: event.name: foo + event_name: foo body: structured: foo: @@ -119,6 +120,7 @@ Event body: index: $datastream-backing-index - is_true: $datastream-backing-index - match: { .$datastream-backing-index.mappings.properties.body.properties.structured.properties.foo\.bar.type: "keyword" } + - match: { .$datastream-backing-index.mappings.properties.event_name.type: "keyword" } --- Structured log body: - do: diff --git a/x-pack/plugin/rank-vectors/build.gradle b/x-pack/plugin/rank-vectors/build.gradle new file mode 100644 index 0000000000000..53aabb8fdbf74 --- /dev/null +++ b/x-pack/plugin/rank-vectors/build.gradle @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +apply plugin: 'elasticsearch.internal-es-plugin' +apply plugin: 'elasticsearch.internal-cluster-test' + +esplugin { + name = 'rank-vectors' + description = 'Rank vectors in search.' + classname = 'org.elasticsearch.xpack.rank.vectors.RankVectorsPlugin' + extendedPlugins = ['x-pack-core', 'lang-painless'] +} + +dependencies { + compileOnly project(path: xpackModule('core')) + compileOnly(project(':modules:lang-painless:spi')) + + testImplementation(testArtifact(project(xpackModule('core')))) + testImplementation(testArtifact(project(':server'))) +} diff --git a/x-pack/plugin/rank-vectors/src/main/java/module-info.java b/x-pack/plugin/rank-vectors/src/main/java/module-info.java new file mode 100644 index 0000000000000..4af3c994edd38 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/java/module-info.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module org.elasticsearch.rank.vectors { + requires org.elasticsearch.xcore; + requires org.elasticsearch.painless.spi; + requires org.elasticsearch.server; + requires org.apache.lucene.core; + requires org.elasticsearch.xcontent; + + exports org.elasticsearch.xpack.rank.vectors; + exports org.elasticsearch.xpack.rank.vectors.mapper; + exports org.elasticsearch.xpack.rank.vectors.script; + + // whitelist resource access + opens org.elasticsearch.xpack.rank.vectors.script to org.elasticsearch.painless.spi; + + provides org.elasticsearch.painless.spi.PainlessExtension with org.elasticsearch.xpack.rank.vectors.script.RankVectorsPainlessExtension; + provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.xpack.rank.vectors.RankVectorsFeatures; + +} diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsFeatures.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsFeatures.java new file mode 100644 index 0000000000000..44b1b7a068860 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsFeatures.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.rank.vectors; + +import org.elasticsearch.features.FeatureSpecification; +import org.elasticsearch.features.NodeFeature; + +import java.util.Set; + +public class RankVectorsFeatures implements FeatureSpecification { + public static final NodeFeature RANK_VECTORS_FEATURE = new NodeFeature("rank_vectors"); + + @Override + public Set getTestFeatures() { + return Set.of(RANK_VECTORS_FEATURE); + } + +} diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java new file mode 100644 index 0000000000000..35c87f1fc1847 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/RankVectorsPlugin.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.rank.vectors; + +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.license.License; +import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.license.LicensedFeature; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsFieldMapper; + +import java.util.Map; + +import static org.elasticsearch.index.mapper.FieldMapper.notInMultiFields; +import static org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsFieldMapper.CONTENT_TYPE; + +public class RankVectorsPlugin extends Plugin implements MapperPlugin { + public static final LicensedFeature.Momentary RANK_VECTORS_FEATURE = LicensedFeature.momentary( + null, + "rank-vectors", + License.OperationMode.ENTERPRISE + ); + + @Override + public Map getMappers() { + return Map.of(CONTENT_TYPE, new FieldMapper.TypeParser((n, c) -> { + if (RANK_VECTORS_FEATURE.check(getLicenseState()) == false) { + throw LicenseUtils.newComplianceException("Rank Vectors"); + } + return new RankVectorsFieldMapper.Builder(n, c.indexVersionCreated(), getLicenseState()); + }, notInMultiFields(CONTENT_TYPE))); + } + + protected XPackLicenseState getLicenseState() { + return XPackPlugin.getSharedLicenseState(); + } +} diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java new file mode 100644 index 0000000000000..b858b935c1483 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.rank.vectors.mapper; + +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.fielddata.FormattedDocValues; +import org.elasticsearch.index.fielddata.LeafFieldData; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.script.field.DocValuesScriptFieldFactory; +import org.elasticsearch.script.field.vectors.BitRankVectorsDocValuesField; +import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField; +import org.elasticsearch.script.field.vectors.FloatRankVectorsDocValuesField; +import org.elasticsearch.script.field.vectors.VectorIterator; +import org.elasticsearch.search.DocValueFormat; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +final class RankVectorsDVLeafFieldData implements LeafFieldData { + private final LeafReader reader; + private final String field; + private final DenseVectorFieldMapper.ElementType elementType; + private final int dims; + + RankVectorsDVLeafFieldData(LeafReader reader, String field, DenseVectorFieldMapper.ElementType elementType, int dims) { + this.reader = reader; + this.field = field; + this.elementType = elementType; + this.dims = dims; + } + + @Override + public FormattedDocValues getFormattedValues(DocValueFormat format) { + int dims = elementType == DenseVectorFieldMapper.ElementType.BIT ? this.dims / Byte.SIZE : this.dims; + return switch (elementType) { + case BYTE, BIT -> new FormattedDocValues() { + private final byte[] vector = new byte[dims]; + private BytesRef ref = null; + private int numVecs = -1; + private final BinaryDocValues binary; + { + try { + binary = DocValues.getBinary(reader, field); + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } + + @Override + public boolean advanceExact(int docId) throws IOException { + if (binary == null || binary.advanceExact(docId) == false) { + return false; + } + ref = binary.binaryValue(); + assert ref.length % dims == 0; + numVecs = ref.length / dims; + return true; + } + + @Override + public int docValueCount() { + return 1; + } + + public Object nextValue() { + // Boxed to keep from `byte[]` being transformed into a string + List vectors = new ArrayList<>(numVecs); + VectorIterator iterator = new ByteRankVectorsDocValuesField.ByteVectorIterator(ref, vector, numVecs); + while (iterator.hasNext()) { + byte[] v = iterator.next(); + Byte[] vec = new Byte[dims]; + for (int i = 0; i < dims; i++) { + vec[i] = v[i]; + } + vectors.add(vec); + } + return vectors; + } + }; + case FLOAT -> new FormattedDocValues() { + private final float[] vector = new float[dims]; + private BytesRef ref = null; + private int numVecs = -1; + private final BinaryDocValues binary; + { + try { + binary = DocValues.getBinary(reader, field); + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } + + @Override + public boolean advanceExact(int docId) throws IOException { + if (binary == null || binary.advanceExact(docId) == false) { + return false; + } + ref = binary.binaryValue(); + assert ref.length % (Float.BYTES * dims) == 0; + numVecs = ref.length / (Float.BYTES * dims); + return true; + } + + @Override + public int docValueCount() { + return 1; + } + + @Override + public Object nextValue() { + List vectors = new ArrayList<>(numVecs); + VectorIterator iterator = new FloatRankVectorsDocValuesField.FloatVectorIterator(ref, vector, numVecs); + while (iterator.hasNext()) { + float[] v = iterator.next(); + vectors.add(Arrays.copyOf(v, v.length)); + } + return vectors; + } + }; + }; + } + + @Override + public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { + try { + BinaryDocValues values = DocValues.getBinary(reader, field); + BinaryDocValues magnitudeValues = DocValues.getBinary(reader, field + RankVectorsFieldMapper.VECTOR_MAGNITUDES_SUFFIX); + return switch (elementType) { + case BYTE -> new ByteRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims); + case FLOAT -> new FloatRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims); + case BIT -> new BitRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims); + }; + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values for multi-vector field!", e); + } + } + + @Override + public SortedBinaryDocValues getBytesValues() { + throw new UnsupportedOperationException("String representation of doc values for multi-vector fields is not supported"); + } + + @Override + public long ramBytesUsed() { + return 0; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapper.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java similarity index 88% rename from server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapper.java rename to x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java index d57dbf79b450c..873d67e76b04a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapper.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.index.mapper.vectors; +package org.elasticsearch.xpack.rank.vectors.mapper; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.index.BinaryDocValues; @@ -15,7 +13,6 @@ import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -31,7 +28,10 @@ import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.vectors.VectorData; @@ -50,11 +50,11 @@ import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT_BIT; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.namesToElementType; +import static org.elasticsearch.xpack.rank.vectors.RankVectorsPlugin.RANK_VECTORS_FEATURE; public class RankVectorsFieldMapper extends FieldMapper { public static final String VECTOR_MAGNITUDES_SUFFIX = "._magnitude"; - public static final FeatureFlag FEATURE_FLAG = new FeatureFlag("rank_vectors"); public static final String CONTENT_TYPE = "rank_vectors"; private static RankVectorsFieldMapper toType(FieldMapper in) { @@ -111,10 +111,12 @@ public static class Builder extends FieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); private final IndexVersion indexCreatedVersion; + private final XPackLicenseState licenseState; - public Builder(String name, IndexVersion indexCreatedVersion) { + public Builder(String name, IndexVersion indexCreatedVersion, XPackLicenseState licenseState) { super(name); this.indexCreatedVersion = indexCreatedVersion; + this.licenseState = licenseState; } @Override @@ -122,18 +124,12 @@ protected Parameter[] getParameters() { return new Parameter[] { elementType, dims, meta }; } - public RankVectorsFieldMapper.Builder dimensions(int dimensions) { - this.dims.setValue(dimensions); - return this; - } - - public RankVectorsFieldMapper.Builder elementType(DenseVectorFieldMapper.ElementType elementType) { - this.elementType.setValue(elementType); - return this; - } - @Override public RankVectorsFieldMapper build(MapperBuilderContext context) { + // Validate on Mapping creation + if (RANK_VECTORS_FEATURE.check(licenseState) == false) { + throw LicenseUtils.newComplianceException("Rank Vectors"); + } // Validate again here because the dimensions or element type could have been set programmatically, // which affects index option validity validate(); @@ -143,36 +139,32 @@ public RankVectorsFieldMapper build(MapperBuilderContext context) { context.buildFullName(leafName()), elementType.getValue(), dims.getValue(), - indexCreatedVersion, + licenseState, meta.getValue() ), builderParams(this, context), - indexCreatedVersion + indexCreatedVersion, + licenseState ); } } - public static final TypeParser PARSER = new TypeParser( - (n, c) -> new RankVectorsFieldMapper.Builder(n, c.indexVersionCreated()), - notInMultiFields(CONTENT_TYPE) - ); - public static final class RankVectorsFieldType extends SimpleMappedFieldType { private final DenseVectorFieldMapper.ElementType elementType; private final Integer dims; - private final IndexVersion indexCreatedVersion; + private final XPackLicenseState licenseState; public RankVectorsFieldType( String name, DenseVectorFieldMapper.ElementType elementType, Integer dims, - IndexVersion indexCreatedVersion, + XPackLicenseState licenseState, Map meta ) { super(name, false, false, true, TextSearchInfo.NONE, meta); this.elementType = elementType; this.dims = dims; - this.indexCreatedVersion = indexCreatedVersion; + this.licenseState = licenseState; } @Override @@ -195,9 +187,7 @@ protected Object parseSourceValue(Object value) { @Override public DocValueFormat docValueFormat(String format, ZoneId timeZone) { - throw new IllegalArgumentException( - "Field [" + name() + "] of type [" + typeName() + "] doesn't support docvalue_fields or aggregations" - ); + return DocValueFormat.DENSE_VECTOR; } @Override @@ -207,7 +197,10 @@ public boolean isAggregatable() { @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { - return new RankVectorsIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, indexCreatedVersion, dims, elementType); + if (RANK_VECTORS_FEATURE.check(licenseState) == false) { + throw LicenseUtils.newComplianceException("Rank Vectors"); + } + return new RankVectorsIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, dims, elementType); } @Override @@ -230,10 +223,18 @@ DenseVectorFieldMapper.ElementType getElementType() { } private final IndexVersion indexCreatedVersion; - - private RankVectorsFieldMapper(String simpleName, MappedFieldType fieldType, BuilderParams params, IndexVersion indexCreatedVersion) { + private final XPackLicenseState licenseState; + + private RankVectorsFieldMapper( + String simpleName, + MappedFieldType fieldType, + BuilderParams params, + IndexVersion indexCreatedVersion, + XPackLicenseState licenseState + ) { super(simpleName, fieldType, params); this.indexCreatedVersion = indexCreatedVersion; + this.licenseState = licenseState; } @Override @@ -248,6 +249,9 @@ public boolean parsesArrayValue() { @Override public void parse(DocumentParserContext context) throws IOException { + if (RANK_VECTORS_FEATURE.check(licenseState) == false) { + throw LicenseUtils.newComplianceException("Rank Vectors"); + } if (context.doc().getByKey(fieldType().name()) != null) { throw new IllegalArgumentException( "Field [" @@ -281,10 +285,10 @@ public void parse(DocumentParserContext context) throws IOException { fieldType().name(), fieldType().elementType, currentDims, - indexCreatedVersion, + licenseState, fieldType().meta() ); - Mapper update = new RankVectorsFieldMapper(leafName(), updatedFieldType, builderParams, indexCreatedVersion); + Mapper update = new RankVectorsFieldMapper(leafName(), updatedFieldType, builderParams, indexCreatedVersion, licenseState); context.addDynamicMapper(update); return; } @@ -366,12 +370,12 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new RankVectorsFieldMapper.Builder(leafName(), indexCreatedVersion).init(this); + return new Builder(leafName(), indexCreatedVersion, licenseState).init(this); } @Override protected SyntheticSourceSupport syntheticSourceSupport() { - return new SyntheticSourceSupport.Native(new RankVectorsFieldMapper.DocValuesSyntheticFieldLoader()); + return new SyntheticSourceSupport.Native(new DocValuesSyntheticFieldLoader()); } private class DocValuesSyntheticFieldLoader extends SourceLoader.DocValuesBasedSyntheticFieldLoader { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsIndexFieldData.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsIndexFieldData.java similarity index 76% rename from server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsIndexFieldData.java rename to x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsIndexFieldData.java index 7f54d2b9a8ad8..7ec426b904ce0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsIndexFieldData.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsIndexFieldData.java @@ -1,20 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.index.mapper.vectors; +package org.elasticsearch.xpack.rank.vectors.mapper; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.SortField; import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; @@ -26,19 +24,16 @@ public class RankVectorsIndexFieldData implements IndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { - return new RankVectorsIndexFieldData(name, dims, valuesSourceType, indexVersion, elementType); + return new RankVectorsIndexFieldData(name, dims, valuesSourceType, elementType); } } } diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsPainlessExtension.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsPainlessExtension.java new file mode 100644 index 0000000000000..5893452f5fdf8 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsPainlessExtension.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.rank.vectors.script; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScoreScript; +import org.elasticsearch.script.ScriptContext; + +import java.util.List; +import java.util.Map; + +public class RankVectorsPainlessExtension implements PainlessExtension { + private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles( + RankVectorsPainlessExtension.class, + "rank_vector_whitelist.txt" + ); + + @Override + public Map, List> getContextWhitelists() { + return Map.of(ScoreScript.CONTEXT, List.of(WHITELIST)); + } +} diff --git a/server/src/main/java/org/elasticsearch/script/RankVectorsScoreScriptUtils.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java similarity index 97% rename from server/src/main/java/org/elasticsearch/script/RankVectorsScoreScriptUtils.java rename to x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java index 2d11641cb5aa7..3d692db08ff9e 100644 --- a/server/src/main/java/org/elasticsearch/script/RankVectorsScoreScriptUtils.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java @@ -1,16 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.script; +package org.elasticsearch.xpack.rank.vectors.script; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.script.ScoreScript; import org.elasticsearch.script.field.vectors.DenseVector; import org.elasticsearch.script.field.vectors.RankVectorsDocValuesField; diff --git a/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification new file mode 100644 index 0000000000000..388dbe7dead96 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification @@ -0,0 +1,8 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. +# + +org.elasticsearch.xpack.rank.vectors.RankVectorsFeatures diff --git a/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension new file mode 100644 index 0000000000000..06c77434c77a6 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -0,0 +1 @@ +org.elasticsearch.xpack.rank.vectors.script.RankVectorsPainlessExtension diff --git a/x-pack/plugin/rank-vectors/src/main/resources/org/elasticsearch/xpack/rank/vectors/script/rank_vector_whitelist.txt b/x-pack/plugin/rank-vectors/src/main/resources/org/elasticsearch/xpack/rank/vectors/script/rank_vector_whitelist.txt new file mode 100644 index 0000000000000..ce3d04f73dea8 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/main/resources/org/elasticsearch/xpack/rank/vectors/script/rank_vector_whitelist.txt @@ -0,0 +1,13 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. +# + +# This file contains a rank vectors whitelist for functions to be used in Score context + +static_import { + double maxSimDotProduct(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.xpack.rank.vectors.script.RankVectorsScoreScriptUtils$MaxSimDotProduct + double maxSimInvHamming(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.xpack.rank.vectors.script.RankVectorsScoreScriptUtils$MaxSimInvHamming +} diff --git a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/LocalStateRankVectors.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/LocalStateRankVectors.java new file mode 100644 index 0000000000000..ba085b8f0d705 --- /dev/null +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/LocalStateRankVectors.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.rank.vectors; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.license.License; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.license.internal.XPackLicenseStatus; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; + +import java.util.Map; + +public class LocalStateRankVectors extends LocalStateCompositeXPackPlugin { + + private final RankVectorsPlugin rankVectorsPlugin; + private final XPackLicenseState licenseState = new XPackLicenseState( + System::currentTimeMillis, + new XPackLicenseStatus(License.OperationMode.TRIAL, true, null) + ); + + public LocalStateRankVectors(Settings settings) { + super(settings, null); + LocalStateRankVectors thisVar = this; + rankVectorsPlugin = new RankVectorsPlugin() { + @Override + protected XPackLicenseState getLicenseState() { + return licenseState; + } + }; + } + + @Override + public Map getMappers() { + return rankVectorsPlugin.getMappers(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapperTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java similarity index 97% rename from server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapperTests.java rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java index e81c28cbcc444..25264976c58a5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapperTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.index.mapper.vectors; +package org.elasticsearch.xpack.rank.vectors.mapper; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.index.IndexableField; @@ -29,18 +27,21 @@ import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.search.lookup.SourceProvider; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.rank.vectors.LocalStateRankVectors; import org.junit.AssumptionViolatedException; -import org.junit.BeforeClass; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.FloatBuffer; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Stream; @@ -53,11 +54,6 @@ public class RankVectorsFieldMapperTests extends MapperTestCase { - @BeforeClass - public static void setup() { - assumeTrue("Requires rank vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); - } - private final ElementType elementType; private final int dims; @@ -66,6 +62,11 @@ public RankVectorsFieldMapperTests() { this.dims = ElementType.BIT == elementType ? 4 * Byte.SIZE : 4; } + @Override + protected Collection getPlugins() { + return Collections.singletonList(new LocalStateRankVectors(SETTINGS)); + } + @Override protected void minimalMapping(XContentBuilder b) throws IOException { indexMapping(b, IndexVersion.current()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldTypeTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java similarity index 67% rename from server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldTypeTests.java rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java index b4cbbc4730d7c..59c5b0414910d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldTypeTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldTypeTests.java @@ -1,20 +1,21 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.index.mapper.vectors; +package org.elasticsearch.xpack.rank.vectors.mapper; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper.RankVectorsFieldType; -import org.junit.BeforeClass; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.license.License; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.license.internal.XPackLicenseStatus; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsFieldMapper.RankVectorsFieldType; import java.io.IOException; import java.util.Collections; @@ -25,23 +26,17 @@ public class RankVectorsFieldTypeTests extends FieldTypeTestCase { - @BeforeClass - public static void setup() { - assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); - } + private final XPackLicenseState licenseState = new XPackLicenseState( + System::currentTimeMillis, + new XPackLicenseStatus(License.OperationMode.TRIAL, true, null) + ); private RankVectorsFieldType createFloatFieldType() { - return new RankVectorsFieldType( - "f", - DenseVectorFieldMapper.ElementType.FLOAT, - BBQ_MIN_DIMS, - IndexVersion.current(), - Collections.emptyMap() - ); + return new RankVectorsFieldType("f", DenseVectorFieldMapper.ElementType.FLOAT, BBQ_MIN_DIMS, licenseState, Collections.emptyMap()); } - private RankVectorsFieldType createByteFieldType() { - return new RankVectorsFieldType("f", DenseVectorFieldMapper.ElementType.BYTE, 5, IndexVersion.current(), Collections.emptyMap()); + private RankVectorsFieldMapper.RankVectorsFieldType createByteFieldType() { + return new RankVectorsFieldType("f", DenseVectorFieldMapper.ElementType.BYTE, 5, licenseState, Collections.emptyMap()); } public void testHasDocValues() { @@ -84,9 +79,9 @@ public void testFielddataBuilder() { public void testDocValueFormat() { RankVectorsFieldType fft = createFloatFieldType(); - expectThrows(IllegalArgumentException.class, () -> fft.docValueFormat(null, null)); + assertEquals(DocValueFormat.DENSE_VECTOR, fft.docValueFormat(null, null)); RankVectorsFieldType bft = createByteFieldType(); - expectThrows(IllegalArgumentException.class, () -> bft.docValueFormat(null, null)); + assertEquals(DocValueFormat.DENSE_VECTOR, bft.docValueFormat(null, null)); } public void testFetchSourceValue() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsScriptDocValuesTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsScriptDocValuesTests.java similarity index 95% rename from server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsScriptDocValuesTests.java rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsScriptDocValuesTests.java index c38ed0f60f0ae..f0b00849557bd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsScriptDocValuesTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsScriptDocValuesTests.java @@ -1,24 +1,22 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.index.mapper.vectors; +package org.elasticsearch.xpack.rank.vectors.mapper; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; +import org.elasticsearch.index.mapper.vectors.RankVectorsScriptDocValues; import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField; import org.elasticsearch.script.field.vectors.FloatRankVectorsDocValuesField; import org.elasticsearch.script.field.vectors.RankVectors; import org.elasticsearch.script.field.vectors.RankVectorsDocValuesField; import org.elasticsearch.test.ESTestCase; -import org.junit.BeforeClass; import java.io.IOException; import java.nio.ByteBuffer; @@ -29,11 +27,6 @@ public class RankVectorsScriptDocValuesTests extends ESTestCase { - @BeforeClass - public static void setup() { - assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); - } - public void testFloatGetVectorValueAndGetMagnitude() throws IOException { int dims = 3; float[][][] vectors = { { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } }, { { 1, 0, 2 } } }; diff --git a/server/src/test/java/org/elasticsearch/script/RankVectorsScoreScriptUtilsTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtilsTests.java similarity index 93% rename from server/src/test/java/org/elasticsearch/script/RankVectorsScoreScriptUtilsTests.java rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtilsTests.java index 917cc2069a293..da0340a22c074 100644 --- a/server/src/test/java/org/elasticsearch/script/RankVectorsScoreScriptUtilsTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtilsTests.java @@ -1,26 +1,23 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.script; +package org.elasticsearch.xpack.rank.vectors.script; import org.apache.lucene.util.VectorUtil; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; -import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; -import org.elasticsearch.index.mapper.vectors.RankVectorsScriptDocValuesTests; -import org.elasticsearch.script.RankVectorsScoreScriptUtils.MaxSimDotProduct; -import org.elasticsearch.script.RankVectorsScoreScriptUtils.MaxSimInvHamming; +import org.elasticsearch.script.ScoreScript; import org.elasticsearch.script.field.vectors.BitRankVectorsDocValuesField; import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField; import org.elasticsearch.script.field.vectors.FloatRankVectorsDocValuesField; import org.elasticsearch.script.field.vectors.RankVectorsDocValuesField; import org.elasticsearch.test.ESTestCase; -import org.junit.BeforeClass; +import org.elasticsearch.xpack.rank.vectors.mapper.RankVectorsScriptDocValuesTests; +import org.elasticsearch.xpack.rank.vectors.script.RankVectorsScoreScriptUtils.MaxSimDotProduct; +import org.elasticsearch.xpack.rank.vectors.script.RankVectorsScoreScriptUtils.MaxSimInvHamming; import java.io.IOException; import java.util.Arrays; @@ -33,11 +30,6 @@ public class RankVectorsScoreScriptUtilsTests extends ESTestCase { - @BeforeClass - public static void setup() { - assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); - } - public void testFloatMultiVectorClassBindings() throws IOException { String fieldName = "vector"; int dims = 5; @@ -88,7 +80,7 @@ public void testFloatMultiVectorClassBindings() throws IOException { // Check each function rejects query vectors with the wrong dimension IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> new RankVectorsScoreScriptUtils.MaxSimDotProduct(scoreScript, invalidQueryVector, fieldName) + () -> new MaxSimDotProduct(scoreScript, invalidQueryVector, fieldName) ); assertThat( e.getMessage(), @@ -336,7 +328,4 @@ public void testByteBoundaries() throws IOException { } } - public void testDimMismatch() throws IOException { - - } } diff --git a/server/src/test/java/org/elasticsearch/script/field/vectors/RankVectorsTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsTests.java similarity index 80% rename from server/src/test/java/org/elasticsearch/script/field/vectors/RankVectorsTests.java rename to x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsTests.java index ca7608b10aed9..570e2a59926aa 100644 --- a/server/src/test/java/org/elasticsearch/script/field/vectors/RankVectorsTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsTests.java @@ -1,19 +1,19 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.script.field.vectors; +package org.elasticsearch.xpack.rank.vectors.script; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.VectorUtil; -import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; +import org.elasticsearch.script.field.vectors.ByteRankVectors; +import org.elasticsearch.script.field.vectors.FloatRankVectors; +import org.elasticsearch.script.field.vectors.RankVectors; +import org.elasticsearch.script.field.vectors.VectorIterator; import org.elasticsearch.test.ESTestCase; -import org.junit.BeforeClass; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -21,11 +21,6 @@ public class RankVectorsTests extends ESTestCase { - @BeforeClass - public static void setup() { - assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); - } - public void testByteUnsupported() { int count = randomIntBetween(1, 16); int dims = randomIntBetween(1, 16); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java index 2353d710059ff..df6189351162f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java @@ -528,7 +528,12 @@ private static boolean isChildActionAuthorizedByParentOnLocalNode(RequestInfo re + Arrays.stream(indices).filter(Regex::isSimpleMatchPattern).toList(); // Check if the parent context has already successfully authorized access to the child's indices - return Arrays.stream(indices).allMatch(indicesAccessControl::hasIndexPermissions); + for (String index : indices) { + if (indicesAccessControl.hasIndexPermissions(index) == false) { + return false; + } + } + return true; } @Override diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/slowlog/SecuritySlowLogFieldProvider.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/slowlog/SecuritySlowLogFieldProvider.java index 1610aedd1d363..b5327b6779656 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/slowlog/SecuritySlowLogFieldProvider.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/slowlog/SecuritySlowLogFieldProvider.java @@ -9,6 +9,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.SlowLogFieldProvider; +import org.elasticsearch.index.SlowLogFields; import org.elasticsearch.xpack.security.Security; import java.util.Map; @@ -18,8 +19,36 @@ public class SecuritySlowLogFieldProvider implements SlowLogFieldProvider { private final Security plugin; - private boolean includeUserInIndexing = false; - private boolean includeUserInSearch = false; + + private class SecuritySlowLogFields implements SlowLogFields { + private boolean includeUserInIndexing = false; + private boolean includeUserInSearch = false; + + SecuritySlowLogFields(IndexSettings indexSettings) { + indexSettings.getScopedSettings() + .addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_INCLUDE_USER_SETTING, newValue -> this.includeUserInSearch = newValue); + this.includeUserInSearch = indexSettings.getValue(INDEX_SEARCH_SLOWLOG_INCLUDE_USER_SETTING); + indexSettings.getScopedSettings() + .addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_INCLUDE_USER_SETTING, newValue -> this.includeUserInIndexing = newValue); + this.includeUserInIndexing = indexSettings.getValue(INDEX_INDEXING_SLOWLOG_INCLUDE_USER_SETTING); + } + + @Override + public Map indexFields() { + if (includeUserInIndexing) { + return plugin.getAuthContextForSlowLog(); + } + return Map.of(); + } + + @Override + public Map searchFields() { + if (includeUserInSearch) { + return plugin.getAuthContextForSlowLog(); + } + return Map.of(); + } + } public SecuritySlowLogFieldProvider() { throw new IllegalStateException("Provider must be constructed using PluginsService"); @@ -30,28 +59,7 @@ public SecuritySlowLogFieldProvider(Security plugin) { } @Override - public void init(IndexSettings indexSettings) { - indexSettings.getScopedSettings() - .addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_INCLUDE_USER_SETTING, newValue -> this.includeUserInSearch = newValue); - this.includeUserInSearch = indexSettings.getValue(INDEX_SEARCH_SLOWLOG_INCLUDE_USER_SETTING); - indexSettings.getScopedSettings() - .addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_INCLUDE_USER_SETTING, newValue -> this.includeUserInIndexing = newValue); - this.includeUserInIndexing = indexSettings.getValue(INDEX_INDEXING_SLOWLOG_INCLUDE_USER_SETTING); - } - - @Override - public Map indexSlowLogFields() { - if (includeUserInIndexing) { - return plugin.getAuthContextForSlowLog(); - } - return Map.of(); - } - - @Override - public Map searchSlowLogFields() { - if (includeUserInSearch) { - return plugin.getAuthContextForSlowLog(); - } - return Map.of(); + public SlowLogFields create(IndexSettings indexSettings) { + return new SecuritySlowLogFields(indexSettings); } } diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/GCSRepositoryAnalysisRestIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/GCSRepositoryAnalysisRestIT.java index 7f7540d138825..01ca881f9a119 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/GCSRepositoryAnalysisRestIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/GCSRepositoryAnalysisRestIT.java @@ -46,12 +46,6 @@ public class GCSRepositoryAnalysisRestIT extends AbstractRepositoryAnalysisRestT ); } }) - .apply(c -> { - if (USE_FIXTURE) { - // test fixture does not support CAS yet; TODO fix this - c.systemProperty("test.repository_test_kit.skip_cas", "true"); - } - }) .build(); @ClassRule diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_rank_vectors.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml similarity index 92% rename from rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_rank_vectors.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml index ecf34f46c3383..791712ee925a5 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_rank_vectors.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml @@ -1,11 +1,7 @@ setup: - requires: - capabilities: - - method: POST - path: /_search - capabilities: [ rank_vectors_field_mapper ] - test_runner_features: capabilities - reason: "Support for rank vectors field mapper capability required" + cluster_features: [ "rank_vectors" ] + reason: "requires rank_vectors feature" --- "Test create multi-vector field": - do: diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_rank_vectors_dv_fields_api.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_dv_fields_api.yml similarity index 71% rename from modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_rank_vectors_dv_fields_api.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_dv_fields_api.yml index f37e554fca7bf..bcfb9bcd79b90 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_rank_vectors_dv_fields_api.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_dv_fields_api.yml @@ -1,11 +1,7 @@ setup: - requires: - capabilities: - - method: POST - path: /_search - capabilities: [ rank_vectors_script_access ] - test_runner_features: capabilities - reason: "Support for rank vector field script access capability required" + cluster_features: [ "rank_vectors" ] + reason: "requires rank_vectors feature" - skip: features: headers @@ -17,6 +13,8 @@ setup: number_of_shards: 1 mappings: properties: + id: + type: keyword vector: type: rank_vectors dims: 5 @@ -33,6 +31,7 @@ setup: index: test-index id: "1" body: + id: "1" vector: [[230.0, 300.33, -34.8988, 15.555, -200.0], [-0.5, 100.0, -13, 14.8, -156.0]] byte_vector: [[8, 5, -15, 1, -7], [-1, 115, -3, 4, -128]] bit_vector: [[8, 5, -15, 1, -7], [-1, 115, -3, 4, -128]] @@ -42,6 +41,7 @@ setup: index: test-index id: "3" body: + id: "3" vector: [[0.5, 111.3, -13.0, 14.8, -156.0]] byte_vector: [[2, 18, -5, 0, -124]] bit_vector: [[2, 18, -5, 0, -124]] @@ -176,3 +176,35 @@ setup: - match: {hits.hits.1._id: "3"} - close_to: {hits.hits.1._score: {value: 2, error: 0.01}} +--- +"Test doc value vector access": + - skip: + features: close_to + + - do: + search: + _source: false + index: test-index + body: + docvalue_fields: [name, bit_vector, byte_vector, vector] + sort: [{id: "asc"}] + + - match: {hits.hits.0._id: "1"} + # vector: [[230.0, 300.33, -34.8988, 15.555, -200.0], [-0.5, 100.0, -13, 14.8, -156.0]] + - close_to: {hits.hits.0.fields.vector.0.0.0: {value: 230, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.0.1: {value: 300.33, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.0.2: {value: -34.8988, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.0.3: {value: 15.555, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.0.4: {value: -200, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.1.0: {value: -0.5, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.1.1: {value: 100, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.1.2: {value: -13, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.1.3: {value: 14.8, error: 0.0001}} + - close_to: {hits.hits.0.fields.vector.0.1.4: {value: -156, error: 0.0001}} + + # byte_vector: [[8, 5, -15, 1, -7], [-1, 115, -3, 4, -128]] + - match: {hits.hits.0.fields.byte_vector.0.0: [8, 5, -15, 1, -7]} + - match: {hits.hits.0.fields.bit_vector.0.0: [8, 5, -15, 1, -7]} + - match: {hits.hits.0.fields.byte_vector.0.1: [-1, 115, -3, 4, -128]} + - match: {hits.hits.0.fields.bit_vector.0.1: [-1, 115, -3, 4, -128]} + diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_rank_vectors_max_sim.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_max_sim.yml similarity index 95% rename from modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_rank_vectors_max_sim.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_max_sim.yml index 7c46fbc9a26a5..acaf1b99b626e 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_rank_vectors_max_sim.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors_max_sim.yml @@ -1,11 +1,7 @@ setup: - requires: - capabilities: - - method: POST - path: /_search - capabilities: [ rank_vectors_script_max_sim_with_bugfix ] - test_runner_features: capabilities - reason: "Support for rank vectors max-sim functions capability required" + cluster_features: [ "rank_vectors" ] + reason: "requires rank_vectors feature" - skip: features: headers diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 108efaa0f7691..0b31e96ece84a 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -1093,7 +1093,7 @@ protected final SearchExecutionContext createMockContext() { IndexFieldData.Builder builder = fieldType.fielddataBuilder(fdc); return builder.build(new IndexFieldDataCache.None(), null); }; - MappingLookup lookup = MappingLookup.fromMapping(Mapping.EMPTY, null); + MappingLookup lookup = MappingLookup.fromMapping(Mapping.EMPTY); return new SearchExecutionContext( 0, 0, diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5IT.java new file mode 100644 index 0000000000000..3e371b5128b6a --- /dev/null +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5IT.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.oldrepos.searchablesnapshot; + +import org.elasticsearch.test.cluster.util.Version; + +public class MountFromVersion5IT extends SearchableSnapshotTestCase { + + public MountFromVersion5IT(Version version) { + super(version); + } + + public void testSearchableSnapshot() throws Exception { + verifyCompatibility("5"); + } +} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6IT.java new file mode 100644 index 0000000000000..29b81fe595e5f --- /dev/null +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6IT.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.oldrepos.searchablesnapshot; + +import org.elasticsearch.test.cluster.util.Version; + +public class MountFromVersion6IT extends SearchableSnapshotTestCase { + + public MountFromVersion6IT(Version version) { + super(version); + } + + public void testSearchableSnapshot() throws Exception { + verifyCompatibility("6"); + } +} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/SearchableSnapshotTestCase.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/SearchableSnapshotTestCase.java new file mode 100644 index 0000000000000..08a5db2111904 --- /dev/null +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/SearchableSnapshotTestCase.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.oldrepos.searchablesnapshot; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.oldrepos.AbstractUpgradeCompatibilityTestCase; +import org.elasticsearch.test.cluster.util.Version; + +import static org.elasticsearch.test.rest.ObjectPath.createFromResponse; + +/** + * Test suite for Archive indices backward compatibility with N-2 versions. + * The test suite creates a cluster in the N-1 version, where N is the current version. + * Restores snapshots from old-clusters (version 5/6) and upgrades it to the current version. + * Test methods are executed after each upgrade. + */ +public class SearchableSnapshotTestCase extends AbstractUpgradeCompatibilityTestCase { + + static { + clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial"); + } + + public SearchableSnapshotTestCase(Version version) { + super(version); + } + + /** + * Overrides the snapshot-restore operation for archive-indices scenario. + */ + @Override + public void recover(RestClient client, String repository, String snapshot, String index) throws Exception { + var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_mount"); + request.addParameter("wait_for_completion", "true"); + request.addParameter("storage", "full_copy"); + request.setJsonEntity(Strings.format(""" + { + "index": "%s", + "renamed_index": "%s" + }""", index, index)); + createFromResponse(client.performRequest(request)); + } +}