diff --git a/.buildkite/pipelines/ecs-dynamic-template-tests.yml b/.buildkite/pipelines/ecs-dynamic-template-tests.yml index b1fe972b724f1..a8145c61a2d40 100644 --- a/.buildkite/pipelines/ecs-dynamic-template-tests.yml +++ b/.buildkite/pipelines/ecs-dynamic-template-tests.yml @@ -7,3 +7,8 @@ steps: image: family/elasticsearch-ubuntu-2004 diskSizeGb: 350 machineType: custom-32-98304 +notify: + - slack: "#es-delivery" + if: build.state == "failed" + - email: "logs-plus@elastic.co" + if: build.state == "failed" diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java index aa16523e38097..8cad100de27f4 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java @@ -141,7 +141,7 @@ private static Operator operator(String grouping, String op, String dataType) { }; return new HashAggregationOperator( List.of(supplier(op, dataType, groups.size()).groupingAggregatorFactory(AggregatorMode.SINGLE)), - () -> BlockHash.build(groups, BIG_ARRAYS, 16 * 1024), + () -> BlockHash.build(groups, BIG_ARRAYS, 16 * 1024, false), new DriverContext() ); } diff --git a/docs/changelog/98944.yaml b/docs/changelog/98944.yaml deleted file mode 100644 index 3fc02674cb870..0000000000000 --- a/docs/changelog/98944.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 98944 -summary: Auto-normalize `dot_product` vectors at index & query -area: Vector Search -type: enhancement -issues: [] diff --git a/docs/changelog/99223.yaml b/docs/changelog/99223.yaml new file mode 100644 index 0000000000000..914441931033b --- /dev/null +++ b/docs/changelog/99223.yaml @@ -0,0 +1,11 @@ +pr: 99223 +summary: Remove `transport_versions` from cluster state API +area: Infra/Core +type: breaking +issues: [] +breaking: + title: Remove `transport_versions` from cluster state API + area: REST API + details: The `transport_versions` subobject of the response to `GET _cluster/state` has been replaced by the `nodes_versions` subobject. + impact: If needed, retrieve the per-node transport versions from the `nodes_versions` subobject. + notable: false diff --git a/docs/changelog/99231.yaml b/docs/changelog/99231.yaml new file mode 100644 index 0000000000000..9f5dfa1137587 --- /dev/null +++ b/docs/changelog/99231.yaml @@ -0,0 +1,5 @@ +pr: 99231 +summary: Add manage permission for fleet managed threat intel indices +area: Authorization +type: enhancement +issues: [] diff --git a/docs/changelog/99434.yaml b/docs/changelog/99434.yaml new file mode 100644 index 0000000000000..b03bc4f3c9b41 --- /dev/null +++ b/docs/changelog/99434.yaml @@ -0,0 +1,5 @@ +pr: 99434 +summary: "ESQL: Disable optimizations with bad null handling" +area: ES|QL +type: bug +issues: [] diff --git a/docs/reference/esql/functions/auto_bucket.asciidoc b/docs/reference/esql/functions/auto_bucket.asciidoc index a61e8365716c5..99364c785a804 100644 --- a/docs/reference/esql/functions/auto_bucket.asciidoc +++ b/docs/reference/esql/functions/auto_bucket.asciidoc @@ -51,7 +51,7 @@ include::{esql-specs}/date.csv-spec[tag=auto_bucket_in_agg-result] |=== NOTE: `AUTO_BUCKET` does not create buckets that don't match any documents. That's -why the example above is missing `1985-02-01` and other dates. +why the example above is missing `1985-03-01` and other dates. ==== Numeric fields diff --git a/docs/reference/esql/functions/ceil.asciidoc b/docs/reference/esql/functions/ceil.asciidoc index b35ab6d68b4e3..d7446916560c7 100644 --- a/docs/reference/esql/functions/ceil.asciidoc +++ b/docs/reference/esql/functions/ceil.asciidoc @@ -1,7 +1,7 @@ [[esql-ceil]] === `CEIL` [.text-center] -image::esql/functions/signature/floor.svg[Embedded,opts=inline] +image::esql/functions/signature/ceil.svg[Embedded,opts=inline] Round a number up to the nearest integer. diff --git a/docs/reference/esql/functions/pow.asciidoc b/docs/reference/esql/functions/pow.asciidoc index 04ea23f5807ea..1c19e1af579fb 100644 --- a/docs/reference/esql/functions/pow.asciidoc +++ b/docs/reference/esql/functions/pow.asciidoc @@ -59,7 +59,7 @@ include::{esql-specs}/math.csv-spec[tag=powULOverrun-warning] include::{esql-specs}/math.csv-spec[tag=powULOverrun-result] |=== -If it is desired to protect against numerical overruns, use `to_double` on either of the arguments: +If it is desired to protect against numerical overruns, use `TO_DOUBLE` on either of the arguments: [source.merge.styled,esql] ---- diff --git a/docs/reference/esql/functions/signature/auto_bucket.svg b/docs/reference/esql/functions/signature/auto_bucket.svg new file mode 100644 index 0000000000000..8343661db064e --- /dev/null +++ b/docs/reference/esql/functions/signature/auto_bucket.svg @@ -0,0 +1 @@ +AUTO_BUCKET(arg1,arg2,arg3,arg4) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/date_extract.svg b/docs/reference/esql/functions/signature/date_extract.svg new file mode 100644 index 0000000000000..ec69633c02e8b --- /dev/null +++ b/docs/reference/esql/functions/signature/date_extract.svg @@ -0,0 +1 @@ +DATE_EXTRACT(arg1,arg2) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/date_parse.svg b/docs/reference/esql/functions/signature/date_parse.svg new file mode 100644 index 0000000000000..93c37ba3b389f --- /dev/null +++ b/docs/reference/esql/functions/signature/date_parse.svg @@ -0,0 +1 @@ +DATE_PARSE(arg1,arg2) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/is_finite.svg b/docs/reference/esql/functions/signature/is_finite.svg new file mode 100644 index 0000000000000..0ff65a876d21f --- /dev/null +++ b/docs/reference/esql/functions/signature/is_finite.svg @@ -0,0 +1 @@ +IS_FINITE(arg1) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/is_infinite.svg b/docs/reference/esql/functions/signature/is_infinite.svg new file mode 100644 index 0000000000000..aef9e3873c918 --- /dev/null +++ b/docs/reference/esql/functions/signature/is_infinite.svg @@ -0,0 +1 @@ +IS_INFINITE(arg1) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/mv_avg.svg b/docs/reference/esql/functions/signature/mv_avg.svg new file mode 100644 index 0000000000000..4fc02033e4fdb --- /dev/null +++ b/docs/reference/esql/functions/signature/mv_avg.svg @@ -0,0 +1 @@ +MV_AVG(arg1) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/mv_concat.svg b/docs/reference/esql/functions/signature/mv_concat.svg new file mode 100644 index 0000000000000..d12153de7241c --- /dev/null +++ b/docs/reference/esql/functions/signature/mv_concat.svg @@ -0,0 +1 @@ +MV_CONCAT(arg1,arg2) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/mv_count.svg b/docs/reference/esql/functions/signature/mv_count.svg new file mode 100644 index 0000000000000..23d1f3a9f5bea --- /dev/null +++ b/docs/reference/esql/functions/signature/mv_count.svg @@ -0,0 +1 @@ +MV_COUNT(arg1) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/mv_dedupe.svg b/docs/reference/esql/functions/signature/mv_dedupe.svg new file mode 100644 index 0000000000000..460dcae11e46c --- /dev/null +++ b/docs/reference/esql/functions/signature/mv_dedupe.svg @@ -0,0 +1 @@ +MV_DEDUPE(arg1) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/mv_max.svg b/docs/reference/esql/functions/signature/mv_max.svg new file mode 100644 index 0000000000000..aec9dbf82a445 --- /dev/null +++ b/docs/reference/esql/functions/signature/mv_max.svg @@ -0,0 +1 @@ +MV_MAX(arg1) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/mv_median.svg b/docs/reference/esql/functions/signature/mv_median.svg new file mode 100644 index 0000000000000..3fd5dd009b143 --- /dev/null +++ b/docs/reference/esql/functions/signature/mv_median.svg @@ -0,0 +1 @@ +MV_MEDIAN(arg1) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/mv_min.svg b/docs/reference/esql/functions/signature/mv_min.svg new file mode 100644 index 0000000000000..386057b5aa287 --- /dev/null +++ b/docs/reference/esql/functions/signature/mv_min.svg @@ -0,0 +1 @@ +MV_MIN(arg1) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/mv_sum.svg b/docs/reference/esql/functions/signature/mv_sum.svg new file mode 100644 index 0000000000000..ff0fd374025ac --- /dev/null +++ b/docs/reference/esql/functions/signature/mv_sum.svg @@ -0,0 +1 @@ +MV_SUM(arg1) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/starts_with.svg b/docs/reference/esql/functions/signature/starts_with.svg new file mode 100644 index 0000000000000..0a2d3a1ef0c36 --- /dev/null +++ b/docs/reference/esql/functions/signature/starts_with.svg @@ -0,0 +1 @@ +STARTS_WITH(arg1,arg2) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/to_string.svg b/docs/reference/esql/functions/signature/to_string.svg new file mode 100644 index 0000000000000..72fc676289d64 --- /dev/null +++ b/docs/reference/esql/functions/signature/to_string.svg @@ -0,0 +1 @@ +TO_STRING(v) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/to_version.svg b/docs/reference/esql/functions/signature/to_version.svg new file mode 100644 index 0000000000000..e6337280c2e8d --- /dev/null +++ b/docs/reference/esql/functions/signature/to_version.svg @@ -0,0 +1 @@ +TO_VERSION(v) \ No newline at end of file diff --git a/docs/reference/esql/functions/to_string.asciidoc b/docs/reference/esql/functions/to_string.asciidoc index d5a08a6398189..ce396b0493384 100644 --- a/docs/reference/esql/functions/to_string.asciidoc +++ b/docs/reference/esql/functions/to_string.asciidoc @@ -1,5 +1,8 @@ [[esql-to_string]] === `TO_STRING` +[.text-center] +image::esql/functions/signature/to_string.svg[Embedded,opts=inline] + Converts a field into a string. For example: [source.merge.styled,esql] @@ -23,3 +26,7 @@ include::{esql-specs}/string.csv-spec[tag=to_string_multivalue-result] |=== Alias: TO_STR + +Supported types: + +include::types/to_string.asciidoc[] diff --git a/docs/reference/esql/functions/to_version.asciidoc b/docs/reference/esql/functions/to_version.asciidoc index 7ca42b89aa6e6..c1822393eb32f 100644 --- a/docs/reference/esql/functions/to_version.asciidoc +++ b/docs/reference/esql/functions/to_version.asciidoc @@ -1,5 +1,8 @@ [[esql-to_version]] === `TO_VERSION` +[.text-center] +image::esql/functions/signature/to_version.svg[Embedded,opts=inline] + Converts an input string to a version value. For example: [source.merge.styled,esql] @@ -14,3 +17,7 @@ include::{esql-specs}/version.csv-spec[tag=to_version-result] The input can be a single- or multi-valued field or an expression. Alias: TO_VER + +Supported types: + +include::types/to_version.asciidoc[] diff --git a/docs/reference/esql/functions/types/auto_bucket.asciidoc b/docs/reference/esql/functions/types/auto_bucket.asciidoc new file mode 100644 index 0000000000000..d2f134b99fbb0 --- /dev/null +++ b/docs/reference/esql/functions/types/auto_bucket.asciidoc @@ -0,0 +1,5 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +arg1 | arg2 | arg3 | arg4 | result + +|=== diff --git a/docs/reference/esql/functions/types/date_extract.asciidoc b/docs/reference/esql/functions/types/date_extract.asciidoc new file mode 100644 index 0000000000000..57a83810d9b7c --- /dev/null +++ b/docs/reference/esql/functions/types/date_extract.asciidoc @@ -0,0 +1,5 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +arg1 | arg2 | result +datetime | keyword | long +|=== diff --git a/docs/reference/esql/functions/types/date_parse.asciidoc b/docs/reference/esql/functions/types/date_parse.asciidoc new file mode 100644 index 0000000000000..882f49522f553 --- /dev/null +++ b/docs/reference/esql/functions/types/date_parse.asciidoc @@ -0,0 +1,5 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +arg1 | arg2 | result +keyword | keyword | datetime +|=== diff --git a/docs/reference/esql/functions/types/is_finite.asciidoc b/docs/reference/esql/functions/types/is_finite.asciidoc new file mode 100644 index 0000000000000..0c555059004c1 --- /dev/null +++ b/docs/reference/esql/functions/types/is_finite.asciidoc @@ -0,0 +1,5 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +arg1 | result +double | boolean +|=== diff --git a/docs/reference/esql/functions/types/is_infinite.asciidoc b/docs/reference/esql/functions/types/is_infinite.asciidoc new file mode 100644 index 0000000000000..0c555059004c1 --- /dev/null +++ b/docs/reference/esql/functions/types/is_infinite.asciidoc @@ -0,0 +1,5 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +arg1 | result +double | boolean +|=== diff --git a/docs/reference/esql/functions/types/mv_avg.asciidoc b/docs/reference/esql/functions/types/mv_avg.asciidoc new file mode 100644 index 0000000000000..dd4f6b0725cc8 --- /dev/null +++ b/docs/reference/esql/functions/types/mv_avg.asciidoc @@ -0,0 +1,8 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +arg1 | result +double | double +integer | double +long | double +unsigned_long | double +|=== diff --git a/docs/reference/esql/functions/types/mv_concat.asciidoc b/docs/reference/esql/functions/types/mv_concat.asciidoc new file mode 100644 index 0000000000000..f1f744dbe4126 --- /dev/null +++ b/docs/reference/esql/functions/types/mv_concat.asciidoc @@ -0,0 +1,5 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +arg1 | arg2 | result +keyword | keyword | keyword +|=== diff --git a/docs/reference/esql/functions/types/mv_count.asciidoc b/docs/reference/esql/functions/types/mv_count.asciidoc new file mode 100644 index 0000000000000..2fcdfc65fa63b --- /dev/null +++ b/docs/reference/esql/functions/types/mv_count.asciidoc @@ -0,0 +1,10 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +arg1 | result +boolean | integer +double | integer +integer | integer +keyword | integer +long | integer +unsigned_long | integer +|=== diff --git a/docs/reference/esql/functions/types/mv_dedupe.asciidoc b/docs/reference/esql/functions/types/mv_dedupe.asciidoc new file mode 100644 index 0000000000000..4e12c68422662 --- /dev/null +++ b/docs/reference/esql/functions/types/mv_dedupe.asciidoc @@ -0,0 +1,9 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +arg1 | result +boolean | boolean +double | double +integer | integer +keyword | keyword +long | long +|=== diff --git a/docs/reference/esql/functions/types/mv_max.asciidoc b/docs/reference/esql/functions/types/mv_max.asciidoc new file mode 100644 index 0000000000000..50740a71e4b49 --- /dev/null +++ b/docs/reference/esql/functions/types/mv_max.asciidoc @@ -0,0 +1,10 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +arg1 | result +boolean | boolean +double | double +integer | integer +keyword | keyword +long | long +unsigned_long | unsigned_long +|=== diff --git a/docs/reference/esql/functions/types/mv_median.asciidoc b/docs/reference/esql/functions/types/mv_median.asciidoc new file mode 100644 index 0000000000000..f1831429aa95c --- /dev/null +++ b/docs/reference/esql/functions/types/mv_median.asciidoc @@ -0,0 +1,8 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +arg1 | result +double | double +integer | integer +long | long +unsigned_long | unsigned_long +|=== diff --git a/docs/reference/esql/functions/types/mv_min.asciidoc b/docs/reference/esql/functions/types/mv_min.asciidoc new file mode 100644 index 0000000000000..50740a71e4b49 --- /dev/null +++ b/docs/reference/esql/functions/types/mv_min.asciidoc @@ -0,0 +1,10 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +arg1 | result +boolean | boolean +double | double +integer | integer +keyword | keyword +long | long +unsigned_long | unsigned_long +|=== diff --git a/docs/reference/esql/functions/types/mv_sum.asciidoc b/docs/reference/esql/functions/types/mv_sum.asciidoc new file mode 100644 index 0000000000000..09cb78511d275 --- /dev/null +++ b/docs/reference/esql/functions/types/mv_sum.asciidoc @@ -0,0 +1,5 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +arg1 | result +double | double +|=== diff --git a/docs/reference/esql/functions/types/starts_with.asciidoc b/docs/reference/esql/functions/types/starts_with.asciidoc new file mode 100644 index 0000000000000..6c406b80c0cad --- /dev/null +++ b/docs/reference/esql/functions/types/starts_with.asciidoc @@ -0,0 +1,5 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +arg1 | arg2 | result +keyword | keyword | boolean +|=== diff --git a/docs/reference/esql/functions/types/to_string.asciidoc b/docs/reference/esql/functions/types/to_string.asciidoc new file mode 100644 index 0000000000000..c4c0129c32d53 --- /dev/null +++ b/docs/reference/esql/functions/types/to_string.asciidoc @@ -0,0 +1,13 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +v | result +boolean | keyword +datetime | keyword +double | keyword +integer | keyword +ip | keyword +keyword | keyword +long | keyword +text | keyword +version | keyword +|=== diff --git a/docs/reference/esql/functions/types/to_version.asciidoc b/docs/reference/esql/functions/types/to_version.asciidoc new file mode 100644 index 0000000000000..ebb83f03a6fe6 --- /dev/null +++ b/docs/reference/esql/functions/types/to_version.asciidoc @@ -0,0 +1,7 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +v | result +keyword | version +text | version +version | version +|=== diff --git a/docs/reference/how-to/knn-search.asciidoc b/docs/reference/how-to/knn-search.asciidoc index 0501fd5f59d01..10dd57efc3ced 100644 --- a/docs/reference/how-to/knn-search.asciidoc +++ b/docs/reference/how-to/knn-search.asciidoc @@ -21,8 +21,8 @@ options. The `cosine` option accepts any float vector and computes the cosine similarity. While this is convenient for testing, it's not the most efficient approach. Instead, we recommend using the `dot_product` option to compute the -similarity. When using `dot_product`, all vectors are normalized during index to have -a magnitude of 1. The `dot_product` option is significantly faster, since it +similarity. To use `dot_product`, all vectors need to be normalized in advance +to have length 1. The `dot_product` option is significantly faster, since it avoids performing extra vector length computations during the search. [discrete] diff --git a/docs/reference/ilm/actions/ilm-shrink.asciidoc b/docs/reference/ilm/actions/ilm-shrink.asciidoc index 727124a18ea43..d440943c2ee75 100644 --- a/docs/reference/ilm/actions/ilm-shrink.asciidoc +++ b/docs/reference/ilm/actions/ilm-shrink.asciidoc @@ -17,7 +17,7 @@ stream. You cannot perform the `shrink` action on a write index. To use the `shrink` action in the `hot` phase, the `rollover` action *must* be present. If no rollover action is configured, {ilm-init} will reject the policy. -The shrink action will unset the index's index.routing.allocation.total_shards_per_node +The shrink action will unset the index's `index.routing.allocation.total_shards_per_node` setting, meaning that there will be no limit. This is to ensure that all shards of the index can be copied to a single node. This setting change will persist on the index even after the step completes. diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc index 8098a4b0b3299..fb50ee36644a6 100644 --- a/docs/reference/mapping/types/dense-vector.asciidoc +++ b/docs/reference/mapping/types/dense-vector.asciidoc @@ -163,9 +163,9 @@ Computes the dot product of two vectors. This option provides an optimized way to perform cosine similarity. The constraints and computed score are defined by `element_type`. + -When `element_type` is `float`, all vectors are automatically converted to unit length, including both -document and query vectors. Consequently, `dot_product` does not allow vectors with a zero magnitude. -The document `_score` is computed as `(1 + dot_product(query, vector)) / 2`. +When `element_type` is `float`, all vectors must be unit length, including both +document and query vectors. The document `_score` is computed as +`(1 + dot_product(query, vector)) / 2`. + When `element_type` is `byte`, all vectors must have the same length including both document and query vectors or results will be inaccurate. @@ -175,9 +175,9 @@ where `dims` is the number of dimensions per vector. `cosine`::: Computes the cosine similarity. Note that the most efficient way to perform -cosine similarity is to have all vectors normalized to unit length, and instead use +cosine similarity is to normalize all vectors to unit length, and instead use `dot_product`. You should only use `cosine` if you need to preserve the -original vectors and cannot allow Elasticsearch to normalize them. The document `_score` +original vectors and cannot normalize them in advance. The document `_score` is computed as `(1 + cosine(query, vector)) / 2`. The `cosine` similarity does not allow vectors with zero magnitude, since cosine is not defined in this case. diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index cf360bdfb9de7..8e1023c47b929 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -1922,7 +1922,13 @@ See <> Refer to <> +[float] +[[configure-remote-clusters-dynamic]] +==== Dynamically configure remote clusters + +Refer to <>. + [role="exclude",id="remote-clusters-privileges"] === Configure roles and users for remote clusters -Refer to <> +Refer to <>. diff --git a/docs/reference/setup/install/docker.asciidoc b/docs/reference/setup/install/docker.asciidoc index d4ebacb292224..0c518d520bdd5 100644 --- a/docs/reference/setup/install/docker.asciidoc +++ b/docs/reference/setup/install/docker.asciidoc @@ -1,83 +1,13 @@ [[docker]] === Install {es} with Docker -{es} is available as a Docker image. A list of all published Docker images and -tags is available at https://www.docker.elastic.co[www.docker.elastic.co]. The -source files are in -https://github.com/elastic/elasticsearch/blob/{branch}/distribution/docker[Github]. +Docker images for {es} are available from the Elastic Docker registry. A list of +all published Docker images and tags is available at +https://www.docker.elastic.co[www.docker.elastic.co]. The source code is in +https://github.com/elastic/elasticsearch/blob/{branch}/distribution/docker[GitHub]. include::license.asciidoc[] -Starting in {es} 8.0, security is enabled by default. With security enabled, -{stack} {security-features} require TLS encryption for the transport networking -layer, or your cluster will fail to start. - -==== Install Docker - -Visit https://docs.docker.com/get-docker/[Get Docker] to install Docker for your -environment. - -IMPORTANT: If using Docker Desktop, make sure to allocate at least 4GB of -memory. You can adjust memory usage in Docker Desktop by going to **Settings > -Resources**. - -==== Pull the Docker image - -Use the `docker pull` command to pull the {es} image from the the Elastic Docker -registry. - -ifeval::["{release-state}"=="unreleased"] - -WARNING: Version {version} of {es} has not yet been released, so no -Docker image is currently available for this version. - -endif::[] - -[source,sh,subs="attributes"] ----- -docker pull {docker-image} ----- - -[[docker-verify-signature]] -==== Optional: Verify the image signature - -Verify the signatures included in your {es} Docker images to ensure they're valid. - -Elastic images are signed with https://docs.sigstore.dev/cosign/overview/[Cosign] which is part of the https://www.sigstore.dev/[Sigstore] project. -Cosign supports container signing, verification, and storage in an OCI registry. - -ifeval::["{release-state}"=="unreleased"] - -WARNING: Version {version} of {es} has not yet been released, so no -Docker image signature is currently available for this version. - -endif::[] - -Install the appropriate https://docs.sigstore.dev/cosign/installation/[Cosign application] -for your operating system. - -The container image signature for {es} v{version} can be verified as follows: - -["source","sh",subs="attributes"] --------------------------------------------- -wget https://artifacts.elastic.co/cosign.pub <1> -cosign verify --key cosign.pub {docker-image} <2> --------------------------------------------- -<1> Download the Elastic public key to verify container signature -<2> Verify the container against the Elastic public key - -The command prints the check results and the signature payload in JSON format: - -[source,sh,subs="attributes"] --------------------------------------------- -Verification for {docker-image} -- -The following checks were performed on each of these signatures: - - The cosign claims were validated - - Existence of the claims in the transparency log was verified offline - - The signatures were verified against the specified public key --------------------------------------------- - - [[docker-cli-run-dev-mode]] ==== Run {es} in Docker @@ -89,14 +19,13 @@ TIP: This setup doesn't run multiple {es} nodes or {kib} by default. To create a multi-node cluster with {kib}, use Docker Compose instead. See <>. - ===== Start a single-node cluster -ifeval::["{release-state}"=="unreleased"] - -WARNING: Version {version} of the {es} Docker image has not yet been released. - -endif::[] +. Install Docker. Visit https://docs.docker.com/get-docker/[Get Docker] to +install Docker for your environment. ++ +If using Docker Desktop, make sure to allocate at least 4GB of memory. You can +adjust memory usage in Docker Desktop by going to **Settings > Resources**. . Create a new docker network. + @@ -105,45 +34,70 @@ endif::[] docker network create elastic ---- -. Start an {es} container. +. Pull the {es} Docker image. + -- ifeval::["{release-state}"=="unreleased"] +WARNING: Version {version} has not yet been released. +No Docker image is currently available for {es} {version}. +endif::[] -WARNING: Version {version} of {es} has not yet been released, so no -Docker image is currently available for this version. +[source,sh,subs="attributes"] +---- +docker pull {docker-image} +---- +-- -endif::[] +. Optional: Install +https://docs.sigstore.dev/system_config/installation/[Cosign] for your +environment. Then use Cosign to verify the {es} image's signature. ++ +[[docker-verify-signature]] +[source,sh,subs="attributes"] +---- +wget https://artifacts.elastic.co/cosign.pub +cosign verify --key cosign.pub {docker-image} +---- ++ +The `cosign` command prints the check results and the signature payload in JSON format: ++ +[source,sh,subs="attributes"] +---- +Verification for {docker-image} -- +The following checks were performed on each of these signatures: + - The cosign claims were validated + - Existence of the claims in the transparency log was verified offline + - The signatures were verified against the specified public key +---- +. Start an {es} container. ++ [source,sh,subs="attributes"] ---- docker run --name es01 --net elastic -p 9200:9200 -it -m 1GB {docker-image} ---- - ++ TIP: Use the `-m` flag to set a memory limit for the container. This removes the need to <>. - ++ The command prints the `elastic` user password and an enrollment token for {kib}. --- . Copy the generated `elastic` password and enrollment token. These credentials are only shown when you start {es} for the first time. You can regenerate the credentials using the following commands. + --- [source,sh,subs="attributes"] ---- docker exec -it es01 /usr/share/elasticsearch/bin/elasticsearch-reset-password -u elastic docker exec -it es01 /usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s kibana ---- - ++ We recommend storing the `elastic` password as an environment variable in your shell. Example: - ++ [source,sh] ---- export ELASTIC_PASSWORD="your_password" ---- --- . Copy the `http_ca.crt` SSL certificate from the container to your local machine. + @@ -164,30 +118,19 @@ curl --cacert http_ca.crt -u elastic:$ELASTIC_PASSWORD https://localhost:9200 . Use an existing node to generate a enrollment token for the new node. + --- [source,sh] ---- docker exec -it es01 /usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s node ---- - ++ The enrollment token is valid for 30 minutes. --- . Start a new {es} container. Include the enrollment token as an environment variable. + --- -ifeval::["{release-state}"=="unreleased"] - -WARNING: Version {version} of {es} has not yet been released, so no -Docker image is currently available for this version. - -endif::[] - [source,sh,subs="attributes"] ---- docker run -e ENROLLMENT_TOKEN="" --name es02 --net elastic -it -m 1GB {docker-image} ---- --- . Call the <> to verify the node was added to the cluster. + @@ -204,8 +147,8 @@ curl --cacert http_ca.crt -u elastic:$ELASTIC_PASSWORD https://localhost:9200/_c + -- ifeval::["{release-state}"=="unreleased"] -WARNING: Version {version} of {kib} has not yet been released, so no -Docker image is currently available for this version. +WARNING: Version {version} has not yet been released. +No Docker image is currently available for {kib} {version}. endif::[] [source,sh,subs="attributes"] @@ -216,18 +159,11 @@ docker pull {kib-docker-image} . Optional: Verify the {kib} image's signature. + --- -ifeval::["{release-state}"=="unreleased"] -WARNING: Version {version} of {kib} has not yet been released, so no -Docker image signature is currently available for this version. -endif::[] - [source,sh,subs="attributes"] ---- wget https://artifacts.elastic.co/cosign.pub cosign verify --key cosign.pub {kib-docker-image} ---- --- . Start a {kib} container. + @@ -282,125 +218,99 @@ You now have a test {es} environment set up. Before you start serious development or go into production with {es}, review the <> to apply when running {es} in Docker in production. -[[elasticsearch-security-certificates]] -include::security-files-reference.asciidoc[] [[docker-compose-file]] ==== Start a multi-node cluster with Docker Compose -To get a multi-node {es} cluster and {kib} up and running in Docker with -security enabled, you can use Docker Compose. - -This configuration provides a simple method of starting a secured cluster that -you can use for development before building a distributed deployment with -multiple hosts. - -===== Prerequisites +Use Docker Compose to start a three-node {es} cluster with {kib}. Docker Compose +lets you start multiple containers with a single command. -Install the appropriate https://docs.docker.com/get-docker/[Docker application] -for your operating system. - -If you're running on Linux, install https://docs.docker.com/compose/install/[Docker Compose]. - -[NOTE] -==== -Make sure that Docker is allotted at least 4GB of memory. In Docker Desktop, -you configure resource usage on the Advanced tab in Preferences (macOS) or -Settings (Windows). -==== - -===== Prepare the environment - -Create the following configuration files in a new, empty directory. These files -are also available from the -https://github.com/elastic/elasticsearch/tree/master/docs/reference/setup/install/docker[elasticsearch] -repository on GitHub. - --- -ifeval::["{release-state}"=="unreleased"] -WARNING: Version {version} of {es} has not been released, -so the following Docker Compose and configuration files won't work. -See the {stack-gs-current}/get-started-docker.html[current version] -for the latest working files. -endif::[] --- +===== Configure and start the cluster +. Install Docker Compose. Visit the +https://docs.docker.com/compose/install/[Docker Compose docs] to install Docker +Compose for your environment. ++ +If you're using Docker Desktop, Docker Compose is installed automatically. Make +sure to allocate at least 4GB of memory to Docker Desktop. You can adjust memory +usage in Docker Desktop by going to **Settings > Resources**. -[discrete] -[[docker-env-file]] -===== `.env` +. Create or navigate to an empty directory for the project. -The `.env` file sets environment variables that are used when you run the -`docker-compose.yml` configuration file. Ensure that you specify a strong -password for the `elastic` and `kibana_system` users with the -`ELASTIC_PASSWORD` and `KIBANA_PASSWORD` variables. These variable are -referenced by the `docker-compose.yml` file. +. Download and save the following files in the project directory: ++ +- https://github.com/elastic/elasticsearch/blob/{branch}/docs/reference/setup/install/docker/.env[`.env`] +- https://github.com/elastic/elasticsearch/blob/{branch}/docs/reference/setup/install/docker/docker-compose.yml[`docker-compose.yml`] -IMPORTANT: Your passwords must be alphanumeric, and cannot contain special -characters such as `!` or `@`. The `bash` script included in the -`docker-compose.yml` file only operates on alphanumeric characters. -["source","txt",subs="attributes"] ----- -include::docker/.env[] +. In the `.env` file, specify a password for the `ELASTIC_PASSWORD` and +`KIBANA_PASSWORD` variables. ++ +The passwords must be alphanumeric and can't contain special characters, such as +`!` or `@`. The bash script included in the `docker-compose.yml` file only +works with alphanumeric characters. Example: ++ +[source,txt] ---- +# Password for the 'elastic' user (at least 6 characters) +ELASTIC_PASSWORD=changeme -[discrete] -[[docker-file]] -===== `docker-compose.yml` - -This `docker-compose.yml` file creates a three-node secure {es} cluster with authentication and network encryption enabled, and a {kib} instance securely connected to it. - -.Exposing ports -**** -This configuration exposes port `9200` on all network interfaces. Because -of how Docker handles ports, a port that isn't bound to `localhost` leaves your -{es} cluster publicly accessible, potentially ignoring any firewall settings. -If you don't want to expose port `9200` to external hosts, set the value for -`ES_PORT` in the `.env` file to something like `127.0.0.1:9200`. {es} will -then only be accessible from the host machine itself. -**** +# Password for the 'kibana_system' user (at least 6 characters) +KIBANA_PASSWORD=changeme +... +---- -[source,yaml,subs="attributes"] +. In the `.env` file, set `STACK_VERSION` to the current {stack} version. ++ +[source,txt,subs="attributes"] ---- -include::docker/docker-compose.yml[] +... +# Version of Elastic products +STACK_VERSION={version} +... ---- - -===== Start your cluster with security enabled and configured - -. Modify the `.env` file and enter strong password values for both the -`ELASTIC_PASSWORD` and `KIBANA_PASSWORD` variables. +. By default, the Docker Compose configuration exposes port `9200` on all network interfaces. + -NOTE: You must use the `ELASTIC_PASSWORD` value for further interactions with -the cluster. The `KIBANA_PASSWORD` value is only used internally when -configuring {kib}. +To avoid exposing port `9200` to external hosts, set `ES_PORT` to `127.0.0.1:9200` +in the `.env` file. This ensures {es} is only accessible from the host +machine. ++ +[source,txt] +---- +... +# Port to expose Elasticsearch HTTP API to the host +#ES_PORT=9200 +ES_PORT=127.0.0.1:9200 +... +---- -. Create and start the three-node {es} cluster and {kib} instance: +. To start the cluster, run the following command from the project directory. + -["source","sh"] +[source,sh] ---- docker-compose up -d ---- -. When the deployment has started, open a browser and navigate to http://localhost:5601[http://localhost:5601] to -access {kib}, where you can load sample data and interact with your cluster. +. After the cluster has started, open http://localhost:5601 in a web browser to +access {kib}. + +. Log in to {kib} as the `elastic` user using the `ELASTIC_PASSWORD` you set +earlier. -===== Stop and remove the deployment +===== Stop and remove the cluster To stop the cluster, run `docker-compose down`. The data in the Docker volumes is preserved and loaded when you restart the cluster with `docker-compose up`. --- -["source","sh"] +[source,sh] ---- docker-compose down ---- --- -To **delete** the network, containers, and volumes when you stop the cluster, +To delete the network, containers, and volumes when you stop the cluster, specify the `-v` option: -["source","sh"] +[source,sh] ---- docker-compose down -v ---- @@ -614,14 +524,14 @@ options>> file under `/usr/share/elasticsearch/config/jvm.options.d` that includes your desired <> settings. For testing, you can also manually set the heap size using the `ES_JAVA_OPTS` -environment variable. For example, to use 1GB, use the following command. +environment variable. For example, to use 1GB, use the following command. [source,sh,subs="attributes"] ---- docker run -e ES_JAVA_OPTS="-Xms1g -Xmx1g" -e ENROLLMENT_TOKEN="" --name es01 -p 9200:9200 --net elastic -it {docker-image} ---- -The `ES_JAVA_OPTS` variable overrides all other JVM options. +The `ES_JAVA_OPTS` variable overrides all other JVM options. We do not recommend using `ES_JAVA_OPTS` in production. ===== Pin deployments to a specific image version diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java index 24b5c0012735c..a7d3749836962 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java @@ -126,6 +126,10 @@ Function getFunction(String format, ZoneId zoneId, Locale abstract Function getFunction(String format, ZoneId timezone, Locale locale); static DateFormat fromString(String format) { + // note: the ALL_CAPS format names here (UNIX_MS, etc) are present for historical reasons: + // they are the format literals that are supported by the logstash date filter plugin + // (see https://www.elastic.co/guide/en/logstash/current/plugins-filters-date.html#plugins-filters-date-match). + // don't extend this list with new special keywords (unless logstash has grown the same keyword). return switch (format) { case "ISO8601" -> Iso8601; case "UNIX" -> Unix; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java index c2110a14db7a6..09801eeda6437 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.grok.PatternBank; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -60,8 +61,9 @@ public void testResponseSerialization() throws Exception { public void testResponseSorting() { List sortedKeys = new ArrayList<>(LEGACY_TEST_PATTERNS.bank().keySet()); Collections.sort(sortedKeys); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); GrokProcessorGetAction.TransportAction transportAction = new GrokProcessorGetAction.TransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), LEGACY_TEST_PATTERNS, ECS_TEST_PATTERNS @@ -109,8 +111,9 @@ public void onFailure(Exception e) { public void testEcsCompatibilityMode() { List sortedKeys = new ArrayList<>(ECS_TEST_PATTERNS.bank().keySet()); Collections.sort(sortedKeys); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); GrokProcessorGetAction.TransportAction transportAction = new GrokProcessorGetAction.TransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), LEGACY_TEST_PATTERNS, ECS_TEST_PATTERNS diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml index 707a2e2ddb0f7..78fb895ea791c 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml @@ -152,6 +152,13 @@ teardown: "target_field" : "date_target_7", "formats" : [ "ISO8601" ] } + }, + { + "date" : { + "field" : "date_source_8", + "target_field" : "date_target_8", + "formats" : [ "epoch_second" ] + } } ] } @@ -162,7 +169,16 @@ teardown: index: test id: "1" pipeline: "my_pipeline" - body: { date_source_1: "2018-02-05T13:44:56.657+0100", date_source_2: "2017-04-04 13:43:09 +0200", date_source_3: "10/Aug/2018:09:45:56 +0200", date_source_4: "1", date_source_5: "1", date_source_6: "4000000050d506482dbdf024", date_source_7: "2018-02-05T13:44:56.657+0100" } + body: { + date_source_1: "2018-02-05T13:44:56.657+0100", + date_source_2: "2017-04-04 13:43:09 +0200", + date_source_3: "10/Aug/2018:09:45:56 +0200", + date_source_4: "1", + date_source_5: "1", + date_source_6: "4000000050d506482dbdf024", + date_source_7: "2018-02-05T13:44:56.657+0100", + date_source_8: "1688548995.987654321" + } - do: get: @@ -182,6 +198,7 @@ teardown: - match: { _source.date_target_6: "2012-12-22T01:00:46.767Z" } - match: { _source.date_source_7: "2018-02-05T13:44:56.657+0100" } - match: { _source.date_target_7: "2018-02-05T12:44:56.657Z" } + - match: { _source.date_target_8: "2023-07-05T09:23:15.987Z" } --- diff --git a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java index e7d6d127174ec..02c17977daf6a 100644 --- a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java +++ b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java @@ -502,6 +502,7 @@ public void testSpecialValueVariable() throws Exception { assertThat(stats.getAvg(), equalTo(3.0)); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99156") public void testStringSpecialValueVariable() throws Exception { // i.e. expression script for term aggregations, which is not allowed assertAcked(indicesAdmin().prepareCreate("test").setMapping("text", "type=keyword").get()); diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java index 1a7f1a2a82883..9b16a08d4ba2f 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java @@ -363,7 +363,7 @@ private static int getLevels(int treeLevels, double precisionInMeters, int defau public LegacyGeoShapeFieldMapper build(MapperBuilderContext context) { LegacyGeoShapeParser parser = new LegacyGeoShapeParser(); GeoShapeFieldType ft = buildFieldType(parser, context); - return new LegacyGeoShapeFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), parser, this); + return new LegacyGeoShapeFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, parser, this); } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index d89c5db66f37b..ad49b305c8ef7 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -76,15 +76,16 @@ public class MatchOnlyTextFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "match_only_text"; public static class Defaults { - public static final FieldType FIELD_TYPE = new FieldType(); + public static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setTokenized(true); - FIELD_TYPE.setStored(false); - FIELD_TYPE.setStoreTermVectors(false); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - FIELD_TYPE.freeze(); + final FieldType ft = new FieldType(); + ft.setTokenized(true); + ft.setStored(false); + ft.setStoreTermVectors(false); + ft.setOmitNorms(true); + ft.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } } @@ -136,15 +137,7 @@ private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context) { public MatchOnlyTextFieldMapper build(MapperBuilderContext context) { MatchOnlyTextFieldType tft = buildFieldType(context); MultiFields multiFields = multiFieldsBuilder.build(this, context); - return new MatchOnlyTextFieldMapper( - name, - Defaults.FIELD_TYPE, - tft, - multiFields, - copyTo.build(), - context.isSourceSynthetic(), - this - ); + return new MatchOnlyTextFieldMapper(name, Defaults.FIELD_TYPE, tft, multiFields, copyTo, context.isSourceSynthetic(), this); } } @@ -375,7 +368,7 @@ private MatchOnlyTextFieldMapper( super(simpleName, mappedFieldType, multiFields, copyTo, false, null); assert mappedFieldType.getTextSearchInfo().isTokenized(); assert mappedFieldType.hasDocValues() == false; - this.fieldType = fieldType; + this.fieldType = freezeAndDeduplicateFieldType(fieldType); this.indexCreatedVersion = builder.indexCreatedVersion; this.indexAnalyzers = builder.analyzers.indexAnalyzers; this.indexAnalyzer = builder.analyzers.getIndexAnalyzer(); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java index f277450aa1833..7342a432dd5df 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java @@ -43,13 +43,14 @@ public class RankFeatureFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "rank_feature"; public static class Defaults { - public static final FieldType FIELD_TYPE = new FieldType(); + public static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setTokenized(false); - FIELD_TYPE.setIndexOptions(IndexOptions.NONE); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.freeze(); + FieldType ft = new FieldType(); + ft.setTokenized(false); + ft.setIndexOptions(IndexOptions.NONE); + ft.setOmitNorms(true); + FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } } @@ -110,7 +111,7 @@ public RankFeatureFieldMapper build(MapperBuilderContext context) { nullValue.getValue() ), multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, positiveScoreImpact.getValue(), nullValue.getValue() ); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java index 1777a9ac4ba66..f36dfb5605633 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java @@ -67,7 +67,7 @@ public RankFeaturesFieldMapper build(MapperBuilderContext context) { name, new RankFeaturesFieldType(context.buildFullName(name), meta.getValue(), positiveScoreImpact.getValue()), multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, positiveScoreImpact.getValue() ); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index bd45ff7a19bfe..43951878933fa 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -195,7 +195,7 @@ public ScaledFloatFieldMapper build(MapperBuilderContext context) { metric.getValue(), indexMode ); - return new ScaledFloatFieldMapper(name, type, multiFieldsBuilder.build(this, context), copyTo.build(), this); + return new ScaledFloatFieldMapper(name, type, multiFieldsBuilder.build(this, context), copyTo, this); } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java index 4a7bf149315a1..ca8231c46736f 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java @@ -262,7 +262,7 @@ public SearchAsYouTypeFieldMapper build(MapperBuilderContext context) { return new SearchAsYouTypeFieldMapper( name, ft, - copyTo.build(), + copyTo, indexAnalyzers, prefixFieldMapper, shingleFieldMappers, @@ -312,7 +312,7 @@ static class SearchAsYouTypeFieldType extends StringFieldType { new TextSearchInfo(fieldType, similarity, searchAnalyzer, searchQuoteAnalyzer), meta ); - this.fieldType = fieldType; + this.fieldType = freezeAndDeduplicateFieldType(fieldType); } public void setPrefixField(PrefixFieldType prefixField) { @@ -499,7 +499,7 @@ static final class PrefixFieldMapper extends FieldMapper { PrefixFieldMapper(FieldType fieldType, PrefixFieldType mappedFieldType) { super(mappedFieldType.name(), mappedFieldType, MultiFields.empty(), CopyTo.empty()); - this.fieldType = fieldType; + this.fieldType = Mapper.freezeAndDeduplicateFieldType(fieldType); } @Override @@ -538,7 +538,7 @@ static final class ShingleFieldMapper extends FieldMapper { ShingleFieldMapper(FieldType fieldType, ShingleFieldType mappedFieldtype) { super(mappedFieldtype.name(), mappedFieldtype, MultiFields.empty(), CopyTo.empty()); - this.fieldType = fieldType; + this.fieldType = freezeAndDeduplicateFieldType(fieldType); } FieldType getLuceneFieldType() { diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java index ced2b900f5a16..12829ca802425 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java @@ -87,7 +87,7 @@ public TokenCountFieldMapper build(MapperBuilderContext context) { nullValue.getValue(), meta.getValue() ); - return new TokenCountFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this); + return new TokenCountFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, this); } } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index f5b3f06793183..d2e30ed1d420d 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -163,7 +163,7 @@ public PercolatorFieldMapper build(MapperBuilderContext context) { name(), fieldType, multiFields, - copyTo.build(), + copyTo, searchExecutionContext, extractedTermsField, extractionResultField, diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java index 1b12f45ac8b51..3838916c15c64 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -74,10 +75,11 @@ public void multiSearch(MultiSearchRequest request, ActionListener { channel.sendResponse(new SearchShardsResponse(List.of(), List.of(), Collections.emptyMap())); } ); - newService.registerRequestHandler(SearchAction.NAME, ThreadPool.Names.SAME, SearchRequest::new, (request, channel, task) -> { - InternalSearchResponse response = new InternalSearchResponse( - new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), - InternalAggregations.EMPTY, - null, - null, - false, - null, - 1 - ); - SearchResponse searchResponse = new SearchResponse( - response, - null, - 1, - 1, - 0, - 100, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ); - channel.sendResponse(searchResponse); - }); + newService.registerRequestHandler( + SearchAction.NAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SearchRequest::new, + (request, channel, task) -> { + InternalSearchResponse response = new InternalSearchResponse( + new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), + InternalAggregations.EMPTY, + null, + null, + false, + null, + 1 + ); + SearchResponse searchResponse = new SearchResponse( + response, + null, + 1, + 1, + 0, + 100, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); + channel.sendResponse(searchResponse); + } + ); newService.registerRequestHandler( ClusterStateAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ClusterStateRequest::new, (request, channel, task) -> { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml index b4851ddd92469..f34aef9b83321 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml @@ -368,123 +368,3 @@ setup: filter: {"term": {"name": "cow.jpg"}} - length: {hits.hits: 0} ---- -"kNN search with dot-product auto-normalized": - - skip: - features: close_to - version: ' - 8.10.99' - reason: 'dense_vector auto-normalized dot_product in 8.11' - - - do: - indices.create: - index: test_dot_product - body: - mappings: - properties: - name: - type: keyword - dot_product_vector: - type: dense_vector - dims: 5 - index: true - similarity: dot_product - cosine_vector: - type: dense_vector - dims: 5 - index: true - similarity: cosine - - - do: - index: - index: test_dot_product - id: "1" - body: - name: cow.jpg - dot_product_vector: [ 230.0, 300.33, -34.8988, 15.555, -200.0 ] - cosine_vector: [ 230.0, 300.33, -34.8988, 15.555, -200.0 ] - - - do: - index: - index: test_dot_product - id: "2" - body: - name: moose.jpg - dot_product_vector: [ -0.5, 100.0, -13, 14.8, -156.0 ] - cosine_vector: [ -0.5, 100.0, -13, 14.8, -156.0 ] - - - do: - index: - index: test_dot_product - id: "3" - body: - name: rabbit.jpg - dot_product_vector: [ 0.5, 111.3, -13.0, 14.8, -156.0 ] - cosine_vector: [ 0.5, 111.3, -13.0, 14.8, -156.0 ] - - - do: - indices.refresh: { } - - - do: - search: - index: test_dot_product - body: - fields: [ "name" ] - knn: - field: dot_product_vector - query_vector: [-0.5, 90.0, -10, 14.8, -156.0] - k: 2 - num_candidates: 3 - - - match: {hits.total.value: 2} - - match: {hits.hits.0._id: "2"} - - set: { hits.hits.0._score: score_0 } - - match: {hits.hits.0.fields.name.0: "moose.jpg"} - - match: {hits.hits.1._id: "3"} - - set: { hits.hits.1._score: score_1 } - - match: {hits.hits.1.fields.name.0: "rabbit.jpg"} - - - do: - search: - index: test_dot_product - body: - fields: [ "name" ] - knn: - field: cosine_vector - query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] - k: 2 - num_candidates: 3 - - - match: {hits.total.value: 2} - - match: {hits.hits.0._id: "2"} - - close_to: { hits.hits.0._score: { value: $score_0, error: 0.00001 } } - - match: {hits.hits.0.fields.name.0: "moose.jpg"} - - match: {hits.hits.1._id: "3"} - - close_to: { hits.hits.1._score: { value: $score_1, error: 0.00001 } } - - match: {hits.hits.1.fields.name.0: "rabbit.jpg"} ---- -"kNN search fails with non-normalized dot-product in older versions": - - skip: - version: '8.10.99 - ' - reason: 'dense_vector auto-normalized dot_product in 8.11' - - - do: - indices.create: - index: test_failing_dot_product - body: - mappings: - properties: - dot_product_vector: - type: dense_vector - dims: 5 - index: true - similarity: dot_product - - - do: - catch: bad_request - index: - index: test_failing_dot_product - id: "1" - body: - name: cow.jpg - dot_product_vector: [ 230.0, 300.33, -34.8988, 15.555, -200.0 ] - diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java index 804ad3cdfd523..e829c6da01b26 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java @@ -43,6 +43,7 @@ protected Collection> nodePlugins() { return Arrays.asList(MockFSIndexStore.TestPlugin.class); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99174") public void testRandomDirectoryIOExceptions() throws IOException, InterruptedException, ExecutionException { String mapping = Strings.toString( XContentFactory.jsonBuilder() diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index a5b456236a94d..6fb02a3ca05d2 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -135,6 +135,7 @@ static TransportVersion def(int id, String uniqueId) { public static final TransportVersion V_8_500_069 = def(8_500_069, "5b804027-d8a0-421b-9970-1f53d766854b"); public static final TransportVersion V_8_500_070 = def(8_500_070, "6BADC9CD-3C9D-4381-8BD9-B305CAA93F86"); public static final TransportVersion V_8_500_071 = def(8_500_071, "a86dfc08-3026-4f01-90ef-6d6de003e217"); + public static final TransportVersion V_8_500_072 = def(8_500_072, "e2df7d80-7b74-4afd-9734-aee0fc256025"); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java index 0bd4458056c20..fbcfe415039ff 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java @@ -62,7 +62,7 @@ public TransportShardFlushAction( ); transportService.registerRequestHandler( PRE_SYNCED_FLUSH_ACTION_NAME, - ThreadPool.Names.FLUSH, + threadPool.executor(ThreadPool.Names.FLUSH), PreShardSyncedFlushRequest::new, new PreSyncedFlushTransportHandler(indicesService) ); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java index 9b1eb1baf197c..06d8eb5d99152 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java @@ -9,7 +9,6 @@ package org.elasticsearch.action.admin.indices.segments; import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.SortedSetSortField; import org.elasticsearch.action.support.DefaultShardOperationFailedException; @@ -18,13 +17,14 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.index.engine.Segment; import org.elasticsearch.xcontent.ToXContent; -import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -79,89 +79,115 @@ public void writeTo(StreamOutput out) throws IOException { @Override protected Iterator customXContentChunks(ToXContent.Params params) { return Iterators.concat( - Iterators.single((builder, p) -> builder.startObject(Fields.INDICES)), - Iterators.map(getIndices().values().iterator(), indexSegments -> (builder, p) -> { - builder.startObject(indexSegments.getIndex()); - - builder.startObject(Fields.SHARDS); - for (IndexShardSegments indexSegment : indexSegments) { - builder.startArray(Integer.toString(indexSegment.shardId().id())); - for (ShardSegments shardSegments : indexSegment) { - builder.startObject(); - - builder.startObject(Fields.ROUTING); - builder.field(Fields.STATE, shardSegments.getShardRouting().state()); - builder.field(Fields.PRIMARY, shardSegments.getShardRouting().primary()); - builder.field(Fields.NODE, shardSegments.getShardRouting().currentNodeId()); - if (shardSegments.getShardRouting().relocatingNodeId() != null) { - builder.field(Fields.RELOCATING_NODE, shardSegments.getShardRouting().relocatingNodeId()); - } - builder.endObject(); - - builder.field(Fields.NUM_COMMITTED_SEGMENTS, shardSegments.getNumberOfCommitted()); - builder.field(Fields.NUM_SEARCH_SEGMENTS, shardSegments.getNumberOfSearch()); - - builder.startObject(Fields.SEGMENTS); - for (Segment segment : shardSegments) { - builder.startObject(segment.getName()); - builder.field(Fields.GENERATION, segment.getGeneration()); - builder.field(Fields.NUM_DOCS, segment.getNumDocs()); - builder.field(Fields.DELETED_DOCS, segment.getDeletedDocs()); - builder.humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, segment.getSize()); - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.humanReadableField(Fields.MEMORY_IN_BYTES, Fields.MEMORY, ByteSizeValue.ZERO); - } - builder.field(Fields.COMMITTED, segment.isCommitted()); - builder.field(Fields.SEARCH, segment.isSearch()); - if (segment.getVersion() != null) { - builder.field(Fields.VERSION, segment.getVersion()); - } - if (segment.isCompound() != null) { - builder.field(Fields.COMPOUND, segment.isCompound()); - } - if (segment.getMergeId() != null) { - builder.field(Fields.MERGE_ID, segment.getMergeId()); - } - if (segment.getSegmentSort() != null) { - toXContent(builder, segment.getSegmentSort()); - } - if (segment.attributes != null && segment.attributes.isEmpty() == false) { - builder.field("attributes", segment.attributes); - } - builder.endObject(); - } - builder.endObject(); - - builder.endObject(); - } - builder.endArray(); - } - builder.endObject(); - builder.endObject(); - return builder; - }), - Iterators.single((builder, p) -> builder.endObject()) + ChunkedToXContentHelper.startObject(Fields.INDICES), + Iterators.flatMap( + getIndices().values().iterator(), + indexSegments -> Iterators.concat( + + ChunkedToXContentHelper.singleChunk( + (builder, p) -> builder.startObject(indexSegments.getIndex()).startObject(Fields.SHARDS) + ), + Iterators.flatMap( + indexSegments.iterator(), + indexSegment -> Iterators.concat( + + ChunkedToXContentHelper.startArray(Integer.toString(indexSegment.shardId().id())), + Iterators.flatMap( + indexSegment.iterator(), + shardSegments -> Iterators.concat( + + ChunkedToXContentHelper.singleChunk((builder, p) -> { + builder.startObject(); + + builder.startObject(Fields.ROUTING); + builder.field(Fields.STATE, shardSegments.getShardRouting().state()); + builder.field(Fields.PRIMARY, shardSegments.getShardRouting().primary()); + builder.field(Fields.NODE, shardSegments.getShardRouting().currentNodeId()); + if (shardSegments.getShardRouting().relocatingNodeId() != null) { + builder.field(Fields.RELOCATING_NODE, shardSegments.getShardRouting().relocatingNodeId()); + } + builder.endObject(); + + builder.field(Fields.NUM_COMMITTED_SEGMENTS, shardSegments.getNumberOfCommitted()); + builder.field(Fields.NUM_SEARCH_SEGMENTS, shardSegments.getNumberOfSearch()); + + builder.startObject(Fields.SEGMENTS); + return builder; + }), + Iterators.flatMap( + shardSegments.iterator(), + segment -> Iterators.concat( + + ChunkedToXContentHelper.singleChunk((builder, p) -> { + builder.startObject(segment.getName()); + builder.field(Fields.GENERATION, segment.getGeneration()); + builder.field(Fields.NUM_DOCS, segment.getNumDocs()); + builder.field(Fields.DELETED_DOCS, segment.getDeletedDocs()); + builder.humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, segment.getSize()); + if (builder.getRestApiVersion() == RestApiVersion.V_7) { + builder.humanReadableField(Fields.MEMORY_IN_BYTES, Fields.MEMORY, ByteSizeValue.ZERO); + } + builder.field(Fields.COMMITTED, segment.isCommitted()); + builder.field(Fields.SEARCH, segment.isSearch()); + if (segment.getVersion() != null) { + builder.field(Fields.VERSION, segment.getVersion()); + } + if (segment.isCompound() != null) { + builder.field(Fields.COMPOUND, segment.isCompound()); + } + if (segment.getMergeId() != null) { + builder.field(Fields.MERGE_ID, segment.getMergeId()); + } + return builder; + }), + getSegmentSortChunks(segment.getSegmentSort()), + ChunkedToXContentHelper.singleChunk((builder, p) -> { + if (segment.attributes != null && segment.attributes.isEmpty() == false) { + builder.field("attributes", segment.attributes); + } + builder.endObject(); + return builder; + }) + ) + ), + ChunkedToXContentHelper.singleChunk((builder, p) -> builder.endObject().endObject()) + ) + ), + ChunkedToXContentHelper.endArray() + ) + ), + ChunkedToXContentHelper.singleChunk((builder, p) -> builder.endObject().endObject()) + ) + ), + ChunkedToXContentHelper.endObject() ); } - private static void toXContent(XContentBuilder builder, Sort sort) throws IOException { - builder.startArray("sort"); - for (SortField field : sort.getSort()) { - builder.startObject(); - builder.field("field", field.getField()); - if (field instanceof SortedNumericSortField) { - builder.field("mode", ((SortedNumericSortField) field).getSelector().toString().toLowerCase(Locale.ROOT)); - } else if (field instanceof SortedSetSortField) { - builder.field("mode", ((SortedSetSortField) field).getSelector().toString().toLowerCase(Locale.ROOT)); - } - if (field.getMissingValue() != null) { - builder.field("missing", field.getMissingValue().toString()); - } - builder.field("reverse", field.getReverse()); - builder.endObject(); + private static Iterator getSegmentSortChunks(@Nullable Sort segmentSort) { + if (segmentSort == null) { + return Collections.emptyIterator(); } - builder.endArray(); + + return Iterators.concat( + ChunkedToXContentHelper.startArray("sort"), + Iterators.map(Iterators.forArray(segmentSort.getSort()), field -> (builder, p) -> { + builder.startObject(); + builder.field("field", field.getField()); + if (field instanceof SortedNumericSortField sortedNumericSortField) { + builder.field("mode", sortedNumericSortField.getSelector().toString().toLowerCase(Locale.ROOT)); + } else if (field instanceof SortedSetSortField sortedSetSortField) { + builder.field("mode", sortedSetSortField.getSelector().toString().toLowerCase(Locale.ROOT)); + } + if (field.getMissingValue() != null) { + builder.field("missing", field.getMissingValue().toString()); + } + builder.field("reverse", field.getReverse()); + builder.endObject(); + return builder; + }), + ChunkedToXContentHelper.endArray() + ); } static final class Fields { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java index 5a16ecf976354..e32934a5706b2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java @@ -10,8 +10,10 @@ import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.ChunkedBroadcastResponse; +import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.xcontent.ToXContent; import java.io.IOException; @@ -50,14 +52,13 @@ public Map> getStats() { @Override protected Iterator customXContentChunks(ToXContent.Params params) { - return stats.entrySet().stream().sorted(Map.Entry.comparingByKey()).map(entry -> (ToXContent) (builder, p) -> { - builder.startObject(entry.getKey()); - builder.startArray("shards"); - for (FieldUsageShardResponse resp : entry.getValue()) { - resp.toXContent(builder, params); - } - builder.endArray(); - return builder.endObject(); - }).iterator(); + return Iterators.flatMap( + stats.entrySet().stream().sorted(Map.Entry.comparingByKey()).iterator(), + entry -> Iterators.concat( + ChunkedToXContentHelper.singleChunk((builder, p) -> builder.startObject(entry.getKey()).startArray("shards")), + entry.getValue().iterator(), + ChunkedToXContentHelper.singleChunk((builder, p) -> builder.endArray().endObject()) + ) + ); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java index 70959babf312e..1ea0820492b04 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java @@ -23,11 +23,13 @@ import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.index.Index; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -181,46 +183,66 @@ public void writeTo(StreamOutput out) throws IOException { protected Iterator customXContentChunks(ToXContent.Params params) { final ClusterStatsLevel level = ClusterStatsLevel.of(params, ClusterStatsLevel.INDICES); if (level == ClusterStatsLevel.INDICES || level == ClusterStatsLevel.SHARDS) { - return Iterators.concat(Iterators.single(((builder, p) -> { - commonStats(builder, p); - return builder.startObject(Fields.INDICES); - })), Iterators.map(getIndices().values().iterator(), indexStats -> (builder, p) -> { - builder.startObject(indexStats.getIndex()); - builder.field("uuid", indexStats.getUuid()); - if (indexStats.getHealth() != null) { - builder.field("health", indexStats.getHealth().toString().toLowerCase(Locale.ROOT)); - } - if (indexStats.getState() != null) { - builder.field("status", indexStats.getState().toString().toLowerCase(Locale.ROOT)); - } - builder.startObject("primaries"); - indexStats.getPrimaries().toXContent(builder, p); - builder.endObject(); - - builder.startObject("total"); - indexStats.getTotal().toXContent(builder, p); - builder.endObject(); - - if (level == ClusterStatsLevel.SHARDS) { - builder.startObject(Fields.SHARDS); - for (IndexShardStats indexShardStats : indexStats) { - builder.startArray(Integer.toString(indexShardStats.getShardId().id())); - for (ShardStats shardStats : indexShardStats) { - builder.startObject(); - shardStats.toXContent(builder, p); + return Iterators.concat( + + ChunkedToXContentHelper.singleChunk((builder, p) -> { + commonStats(builder, p); + return builder.startObject(Fields.INDICES); + }), + Iterators.flatMap( + getIndices().values().iterator(), + indexStats -> Iterators.concat( + + ChunkedToXContentHelper.singleChunk((builder, p) -> { + builder.startObject(indexStats.getIndex()); + builder.field("uuid", indexStats.getUuid()); + if (indexStats.getHealth() != null) { + builder.field("health", indexStats.getHealth().toString().toLowerCase(Locale.ROOT)); + } + if (indexStats.getState() != null) { + builder.field("status", indexStats.getState().toString().toLowerCase(Locale.ROOT)); + } + builder.startObject("primaries"); + indexStats.getPrimaries().toXContent(builder, p); + builder.endObject(); + + builder.startObject("total"); + indexStats.getTotal().toXContent(builder, p); builder.endObject(); - } - builder.endArray(); - } - builder.endObject(); - } - return builder.endObject(); - }), Iterators.single((b, p) -> b.endObject())); + return builder; + }), + + level == ClusterStatsLevel.SHARDS + ? Iterators.concat( + ChunkedToXContentHelper.startObject(Fields.SHARDS), + Iterators.flatMap( + indexStats.iterator(), + indexShardStats -> Iterators.concat( + ChunkedToXContentHelper.startArray(Integer.toString(indexShardStats.getShardId().id())), + Iterators.map(indexShardStats.iterator(), shardStats -> (builder, p) -> { + builder.startObject(); + shardStats.toXContent(builder, p); + builder.endObject(); + return builder; + }), + ChunkedToXContentHelper.endArray() + ) + ), + ChunkedToXContentHelper.endObject() + ) + : Collections.emptyIterator(), + + ChunkedToXContentHelper.endObject() + ) + ), + ChunkedToXContentHelper.endObject() + ); + } else { + return ChunkedToXContentHelper.singleChunk((builder, p) -> { + commonStats(builder, p); + return builder; + }); } - return Iterators.single((b, p) -> { - commonStats(b, p); - return b; - }); } private void commonStats(XContentBuilder builder, ToXContent.Params p) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 4a3781a5865b8..e0fdc28be0b9a 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -802,7 +802,7 @@ public boolean isForceExecution() { // If a processor went async and returned a response on a different thread then // before we continue the bulk request we should fork back on a write thread: if (originalThread == Thread.currentThread()) { - threadPool.executor(Names.SAME).execute(runnable); + runnable.run(); } else { threadPool.executor(executorName).execute(runnable); } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index 79b9fa5099467..1dde340e0f527 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -92,7 +92,7 @@ public TransportFieldCapabilitiesAction( this.indicesService = indicesService; transportService.registerRequestHandler( ACTION_NODE_NAME, - ThreadPool.Names.SEARCH_COORDINATION, + this.searchCoordinationExecutor, FieldCapabilitiesNodeRequest::new, new NodeTransportHandler() ); diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java index 67379821ba014..9e4984d7497a3 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java @@ -50,12 +50,17 @@ public final class SimulateDocumentBaseResult implements SimulateDocumentResult } public SimulateDocumentBaseResult(IngestDocument ingestDocument) { + Exception failure = null; + WriteableIngestDocument wid = null; if (ingestDocument != null) { - this.ingestDocument = new WriteableIngestDocument(ingestDocument); - } else { - this.ingestDocument = null; + try { + wid = new WriteableIngestDocument(ingestDocument); + } catch (Exception ex) { + failure = ex; + } } - this.failure = null; + this.ingestDocument = wid; + this.failure = failure; } public SimulateDocumentBaseResult(Exception failure) { diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java index d4ae5e33f3b10..1d5776896ea19 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java @@ -126,7 +126,19 @@ public SimulateProcessorResult( ) { this.processorTag = processorTag; this.description = description; - this.ingestDocument = (ingestDocument == null) ? null : new WriteableIngestDocument(ingestDocument); + WriteableIngestDocument wid = null; + if (ingestDocument != null) { + try { + wid = new WriteableIngestDocument(ingestDocument); + } catch (Exception ex) { + // if there was a failure already, then track it as a suppressed exception + if (failure != null) { + ex.addSuppressed(failure); + } + failure = ex; + } + } + this.ingestDocument = wid; this.failure = failure; this.conditionalWithResult = conditionalWithResult; this.type = type; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java index 3a7e3c11fa141..40f8da6f7efaa 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java @@ -57,7 +57,7 @@ final class WriteableIngestDocument implements Writeable, ToXContentFragment { sourceAndMetadata.put(Metadata.VERSION_TYPE.getFieldName(), a[4]); } Map ingestMetadata = (Map) a[6]; - return new WriteableIngestDocument(new IngestDocument(sourceAndMetadata, ingestMetadata)); + return new WriteableIngestDocument(sourceAndMetadata, ingestMetadata); } ); static { @@ -83,17 +83,30 @@ final class WriteableIngestDocument implements Writeable, ToXContentFragment { PARSER.declareObject(constructorArg(), INGEST_DOC_PARSER, new ParseField(DOC_FIELD)); } + /** + * Builds a writeable ingest document that wraps a copy of the passed-in, non-null ingest document. + * + * @throws IllegalArgumentException if the passed-in ingest document references itself + */ WriteableIngestDocument(IngestDocument ingestDocument) { assert ingestDocument != null; - this.ingestDocument = ingestDocument; + this.ingestDocument = new IngestDocument(ingestDocument); // internal defensive copy } - WriteableIngestDocument(StreamInput in) throws IOException { - Map sourceAndMetadata = in.readMap(); - Map ingestMetadata = in.readMap(); + /** + * Builds a writeable ingest document by constructing the wrapped ingest document from the passed-in maps. + *

+ * This is intended for cases like deserialization, where we know the passed-in maps aren't self-referencing, + * and where a defensive copy is unnecessary. + */ + private WriteableIngestDocument(Map sourceAndMetadata, Map ingestMetadata) { this.ingestDocument = new IngestDocument(sourceAndMetadata, ingestMetadata); } + WriteableIngestDocument(StreamInput in) throws IOException { + this(in.readMap(), in.readMap()); + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeGenericMap(ingestDocument.getSourceAndMetadata()); diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java index 9714fc7574418..d02958567a873 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.CountDown; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.search.CanMatchShardResponse; import org.elasticsearch.search.SearchPhaseResult; @@ -450,7 +451,7 @@ public void writeTo(StreamOutput out) throws IOException { public static void registerRequestHandler(TransportService transportService, SearchService searchService) { transportService.registerRequestHandler( FREE_CONTEXT_SCROLL_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ScrollFreeContextRequest::new, (request, channel, task) -> { boolean freed = searchService.freeReaderContext(request.id()); @@ -460,7 +461,7 @@ public static void registerRequestHandler(TransportService transportService, Sea TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_SCROLL_ACTION_NAME, false, SearchFreeContextResponse::new); transportService.registerRequestHandler( FREE_CONTEXT_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, SearchFreeContextRequest::new, (request, channel, task) -> { boolean freed = searchService.freeReaderContext(request.id()); @@ -470,7 +471,7 @@ public static void registerRequestHandler(TransportService transportService, Sea TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_ACTION_NAME, false, SearchFreeContextResponse::new); transportService.registerRequestHandler( CLEAR_SCROLL_CONTEXTS_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, TransportRequest.Empty::new, (request, channel, task) -> { searchService.freeAllScrollContexts(); @@ -486,7 +487,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( DFS_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardSearchRequest::new, (request, channel, task) -> searchService.executeDfsPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)) ); @@ -495,7 +496,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardSearchRequest::new, (request, channel, task) -> searchService.executeQueryPhase( request, @@ -512,7 +513,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_ID_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, QuerySearchRequest::new, (request, channel, task) -> { searchService.executeQueryPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)); @@ -522,7 +523,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_SCROLL_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, InternalScrollSearchRequest::new, (request, channel, task) -> { searchService.executeQueryPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)); @@ -532,7 +533,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_FETCH_SCROLL_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, InternalScrollSearchRequest::new, (request, channel, task) -> { searchService.executeFetchPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)); @@ -542,7 +543,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( FETCH_ID_SCROLL_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardFetchRequest::new, (request, channel, task) -> { searchService.executeFetchPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)); @@ -552,7 +553,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( FETCH_ID_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, true, true, ShardFetchSearchRequest::new, @@ -565,7 +566,7 @@ public static void registerRequestHandler(TransportService transportService, Sea // this is cheap, it does not fetch during the rewrite phase, so we can let it quickly execute on a networking thread transportService.registerRequestHandler( QUERY_CAN_MATCH_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardSearchRequest::new, (request, channel, task) -> { searchService.canMatch(request, new ChannelActionListener<>(channel)); @@ -575,7 +576,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_CAN_MATCH_NODE_NAME, - ThreadPool.Names.SEARCH_COORDINATION, + transportService.getThreadPool().executor(ThreadPool.Names.SEARCH_COORDINATION), CanMatchNodeRequest::new, (request, channel, task) -> { searchService.canMatch(request, new ChannelActionListener<>(channel)); diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java index 9c78e5ad62aea..eb854be99562e 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchPhaseResult; @@ -68,7 +69,7 @@ public TransportOpenPointInTimeAction( this.searchTransportService = searchTransportService; transportService.registerRequestHandler( OPEN_SHARD_READER_CONTEXT_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardOpenReaderRequest::new, new ShardOpenReaderRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java index 24668d7130d52..a0e36a538913d 100644 --- a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java @@ -61,7 +61,14 @@ protected HandledTransportAction( String executor ) { super(actionName, actionFilters, transportService.getTaskManager()); - transportService.registerRequestHandler(actionName, executor, false, canTripCircuitBreaker, requestReader, new TransportHandler()); + transportService.registerRequestHandler( + actionName, + transportService.getThreadPool().executor(executor), + false, + canTripCircuitBreaker, + requestReader, + new TransportHandler() + ); } class TransportHandler implements TransportRequestHandler { diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java index 570514355da56..ddd2cc43005e2 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java @@ -75,7 +75,7 @@ protected TransportBroadcastAction( transportService.registerRequestHandler( transportShardAction, - executor, + this.executor, shardRequestReader, (request, channel, task) -> ActionListener.completeWith( new ChannelActionListener<>(channel), diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java index 15c28bdf33c87..4f78d68daa8f2 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java @@ -119,7 +119,7 @@ public TransportBroadcastByNodeAction( transportService.registerRequestHandler( transportNodeBroadcastAction, - executor, + this.executor, false, canTripCircuitBreaker, NodeRequest::new, diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java index d5c7fc763fba6..579ed101b56ae 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java @@ -60,7 +60,12 @@ protected TransportBroadcastUnpromotableAction( this.transportUnpromotableAction = actionName + "[u]"; this.executor = transportService.getThreadPool().executor(executor); - transportService.registerRequestHandler(transportUnpromotableAction, executor, requestReader, new UnpromotableTransportHandler()); + transportService.registerRequestHandler( + transportUnpromotableAction, + this.executor, + requestReader, + new UnpromotableTransportHandler() + ); } protected abstract void unpromotableShardOperation(Task task, Request request, ActionListener listener); diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java index 1efe70bcb5adc..b45c448ecc52c 100644 --- a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java @@ -82,7 +82,7 @@ protected TransportNodesAction( this.transportService = Objects.requireNonNull(transportService); this.finalExecutor = threadPool.executor(executor); this.transportNodeAction = actionName + "[n]"; - transportService.registerRequestHandler(transportNodeAction, executor, nodeRequest, new NodeTransportHandler()); + transportService.registerRequestHandler(transportNodeAction, finalExecutor, nodeRequest, new NodeTransportHandler()); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index 5651af11719cf..d1e2b6dd98faa 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Assertions; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; @@ -187,11 +188,16 @@ protected TransportReplicationAction( this.retryTimeout = REPLICATION_RETRY_TIMEOUT.get(settings); this.forceExecutionOnPrimary = forceExecutionOnPrimary; - transportService.registerRequestHandler(actionName, ThreadPool.Names.SAME, requestReader, this::handleOperationRequest); + transportService.registerRequestHandler( + actionName, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + requestReader, + this::handleOperationRequest + ); transportService.registerRequestHandler( transportPrimaryAction, - executor, + threadPool.executor(executor), forceExecutionOnPrimary, true, in -> new ConcreteShardRequest<>(requestReader, in), @@ -201,7 +207,7 @@ protected TransportReplicationAction( // we must never reject on because of thread pool capacity on replicas transportService.registerRequestHandler( transportReplicaAction, - executor, + threadPool.executor(executor), true, true, in -> new ConcreteReplicaRequest<>(replicaRequestReader, in), diff --git a/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java index 63ff4cef5a924..c526a2679306d 100644 --- a/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexNotFoundException; @@ -33,7 +34,6 @@ import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.threadpool.ThreadPool.Names; import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportException; @@ -72,7 +72,7 @@ protected TransportInstanceSingleOperationAction( this.transportService = transportService; this.indexNameExpressionResolver = indexNameExpressionResolver; this.shardActionName = actionName + "[s]"; - transportService.registerRequestHandler(shardActionName, Names.SAME, request, new ShardTransportHandler()); + transportService.registerRequestHandler(shardActionName, EsExecutors.DIRECT_EXECUTOR_SERVICE, request, new ShardTransportHandler()); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java b/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java index 001a99ecd505d..9d980fabc6714 100644 --- a/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.logging.LoggerMessageFormat; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.tasks.Task; @@ -79,9 +80,14 @@ protected TransportSingleShardAction( this.executor = executor; if (isSubAction() == false) { - transportService.registerRequestHandler(actionName, ThreadPool.Names.SAME, request, new TransportHandler()); + transportService.registerRequestHandler(actionName, EsExecutors.DIRECT_EXECUTOR_SERVICE, request, new TransportHandler()); } - transportService.registerRequestHandler(transportShardAction, ThreadPool.Names.SAME, request, new ShardTransportHandler()); + transportService.registerRequestHandler( + transportShardAction, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + request, + new ShardTransportHandler() + ); } /** diff --git a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java index b509008af2b4f..fb29e8a838db6 100644 --- a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java @@ -79,7 +79,12 @@ protected TransportTasksAction( this.responsesReader = responsesReader; this.responseReader = responseReader; - transportService.registerRequestHandler(transportNodeAction, nodeExecutor, NodeTaskRequest::new, new NodeTransportHandler()); + transportService.registerRequestHandler( + transportNodeAction, + transportService.getThreadPool().executor(nodeExecutor), + NodeTaskRequest::new, + new NodeTransportHandler() + ); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index 6ca8fe26edd54..3f9e7e4d8d9ae 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -629,21 +629,6 @@ public Iterator toXContentChunked(ToXContent.Params outerP (builder, params) -> builder.endObject() ), - // transportVersions - redundant with the nodes_versions section but has to stay for backwards compatibility - // just use NODES again, its node-related information - chunkedSection( - metrics.contains(Metric.NODES), - (builder, params) -> builder.startArray("transport_versions"), - compatibilityVersions.entrySet().iterator(), - e -> Iterators.single( - (builder, params) -> builder.startObject() - .field("node_id", e.getKey()) - .field("transport_version", e.getValue().transportVersion().toString()) - .endObject() - ), - (builder, params) -> builder.endArray() - ), - // per-node version information chunkedSection( metrics.contains(Metric.NODES), diff --git a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java index 97bf6cc01c4e5..f9aa2c6c234be 100644 --- a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; @@ -98,13 +99,13 @@ public ShardStateAction( transportService.registerRequestHandler( SHARD_STARTED_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, StartedShardEntry::new, new ShardStartedTransportHandler(clusterService, new ShardStartedClusterStateTaskExecutor(allocationService, rerouteService)) ); transportService.registerRequestHandler( SHARD_FAILED_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, FailedShardEntry::new, new ShardFailedTransportHandler(clusterService, new ShardFailedClusterStateTaskExecutor(allocationService, rerouteService)) ); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index dc52791a5d5e2..7ccea8e99918b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -271,7 +271,7 @@ public Coordinator( ); transportService.registerRequestHandler( COMMIT_STATE_ACTION_NAME, - Names.CLUSTER_COORDINATION, + this.clusterCoordinationExecutor, false, false, ApplyCommitRequest::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java b/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java index ad2faaccf0e96..99ab8650a52d8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java @@ -124,7 +124,7 @@ public FollowersChecker( updateFastResponseState(0, Mode.CANDIDATE); transportService.registerRequestHandler( FOLLOWER_CHECK_ACTION_NAME, - Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, false, false, FollowerCheckRequest::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java index ce2754fa3854c..247034c88ed62 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.core.TimeValue; @@ -114,7 +115,7 @@ public class JoinHelper { transportService.registerRequestHandler( JOIN_ACTION_NAME, - Names.CLUSTER_COORDINATION, + transportService.getThreadPool().executor(Names.CLUSTER_COORDINATION), false, false, JoinRequest::new, @@ -126,7 +127,7 @@ public class JoinHelper { transportService.registerRequestHandler( START_JOIN_ACTION_NAME, - Names.CLUSTER_COORDINATION, + transportService.getThreadPool().executor(Names.CLUSTER_COORDINATION), false, false, StartJoinRequest::new, @@ -139,7 +140,7 @@ public class JoinHelper { transportService.registerRequestHandler( JOIN_PING_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, false, false, TransportRequest.Empty::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java index 248137f03fcaa..d9911ad12df84 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java @@ -121,7 +121,7 @@ public JoinValidationService( final var dataPaths = Environment.PATH_DATA_SETTING.get(settings); transportService.registerRequestHandler( JoinValidationService.JOIN_VALIDATE_ACTION_NAME, - ThreadPool.Names.CLUSTER_COORDINATION, + this.responseExecutor, ValidateJoinRequest::new, (request, channel, task) -> { final var remoteState = request.getOrReadState(); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java b/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java index 8a20e8e56d751..9fcae5bcf67f8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java @@ -113,7 +113,7 @@ public class LeaderChecker { transportService.registerRequestHandler( LEADER_CHECK_ACTION_NAME, - Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, false, false, LeaderCheckRequest::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java index 45079b2bccd60..781a05d535b16 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java @@ -106,7 +106,7 @@ public PublicationTransportHandler( transportService.registerRequestHandler( PUBLISH_STATE_ACTION_NAME, - ThreadPool.Names.CLUSTER_COORDINATION, + this.clusterCoordinationExecutor, false, false, BytesTransportRequest::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java b/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java index 28ab04e5a7ccc..7bc3514206ffe 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java @@ -64,7 +64,7 @@ public StatefulPreVoteCollector( transportService.registerRequestHandler( REQUEST_PRE_VOTE_ACTION_NAME, - Names.CLUSTER_COORDINATION, + this.clusterCoordinationExecutor, false, false, PreVoteRequest::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 517bc2293ab6b..ab67478192c11 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -475,7 +475,8 @@ public Iterator> settings() { SETTING_INDEX_HIDDEN, false, Property.Dynamic, - Property.IndexScope + Property.IndexScope, + Property.ServerlessPublic ); /** diff --git a/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java b/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java index 046f0a1c64bb5..46e8f5210e447 100644 --- a/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java +++ b/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java @@ -110,7 +110,7 @@ public PeerFinder( transportService.registerRequestHandler( REQUEST_PEERS_ACTION_NAME, - Names.CLUSTER_COORDINATION, + this.clusterCoordinationExecutor, false, false, PeersRequest::new, diff --git a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java index d0579342f5bd0..0262e37dd74a9 100644 --- a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java +++ b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java @@ -35,7 +35,6 @@ import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestHandler; @@ -73,7 +72,7 @@ public LocalAllocateDangledIndices( this.indexMetadataVerifier = indexMetadataVerifier; transportService.registerRequestHandler( ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, AllocateDangledRequest::new, new AllocateDangledRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index f5d2dccab8052..98aaf3fbda596 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -67,7 +67,7 @@ public BinaryFieldMapper build(MapperBuilderContext context) { name, new BinaryFieldType(context.buildFullName(name), stored.getValue(), hasDocValues.getValue(), meta.getValue()), multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, this ); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index f11a3781fd19c..f2383148c31ed 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -123,14 +123,7 @@ public BooleanFieldMapper build(MapperBuilderContext context) { scriptValues(), meta.getValue() ); - return new BooleanFieldMapper( - name, - ft, - multiFieldsBuilder.build(this, context), - copyTo.build(), - context.isSourceSynthetic(), - this - ); + return new BooleanFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, context.isSourceSynthetic(), this); } private FieldValues scriptValues() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index ccf4953608f4d..581c87d224f2d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -85,14 +85,15 @@ public FieldMapper.Builder getMergeBuilder() { } public static class Defaults { - public static final FieldType FIELD_TYPE = new FieldType(); + public static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setTokenized(true); - FIELD_TYPE.setStored(false); - FIELD_TYPE.setStoreTermVectors(false); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.freeze(); + final FieldType ft = new FieldType(); + ft.setTokenized(true); + ft.setStored(false); + ft.setStoreTermVectors(false); + ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); + ft.setOmitNorms(true); + FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } public static final boolean DEFAULT_PRESERVE_SEPARATORS = true; public static final boolean DEFAULT_POSITION_INCREMENTS = true; @@ -205,7 +206,7 @@ public CompletionFieldMapper build(MapperBuilderContext context) { CompletionFieldType ft = new CompletionFieldType(context.buildFullName(name), completionAnalyzer, meta.getValue()); ft.setContextMappings(contexts.getValue()); - return new CompletionFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this); + return new CompletionFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, this); } private void checkCompletionContextsLimit() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java b/server/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java index b4ce98dcdaf9a..115b20d7d88b3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java @@ -22,11 +22,12 @@ // used for binary, geo and range fields public abstract class CustomDocValuesField implements IndexableField { - public static final FieldType TYPE = new FieldType(); + public static final FieldType TYPE; static { - TYPE.setDocValuesType(DocValuesType.BINARY); - TYPE.setOmitNorms(true); - TYPE.freeze(); + FieldType ft = new FieldType(); + ft.setDocValuesType(DocValuesType.BINARY); + ft.setOmitNorms(true); + TYPE = Mapper.freezeAndDeduplicateFieldType(ft); } private final String name; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CustomTermFreqField.java b/server/src/main/java/org/elasticsearch/index/mapper/CustomTermFreqField.java index 6ee379e598578..3b618989a2053 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CustomTermFreqField.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CustomTermFreqField.java @@ -21,11 +21,13 @@ */ public final class CustomTermFreqField extends Field { - private static final FieldType FIELD_TYPE = new FieldType(); + private static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setTokenized(false); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + final FieldType ft = new FieldType(); + ft.setTokenized(false); + ft.setOmitNorms(true); + ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + FIELD_TYPE = Mapper.freezeAndDeduplicateFieldType(ft); } private final int fieldValue; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 3bf593627faa9..9579d921c4176 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -358,7 +358,7 @@ public DateFieldMapper build(MapperBuilderContext context) { ); Long nullTimestamp = parseNullValue(ft); - return new DateFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), nullTimestamp, resolution, this); + return new DateFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, nullTimestamp, resolution, this); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 8119cf5e299d0..a9d90f80c8a18 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -528,7 +528,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws */ public static class CopyTo { - private static final CopyTo EMPTY = new CopyTo(Collections.emptyList()); + private static final CopyTo EMPTY = new CopyTo(List.of()); public static CopyTo empty() { return EMPTY; @@ -537,7 +537,14 @@ public static CopyTo empty() { private final List copyToFields; private CopyTo(List copyToFields) { - this.copyToFields = copyToFields; + this.copyToFields = List.copyOf(copyToFields); + } + + public CopyTo withAddedFields(List fields) { + if (fields.isEmpty()) { + return this; + } + return new CopyTo(CollectionUtils.concatLists(copyToFields, fields)); } public XContentBuilder toXContent(XContentBuilder builder) throws IOException { @@ -551,31 +558,6 @@ public XContentBuilder toXContent(XContentBuilder builder) throws IOException { return builder; } - public static class Builder { - private final List copyToBuilders = new ArrayList<>(); - - public Builder add(String field) { - copyToBuilders.add(field); - return this; - } - - public boolean hasValues() { - return copyToBuilders.isEmpty() == false; - } - - public CopyTo build() { - if (copyToBuilders.isEmpty()) { - return EMPTY; - } - return new CopyTo(Collections.unmodifiableList(copyToBuilders)); - } - - public void reset(CopyTo copyTo) { - copyToBuilders.clear(); - copyToBuilders.addAll(copyTo.copyToFields); - } - } - public List copyToFields() { return copyToFields; } @@ -1216,7 +1198,7 @@ void check() { public abstract static class Builder extends Mapper.Builder implements ToXContentFragment { protected final MultiFields.Builder multiFieldsBuilder = new MultiFields.Builder(); - protected final CopyTo.Builder copyTo = new CopyTo.Builder(); + protected CopyTo copyTo = CopyTo.EMPTY; /** * Creates a new Builder with a field name @@ -1246,7 +1228,7 @@ protected void merge(FieldMapper in, Conflicts conflicts, MapperBuilderContext m for (FieldMapper newSubField : in.multiFields.mappers) { multiFieldsBuilder.update(newSubField, childContext); } - this.copyTo.reset(in.copyTo); + this.copyTo = in.copyTo; validate(); } @@ -1276,7 +1258,7 @@ protected void addScriptValidation( if (s != null && multiFieldsBuilder.hasMultiFields()) { throw new MapperParsingException("Cannot define multifields on a field with a script"); } - if (s != null && copyTo.hasValues()) { + if (s != null && copyTo.copyToFields().isEmpty() == false) { throw new MapperParsingException("Cannot define copy_to parameter on a field with a script"); } }); @@ -1324,7 +1306,7 @@ public final void parse(String name, MappingParserContext parserContext, Map { - TypeParsers.parseCopyFields(propNode).forEach(copyTo::add); + copyTo = copyTo.withAddedFields(TypeParsers.parseCopyFields(propNode)); iterator.remove(); continue; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 815148ea0598f..4f96723235035 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -213,7 +213,7 @@ public FieldMapper build(MapperBuilderContext context) { indexMode ); if (this.script.get() == null) { - return new GeoPointFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), geoParser, this); + return new GeoPointFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, geoParser, this); } return new GeoPointFieldMapper(name, ft, geoParser, this); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 37ccdfe3da36c..7ae410a1a9dcb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -110,7 +110,7 @@ public GeoShapeFieldMapper build(MapperBuilderContext context) { name, ft, multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, new GeoShapeIndexer(orientation.get().value(), context.buildFullName(name)), geoShapeParser, this diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 259f51027ce73..49339c756bc7f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -178,7 +178,7 @@ public IpFieldMapper build(MapperBuilderContext context) { dimension.getValue() ), multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, context.isSourceSynthetic(), this ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 31ca329d03375..8d9c77f503ab9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -87,14 +87,15 @@ public final class KeywordFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "keyword"; public static class Defaults { - public static final FieldType FIELD_TYPE = new FieldType(); + public static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setTokenized(false); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - FIELD_TYPE.setDocValuesType(DocValuesType.SORTED_SET); - FIELD_TYPE.freeze(); + FieldType ft = new FieldType(); + ft.setTokenized(false); + ft.setOmitNorms(true); + ft.setIndexOptions(IndexOptions.DOCS); + ft.setDocValuesType(DocValuesType.SORTED_SET); + FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } public static TextSearchInfo TEXT_SEARCH_INFO = new TextSearchInfo( @@ -323,7 +324,7 @@ public KeywordFieldMapper build(MapperBuilderContext context) { fieldtype, buildFieldType(context, fieldtype), multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, context.isSourceSynthetic(), this ); @@ -831,7 +832,7 @@ private KeywordFieldMapper( this.indexed = builder.indexed.getValue(); this.hasDocValues = builder.hasDocValues.getValue(); this.indexOptions = builder.indexOptions.getValue(); - this.fieldType = fieldType; + this.fieldType = freezeAndDeduplicateFieldType(fieldType); this.similarity = builder.similarity.getValue(); this.normalizerName = builder.normalizer.getValue(); this.splitQueriesOnWhitespace = builder.splitQueriesOnWhitespace.getValue(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 32796aed2f422..598e8c4d394e8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.FieldType; import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.StringLiteralDeduplicator; import org.elasticsearch.index.IndexVersion; @@ -15,8 +16,11 @@ import java.util.Map; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; public abstract class Mapper implements ToXContentFragment, Iterable { + public abstract static class Builder { protected final String name; @@ -105,4 +109,27 @@ public String toString() { public static String internFieldName(String fieldName) { return fieldNameStringDeduplicator.deduplicate(fieldName); } + + private static final Map fieldTypeDeduplicator = new ConcurrentHashMap<>(); + + /** + * Freezes the given {@link FieldType} instances and tries to deduplicate it as long as the field does not return a non-empty value for + * {@link FieldType#getAttributes()}. + * + * @param fieldType field type to deduplicate + * @return deduplicated field type + */ + public static FieldType freezeAndDeduplicateFieldType(FieldType fieldType) { + fieldType.freeze(); + var attributes = fieldType.getAttributes(); + if ((attributes != null && attributes.isEmpty() == false) || fieldType.getClass() != FieldType.class) { + // don't deduplicate subclasses or types with non-empty attribute maps to avoid memory leaks + return fieldType; + } + if (fieldTypeDeduplicator.size() > 1000) { + // guard against the case where we run up too many combinations via (vector-)dimensions combinations + fieldTypeDeduplicator.clear(); + } + return fieldTypeDeduplicator.computeIfAbsent(fieldType, Function.identity()); + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 7590fd36d849a..34763eda69a28 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -268,14 +268,7 @@ protected Parameter[] getParameters() { @Override public NumberFieldMapper build(MapperBuilderContext context) { MappedFieldType ft = new NumberFieldType(context.buildFullName(name), this); - return new NumberFieldMapper( - name, - ft, - multiFieldsBuilder.build(this, context), - copyTo.build(), - context.isSourceSynthetic(), - this - ); + return new NumberFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, context.isSourceSynthetic(), this); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java index a7de1194ddceb..4ad0873b66d50 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java @@ -91,13 +91,7 @@ protected Parameter[] getParameters() { @Override public PlaceHolderFieldMapper build(MapperBuilderContext context) { PlaceHolderFieldType mappedFieldType = new PlaceHolderFieldType(context.buildFullName(name), type, Map.of()); - return new PlaceHolderFieldMapper( - name, - mappedFieldType, - multiFieldsBuilder.build(this, context), - copyTo.build(), - unknownParams - ); + return new PlaceHolderFieldMapper(name, mappedFieldType, multiFieldsBuilder.build(this, context), copyTo, unknownParams); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index a7f0bedc7c060..fcd2a425a6625 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -163,7 +163,7 @@ protected RangeFieldType setupFieldType(MapperBuilderContext context) { @Override public RangeFieldMapper build(MapperBuilderContext context) { RangeFieldType ft = setupFieldType(context); - return new RangeFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), type, this); + return new RangeFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, type, this); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index 1dc1d236d80d0..8e8c58b35c68a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -50,10 +50,12 @@ public class SeqNoFieldMapper extends MetadataFieldMapper { // Like Lucene's LongField but single-valued (NUMERIC doc values instead of SORTED_NUMERIC doc values) private static class SingleValueLongField extends Field { - private static final FieldType FIELD_TYPE = new FieldType(); + private static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setDimensions(1, Long.BYTES); - FIELD_TYPE.setDocValuesType(DocValuesType.NUMERIC); + FieldType ft = new FieldType(); + ft.setDimensions(1, Long.BYTES); + ft.setDocValuesType(DocValuesType.NUMERIC); + FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } private final BytesRef pointValue; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index ee709be35bc82..c5d5dbec1ef15 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -77,13 +77,14 @@ private enum Mode { public static class Defaults { public static final String NAME = SourceFieldMapper.NAME; - public static final FieldType FIELD_TYPE = new FieldType(); + public static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setIndexOptions(IndexOptions.NONE); // not indexed - FIELD_TYPE.setStored(true); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.freeze(); + FieldType ft = new FieldType(); + ft.setIndexOptions(IndexOptions.NONE); // not indexed + ft.setStored(true); + ft.setOmitNorms(true); + FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 6d2c5b1cb71ac..4c3e1d38b6d57 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -99,15 +99,16 @@ public static class Defaults { public static final int INDEX_PREFIX_MIN_CHARS = 2; public static final int INDEX_PREFIX_MAX_CHARS = 5; - public static final FieldType FIELD_TYPE = new FieldType(); + public static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setTokenized(true); - FIELD_TYPE.setStored(false); - FIELD_TYPE.setStoreTermVectors(false); - FIELD_TYPE.setOmitNorms(false); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); - FIELD_TYPE.freeze(); + FieldType ft = new FieldType(); + ft.setTokenized(true); + ft.setStored(false); + ft.setStoreTermVectors(false); + ft.setOmitNorms(false); + ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); + FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } /** @@ -469,7 +470,7 @@ public TextFieldMapper build(MapperBuilderContext context) { throw new MapperParsingException("Cannot use reserved field name [" + mapper.name() + "]"); } } - return new TextFieldMapper(name, fieldType, tft, prefixFieldInfo, phraseFieldInfo, multiFields, copyTo.build(), this); + return new TextFieldMapper(name, fieldType, tft, prefixFieldInfo, phraseFieldInfo, multiFields, copyTo, this); } } @@ -635,7 +636,7 @@ private static final class SubFieldInfo { private final String field; SubFieldInfo(String field, FieldType fieldType, Analyzer analyzer) { - this.fieldType = fieldType; + this.fieldType = Mapper.freezeAndDeduplicateFieldType(fieldType); this.analyzer = analyzer; this.field = field; } @@ -1138,7 +1139,7 @@ protected TextFieldMapper( if (fieldType.indexOptions() == IndexOptions.NONE && fieldType().fielddata()) { throw new IllegalArgumentException("Cannot enable fielddata on a [text] field that is not indexed: [" + name() + "]"); } - this.fieldType = fieldType; + this.fieldType = freezeAndDeduplicateFieldType(fieldType); this.prefixFieldInfo = prefixFieldInfo; this.phraseFieldInfo = phraseFieldInfo; this.indexCreatedVersion = builder.indexCreatedVersion; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextSearchInfo.java b/server/src/main/java/org/elasticsearch/index/mapper/TextSearchInfo.java index 744ebc6d6013a..e0cb747f48ceb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextSearchInfo.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextSearchInfo.java @@ -28,12 +28,13 @@ public record TextSearchInfo( NamedAnalyzer searchQuoteAnalyzer ) { - private static final FieldType SIMPLE_MATCH_ONLY_FIELD_TYPE = new FieldType(); + private static final FieldType SIMPLE_MATCH_ONLY_FIELD_TYPE; static { - SIMPLE_MATCH_ONLY_FIELD_TYPE.setTokenized(false); - SIMPLE_MATCH_ONLY_FIELD_TYPE.setOmitNorms(true); - SIMPLE_MATCH_ONLY_FIELD_TYPE.freeze(); + FieldType ft = new FieldType(); + ft.setTokenized(false); + ft.setOmitNorms(true); + SIMPLE_MATCH_ONLY_FIELD_TYPE = Mapper.freezeAndDeduplicateFieldType(ft); } /** @@ -100,7 +101,7 @@ public TextSearchInfo( NamedAnalyzer searchAnalyzer, NamedAnalyzer searchQuoteAnalyzer ) { - this.luceneFieldType = luceneFieldType; + this.luceneFieldType = Mapper.freezeAndDeduplicateFieldType(luceneFieldType); this.similarity = similarity; this.searchAnalyzer = Objects.requireNonNull(searchAnalyzer); this.searchQuoteAnalyzer = Objects.requireNonNull(searchQuoteAnalyzer); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 35f2d37fa0dd6..51be6290df657 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -208,7 +208,6 @@ public FlattenedFieldMapper build(MapperBuilderContext context) { if (multiFields.iterator().hasNext()) { throw new IllegalArgumentException(CONTENT_TYPE + " field [" + name + "] does not support [fields]"); } - CopyTo copyTo = this.copyTo.build(); if (copyTo.copyToFields().isEmpty() == false) { throw new IllegalArgumentException(CONTENT_TYPE + " field [" + name + "] does not support [copy_to]"); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 155dac09f0b86..0e4f871fbb8ca 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -26,7 +26,6 @@ import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.VectorUtil; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -57,7 +56,6 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.time.ZoneId; -import java.util.Arrays; import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -71,10 +69,8 @@ * A {@link FieldMapper} for indexing a dense vector of floats. */ public class DenseVectorFieldMapper extends FieldMapper { - private static final float EPS = 1e-4f; public static final IndexVersion MAGNITUDE_STORED_INDEX_VERSION = IndexVersion.V_7_5_0; public static final IndexVersion INDEXED_BY_DEFAULT_INDEX_VERSION = IndexVersion.V_8_11_0; - public static final IndexVersion DOT_PRODUCT_AUTO_NORMALIZED = IndexVersion.V_8_11_0; public static final IndexVersion LITTLE_ENDIAN_FLOAT_STORED_INDEX_VERSION = IndexVersion.V_8_9_0; public static final String CONTENT_TYPE = "dense_vector"; @@ -194,7 +190,7 @@ public DenseVectorFieldMapper build(MapperBuilderContext context) { indexOptions.getValue(), indexVersionCreated, multiFieldsBuilder.build(this, context), - copyTo.build() + copyTo ); } } @@ -325,7 +321,6 @@ public void checkVectorBounds(float[] vector) { @Override void checkVectorMagnitude( - IndexVersion indexVersion, VectorSimilarity similarity, Function appender, float squaredMagnitude @@ -388,12 +383,7 @@ public Field parseKnnVector(DocumentParserContext context, DenseVectorFieldMappe squaredMagnitude += value * value; } fieldMapper.checkDimensionMatches(index, context); - checkVectorMagnitude( - fieldMapper.indexCreatedVersion, - fieldMapper.similarity, - errorByteElementsAppender(vector), - squaredMagnitude - ); + checkVectorMagnitude(fieldMapper.similarity, errorByteElementsAppender(vector), squaredMagnitude); return createKnnVectorField(fieldMapper.fieldType().name(), vector, fieldMapper.similarity.function); } @@ -485,31 +475,20 @@ public void checkVectorBounds(float[] vector) { @Override void checkVectorMagnitude( - IndexVersion indexVersion, VectorSimilarity similarity, Function appender, float squaredMagnitude ) { StringBuilder errorBuilder = null; - if (indexVersion.before(DOT_PRODUCT_AUTO_NORMALIZED)) { - if (similarity == VectorSimilarity.DOT_PRODUCT && Math.abs(squaredMagnitude - 1.0f) > EPS) { - errorBuilder = new StringBuilder( - "The [" + VectorSimilarity.DOT_PRODUCT + "] similarity can only be used with unit-length vectors." - ); - } - if (similarity == VectorSimilarity.COSINE && Math.sqrt(squaredMagnitude) == 0.0f) { - errorBuilder = new StringBuilder( - "The [" + similarity + "] similarity does not support vectors with zero magnitude." - ); - } - } else { - if ((similarity == VectorSimilarity.COSINE || similarity == VectorSimilarity.DOT_PRODUCT) - && Math.sqrt(squaredMagnitude) == 0.0f) { - errorBuilder = new StringBuilder( - "The [" + similarity + "] similarity does not support vectors with zero magnitude." - ); - } + if (similarity == VectorSimilarity.DOT_PRODUCT && Math.abs(squaredMagnitude - 1.0f) > 1e-4f) { + errorBuilder = new StringBuilder( + "The [" + VectorSimilarity.DOT_PRODUCT + "] similarity can only be used with unit-length vectors." + ); + } else if (similarity == VectorSimilarity.COSINE && Math.sqrt(squaredMagnitude) == 0.0f) { + errorBuilder = new StringBuilder( + "The [" + VectorSimilarity.COSINE + "] similarity does not support vectors with zero magnitude." + ); } if (errorBuilder != null) { @@ -532,15 +511,7 @@ public Field parseKnnVector(DocumentParserContext context, DenseVectorFieldMappe } fieldMapper.checkDimensionMatches(index, context); checkVectorBounds(vector); - checkVectorMagnitude( - fieldMapper.indexCreatedVersion, - fieldMapper.similarity, - errorFloatElementsAppender(vector), - squaredMagnitude - ); - if (fieldMapper.indexCreatedVersion.onOrAfter(DOT_PRODUCT_AUTO_NORMALIZED)) { - fieldMapper.similarity.floatPreprocessing(vector, squaredMagnitude); - } + checkVectorMagnitude(fieldMapper.similarity, errorFloatElementsAppender(vector), squaredMagnitude); return createKnnVectorField(fieldMapper.fieldType().name(), vector, fieldMapper.similarity.function); } @@ -598,7 +569,6 @@ abstract double parseKnnVectorToByteBuffer(DocumentParserContext context, DenseV public abstract void checkVectorBounds(float[] vector); abstract void checkVectorMagnitude( - IndexVersion indexVersion, VectorSimilarity similarity, Function errorElementsAppender, float squaredMagnitude @@ -717,21 +687,6 @@ float score(float similarity, ElementType elementType, int dim) { case FLOAT -> (1 + similarity) / 2f; }; } - - @Override - void floatPreprocessing(float[] vector, float squareSum) { - if (squareSum == 0) { - throw new IllegalArgumentException("Cannot normalize a zero-length vector"); - } - // Vector already has a magnitude have `1` - if (Math.abs(squareSum - 1.0f) < EPS) { - return; - } - float length = (float) Math.sqrt(squareSum); - for (int i = 0; i < vector.length; i++) { - vector[i] /= length; - } - } }; public final VectorSimilarityFunction function; @@ -746,8 +701,6 @@ public final String toString() { } abstract float score(float similarity, ElementType elementType, int dim); - - void floatPreprocessing(float[] vector, float squareSum) {} } private abstract static class IndexOptions implements ToXContent { @@ -906,13 +859,11 @@ public Query createKnnQuery(byte[] queryVector, int numCands, Query filter, Floa } if (similarity == VectorSimilarity.DOT_PRODUCT || similarity == VectorSimilarity.COSINE) { - int squaredMagnitude = VectorUtil.dotProduct(queryVector, queryVector); - elementType.checkVectorMagnitude( - indexVersionCreated, - similarity, - elementType.errorByteElementsAppender(queryVector), - squaredMagnitude - ); + float squaredMagnitude = 0.0f; + for (byte b : queryVector) { + squaredMagnitude += b * b; + } + elementType.checkVectorMagnitude(similarity, elementType.errorByteElementsAppender(queryVector), squaredMagnitude); } Query knnQuery = new KnnByteVectorQuery(name(), queryVector, numCands, filter); if (similarityThreshold != null) { @@ -940,22 +891,11 @@ public Query createKnnQuery(float[] queryVector, int numCands, Query filter, Flo elementType.checkVectorBounds(queryVector); if (similarity == VectorSimilarity.DOT_PRODUCT || similarity == VectorSimilarity.COSINE) { - float squaredMagnitude = VectorUtil.dotProduct(queryVector, queryVector); - elementType.checkVectorMagnitude( - indexVersionCreated, - similarity, - elementType.errorFloatElementsAppender(queryVector), - squaredMagnitude - ); - // We don't want to normalize the original query vector. - // It mutates it in place and might cause down stream weirdness - // Instead we copy the value and then normalize that copy - if (similarity == VectorSimilarity.DOT_PRODUCT - && elementType == ElementType.FLOAT - && indexVersionCreated.onOrAfter(DOT_PRODUCT_AUTO_NORMALIZED)) { - queryVector = Arrays.copyOf(queryVector, queryVector.length); - similarity.floatPreprocessing(queryVector, squaredMagnitude); + float squaredMagnitude = 0.0f; + for (float e : queryVector) { + squaredMagnitude += e * e; } + elementType.checkVectorMagnitude(similarity, elementType.errorFloatElementsAppender(queryVector), squaredMagnitude); } Query knnQuery = switch (elementType) { case BYTE -> { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java index 082c2d898e637..758ad8e508ea3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java @@ -66,7 +66,7 @@ public SparseVectorFieldMapper build(MapperBuilderContext context) { name, new SparseVectorFieldType(context.buildFullName(name), meta.getValue()), multiFieldsBuilder.build(this, context), - copyTo.build() + copyTo ); } } diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java index 21b8ca31f01a7..eac119f920f6a 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java @@ -79,7 +79,7 @@ public PeerRecoverySourceService( // node. Upon receiving START_RECOVERY, the source node will initiate the peer recovery. transportService.registerRequestHandler( Actions.START_RECOVERY, - ThreadPool.Names.GENERIC, + transportService.getThreadPool().executor(ThreadPool.Names.GENERIC), StartRecoveryRequest::new, (request, channel, task) -> recover(request, task, new ChannelActionListener<>(channel)) ); @@ -89,7 +89,7 @@ public PeerRecoverySourceService( // action will fail and the target node will send a new START_RECOVERY request. transportService.registerRequestHandler( Actions.REESTABLISH_RECOVERY, - ThreadPool.Names.GENERIC, + transportService.getThreadPool().executor(ThreadPool.Names.GENERIC), ReestablishRecoveryRequest::new, (request, channel, task) -> reestablish(request, new ChannelActionListener<>(channel)) ); diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java index 5f06519f84c0c..b3b23ac14d158 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java @@ -122,7 +122,7 @@ public PeerRecoveryTargetService( transportService.registerRequestHandler( Actions.FILES_INFO, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryFilesInfoRequest::new, new RecoveryRequestHandler<>() { @Override @@ -140,7 +140,7 @@ protected void handleRequest(RecoveryFilesInfoRequest request, RecoveryTarget ta ); transportService.registerRequestHandler( Actions.RESTORE_FILE_FROM_SNAPSHOT, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoverySnapshotFileRequest::new, new RecoveryRequestHandler<>() { @Override @@ -151,13 +151,13 @@ protected void handleRequest(RecoverySnapshotFileRequest request, RecoveryTarget ); transportService.registerRequestHandler( Actions.FILE_CHUNK, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryFileChunkRequest::new, new FileChunkTransportRequestHandler() ); transportService.registerRequestHandler( Actions.CLEAN_FILES, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryCleanFilesRequest::new, new RecoveryRequestHandler<>() { @Override @@ -179,7 +179,7 @@ protected void handleRequest(RecoveryCleanFilesRequest request, RecoveryTarget t ); transportService.registerRequestHandler( Actions.PREPARE_TRANSLOG, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryPrepareForTranslogOperationsRequest::new, new RecoveryRequestHandler<>() { @Override @@ -194,13 +194,13 @@ protected void handleRequest( ); transportService.registerRequestHandler( Actions.TRANSLOG_OPS, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryTranslogOperationsRequest::new, new TranslogOperationsRequestHandler() ); transportService.registerRequestHandler( Actions.FINALIZE, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryFinalizeRecoveryRequest::new, new RecoveryRequestHandler<>() { @Override @@ -215,7 +215,7 @@ protected void handleRequest( ); transportService.registerRequestHandler( Actions.HANDOFF_PRIMARY_CONTEXT, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryHandoffPrimaryContextRequest::new, new HandoffPrimaryContextRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java index e12a38e383397..f949dd59e8968 100644 --- a/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java +++ b/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; @@ -102,7 +103,7 @@ public IndicesStore( this.threadPool = threadPool; transportService.registerRequestHandler( ACTION_SHARD_EXISTS, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardActiveRequest::new, new ShardActiveRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index 25827018e44a7..66931c7791bc8 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.VersionType; @@ -94,14 +95,26 @@ public IngestDocument(String index, String id, long version, String routing, Ver /** * Copy constructor that creates a new {@link IngestDocument} which has exactly the same properties as the one provided. + * + * @throws IllegalArgumentException if the passed-in ingest document references itself */ public IngestDocument(IngestDocument other) { this( - new IngestCtxMap(deepCopyMap(other.ctxMap.getSource()), other.ctxMap.getMetadata().clone()), + new IngestCtxMap(deepCopyMap(ensureNoSelfReferences(other.ctxMap.getSource())), other.ctxMap.getMetadata().clone()), deepCopyMap(other.ingestMetadata) ); } + /** + * Internal helper utility method to get around the issue that a {@code this(...) } constructor call must be the first statement + * in a constructor. This is only for use in the {@link IngestDocument#IngestDocument(IngestDocument)} copy constructor, it's not a + * general purpose method. + */ + private static Map ensureNoSelfReferences(Map source) { + CollectionUtils.ensureNoSelfReferences(source, null); + return source; + } + /** * Constructor to create an IngestDocument from its constituent maps. The maps are shallow copied. */ diff --git a/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java b/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java index f09b2c12c99dd..5bd811962a4c4 100644 --- a/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java @@ -98,7 +98,7 @@ public void execute(IngestDocument ingestDocument, BiConsumer persistedClusterStateServiceFactories = pluginsService .filterPlugins(ClusterCoordinationPlugin.class) @@ -1374,7 +1376,7 @@ private PersistedClusterStateService newPersistedClusterStateService( if (persistedClusterStateServiceFactories.size() == 1) { return persistedClusterStateServiceFactories.get(0) - .newPersistedClusterStateService(nodeEnvironment, xContentRegistry, clusterSettings, threadPool); + .newPersistedClusterStateService(nodeEnvironment, xContentRegistry, clusterSettings, threadPool, compatibilityVersions); } return new PersistedClusterStateService(nodeEnvironment, xContentRegistry, clusterSettings, threadPool::relativeTimeInMillis); diff --git a/server/src/main/java/org/elasticsearch/plugins/ClusterCoordinationPlugin.java b/server/src/main/java/org/elasticsearch/plugins/ClusterCoordinationPlugin.java index b5c0536554952..14305e1a8a04f 100644 --- a/server/src/main/java/org/elasticsearch/plugins/ClusterCoordinationPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/ClusterCoordinationPlugin.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.coordination.PreVoteCollector; import org.elasticsearch.cluster.coordination.Reconfigurator; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.version.CompatibilityVersions; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; @@ -76,12 +77,24 @@ CoordinationState.PersistedState createPersistedState( } interface PersistedClusterStateServiceFactory { + + @Deprecated(forRemoval = true) PersistedClusterStateService newPersistedClusterStateService( NodeEnvironment nodeEnvironment, NamedXContentRegistry xContentRegistry, ClusterSettings clusterSettings, ThreadPool threadPool ); + + default PersistedClusterStateService newPersistedClusterStateService( + NodeEnvironment nodeEnvironment, + NamedXContentRegistry xContentRegistry, + ClusterSettings clusterSettings, + ThreadPool threadPool, + CompatibilityVersions compatibilityVersions + ) { + return newPersistedClusterStateService(nodeEnvironment, xContentRegistry, clusterSettings, threadPool); + } } interface ReconfiguratorFactory { diff --git a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java index ad6cbbaad1cdc..d940415c38916 100644 --- a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java @@ -56,7 +56,7 @@ public VerifyNodeRepositoryAction( this.repositoriesService = repositoriesService; transportService.registerRequestHandler( ACTION_NAME, - ThreadPool.Names.SNAPSHOT, + transportService.getThreadPool().executor(ThreadPool.Names.SNAPSHOT), VerifyNodeRepositoryRequest::new, new VerifyNodeRepositoryRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index babce70239871..111bec2c88509 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -317,9 +317,7 @@ public SearchModule(Settings settings, List plugins) { registerIntervalsSourceProviders(); requestCacheKeyDifferentiator = registerRequestCacheKeyDifferentiator(plugins); namedWriteables.addAll(SortValue.namedWriteables()); - registerGenericNamedWriteable( - new SearchPlugin.GenericNamedWriteableSpec(GeoBoundingBox.class.getSimpleName(), GeoBoundingBox::new) - ); + registerGenericNamedWriteable(new SearchPlugin.GenericNamedWriteableSpec("GeoBoundingBox", GeoBoundingBox::new)); } public List getNamedWriteables() { diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java b/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java index 39a76f3508daa..6ab95072727c0 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ListenableFuture; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.NodeDisconnectedException; @@ -60,13 +61,13 @@ public TaskCancellationService(TransportService transportService) { this.deduplicator = new ResultDeduplicator<>(transportService.getThreadPool().getThreadContext()); transportService.registerRequestHandler( BAN_PARENT_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, BanParentTaskRequest::new, new BanParentRequestHandler() ); transportService.registerRequestHandler( CANCEL_CHILD_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, CancelChildRequest::new, new CancelChildRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java index 9542d4b366ded..c38f4b26c665f 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.CountDown; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.TimeValue; import org.elasticsearch.node.ReportingService; @@ -539,7 +540,7 @@ Collection getConnections() { static void registerRemoteClusterHandshakeRequestHandler(TransportService transportService) { transportService.registerRequestHandler( REMOTE_CLUSTER_HANDSHAKE_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, false, false, TransportService.HandshakeRequest::new, diff --git a/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java b/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java index ed3432f66087b..aa28f8a76b58e 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -188,7 +189,7 @@ public static void registerProxyActionWithDynamicResponseType( RequestHandlerRegistry requestHandler = service.getRequestHandler(action); service.registerRequestHandler( getProxyAction(action), - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, true, false, in -> cancellable diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index 83fc21396e0f6..5369b9a9eec13 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -1118,46 +1118,6 @@ public static boolean isValidActionName(String actionName) { return false; } - /** - * Temporary passthrough function that continues to take a String rather than Executor type. - * - * @param action - * @param executor - * @param requestReader - * @param handler - * @param - */ - public void registerRequestHandler( - String action, - String executor, - Writeable.Reader requestReader, - TransportRequestHandler handler - ) { - registerRequestHandler(action, threadPool.executor(executor), requestReader, handler); - } - - /** - * Temporary passthrough function that continues to take a String rather than Executor type. - * - * @param action - * @param executor - * @param forceExecution - * @param canTripCircuitBreaker - * @param requestReader - * @param handler - * @param - */ - public void registerRequestHandler( - String action, - String executor, - boolean forceExecution, - boolean canTripCircuitBreaker, - Writeable.Reader requestReader, - TransportRequestHandler handler - ) { - registerRequestHandler(action, threadPool.executor(executor), forceExecution, canTripCircuitBreaker, requestReader, handler); - } - /** * Registers a new request handler * diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java index f5e812e7d0983..a2bfa68fdf25f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java @@ -38,6 +38,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.gateway.TestGatewayAllocator; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -56,17 +57,20 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TransportDeleteDesiredBalanceActionTests extends ESAllocationTestCase { public void testReturnsErrorIfAllocatorIsNotDesiredBalanced() throws Exception { - var listener = new PlainActionFuture(); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + new TransportDeleteDesiredBalanceAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), mock(AllocationService.class), @@ -131,8 +135,12 @@ public DesiredBalance compute( var listener = new PlainActionFuture(); + // TODO: temporary, remove in #97879 + TransportService transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); + var action = new TransportDeleteDesiredBalanceAction( - mock(TransportService.class), + transportService, clusterService, threadPool, mock(ActionFilters.class), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java index 804dbf66ac0d0..80f0b435645e6 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.shard.ShardId; @@ -45,6 +46,7 @@ import org.elasticsearch.test.AbstractChunkedSerializingTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.junit.Before; import java.util.HashMap; import java.util.List; @@ -59,6 +61,7 @@ import static org.elasticsearch.cluster.ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -66,16 +69,27 @@ public class TransportGetDesiredBalanceActionTests extends ESAllocationTestCase private final DesiredBalanceShardsAllocator desiredBalanceShardsAllocator = mock(DesiredBalanceShardsAllocator.class); private final ClusterInfoService clusterInfoService = mock(ClusterInfoService.class); - private final TransportGetDesiredBalanceAction transportGetDesiredBalanceAction = new TransportGetDesiredBalanceAction( - mock(TransportService.class), - mock(ClusterService.class), - mock(ThreadPool.class), - mock(ActionFilters.class), - mock(IndexNameExpressionResolver.class), - desiredBalanceShardsAllocator, - clusterInfoService, - TEST_WRITE_LOAD_FORECASTER - ); + private TransportService transportService = mock(TransportService.class); + private ThreadPool threadPool = mock(ThreadPool.class); + private TransportGetDesiredBalanceAction transportGetDesiredBalanceAction; + + @Before + public void initialize() { + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + + transportGetDesiredBalanceAction = new TransportGetDesiredBalanceAction( + transportService, + mock(ClusterService.class), + threadPool, + mock(ActionFilters.class), + mock(IndexNameExpressionResolver.class), + desiredBalanceShardsAllocator, + clusterInfoService, + TEST_WRITE_LOAD_FORECASTER + ); + } private static DesiredBalanceResponse execute(TransportGetDesiredBalanceAction action, ClusterState clusterState) throws Exception { return PlainActionFuture.get( @@ -96,11 +110,10 @@ private DesiredBalanceResponse executeAction(ClusterState clusterState) throws E public void testReturnsErrorIfAllocatorIsNotDesiredBalanced() throws Exception { var clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadataWithConfiguredAllocator(BALANCED_ALLOCATOR)).build(); - final var action = new TransportGetDesiredBalanceAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), mock(ShardsAllocator.class), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java index e560959dfacf8..f190c422e165c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.monitor.StatusInfo; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; @@ -30,6 +31,7 @@ import java.util.List; import java.util.Map; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -162,6 +164,10 @@ public void testTransportDoExecute() { Coordinator coordinator = mock(Coordinator.class); ClusterFormationFailureHelper.ClusterFormationState clusterFormationState = getClusterFormationState(); when(coordinator.getClusterFormationState()).thenReturn(clusterFormationState); + + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); ClusterFormationInfoAction.TransportAction action = new ClusterFormationInfoAction.TransportAction( transportService, actionFilters, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java index 3674a711d8968..f362327a3d8e4 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; import org.elasticsearch.threadpool.ThreadPool; @@ -24,6 +25,7 @@ import java.util.ArrayList; import java.util.List; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -83,6 +85,10 @@ public void testTransportDoExecute() { when(threadPool.relativeTimeInMillis()).thenReturn(System.currentTimeMillis()); MasterHistory masterHistory = new MasterHistory(threadPool, clusterService); when(masterHistoryService.getLocalMasterHistory()).thenReturn(masterHistory); + + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); MasterHistoryAction.TransportAction action = new MasterHistoryAction.TransportAction( transportService, actionFilters, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java index e4e8293810f56..508cb9b304c59 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -54,10 +55,12 @@ public void validate(List desiredNodes) {} }; public void testWriteBlocks() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportUpdateDesiredNodesAction action = new TransportUpdateDesiredNodesAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), NO_OP_SETTINGS_VALIDATOR, @@ -79,10 +82,12 @@ public void testWriteBlocks() { } public void testNoBlocks() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportUpdateDesiredNodesAction action = new TransportUpdateDesiredNodesAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), NO_OP_SETTINGS_VALIDATOR, @@ -103,10 +108,13 @@ public void validate(List desiredNodes) { } }; ClusterService clusterService = mock(ClusterService.class); + + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportUpdateDesiredNodesAction action = new TransportUpdateDesiredNodesAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), validator, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java index 3b3a996e50cd3..1b3e0ae6fe7bf 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.reservedstate.TransformState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.XContentParser; @@ -132,14 +133,16 @@ public Repository create(RepositoryMetadata metadata) { } }; + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); RepositoriesService repositoriesService = spy( new RepositoriesService( Settings.EMPTY, mock(ClusterService.class), - mock(TransportService.class), + transportService, Map.of(), Map.of("fs", fsFactory), - mock(ThreadPool.class), + threadPool, null ) ); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java index d2a482fa58b0e..80ff2168f6344 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java @@ -126,12 +126,6 @@ public void testToXContentWithDeprecatedClusterState() { "max_index_version": %s } }, - "transport_versions": [ - { - "node_id": "node0", - "transport_version": "8000099" - } - ], "nodes_versions": [ { "node_id": "node0", diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java index 9e757894bcdda..00b500254883c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -47,10 +48,12 @@ public void testRequestConstruction() { public void testTransportFilters() throws Exception { final SettingsFilter filter = new SettingsFilter(List.of("persistent.foo.filtered", "transient.foo.filtered")); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportClusterGetSettingsAction action = new TransportClusterGetSettingsAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, filter, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java index ebe77f4d3ed4c..6373b94ffb94a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -85,10 +86,12 @@ private static ClusterUpdateSettingsRequest createTestItem() { public void testOperatorHandler() throws IOException { ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportClusterUpdateSettingsAction action = new TransportClusterUpdateSettingsAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), clusterSettings diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java index 671bf13fb0e77..d68641d04dd74 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.junit.Before; @@ -90,10 +91,13 @@ public void setUp() throws Exception { ThreadContext threadContext = new ThreadContext(Settings.EMPTY); IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, SYSTEM_INDICES); this.metadataCreateIndexService = mock(MetadataCreateIndexService.class); + + final ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); this.action = new TransportCreateIndexAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, metadataCreateIndexService, mock(ActionFilters.class), indexNameExpressionResolver, diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java index 12308036689c6..bb1cf85013498 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.cache.query.QueryCacheStats; @@ -78,6 +79,7 @@ import static org.hamcrest.Matchers.is; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -354,6 +356,10 @@ public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPr EmptySystemIndices.INSTANCE, WriteLoadForecaster.DEFAULT ); + + // TODO: temporary, remove in #97879 + when(mockTransportService.getThreadPool()).thenReturn(mockThreadPool); + when(mockThreadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); final TransportRolloverAction transportRolloverAction = new TransportRolloverAction( mockTransportService, mockClusterService, diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java index 5fe7a528d68c7..ad7689320a94f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java @@ -67,7 +67,7 @@ public void testChunking() { 0, Collections.emptyList() ), - response -> response.getIndices().size() + 4 + response -> 11 * response.getIndices().size() + 4 ); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java index 0e1581606f803..807462b6052c3 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.junit.Before; @@ -68,10 +69,13 @@ public void setUp() throws Exception { ThreadContext threadContext = new ThreadContext(Settings.EMPTY); IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, SYSTEM_INDICES); MetadataUpdateSettingsService metadataUpdateSettingsService = mock(MetadataUpdateSettingsService.class); + + final ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); this.action = new TransportUpdateSettingsAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, metadataUpdateSettingsService, mock(ActionFilters.class), indexNameExpressionResolver, diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java index 6fb53760de5c2..32bedb6082842 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java @@ -46,7 +46,7 @@ public void testToXContentChunkPerIndex() { AbstractChunkedSerializingTestCase.assertChunkCount( new FieldUsageStatsResponse(indices, indices, 0, List.of(), perIndex), - ignored -> indices + 2 + ignored -> 3 * indices + 2 ); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java index a1280f7658804..6ee17ec7963b6 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.json.JsonXContent; -import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; @@ -116,7 +115,7 @@ public void testGetIndices() { } } - public void testChunkedEncodingPerIndex() throws IOException { + public void testChunkedEncodingPerIndex() { final int shards = randomIntBetween(1, 10); final List stats = new ArrayList<>(shards); for (int i = 0; i < shards; i++) { @@ -143,7 +142,7 @@ public void testChunkedEncodingPerIndex() throws IOException { AbstractChunkedSerializingTestCase.assertChunkCount( indicesStatsResponse, new ToXContent.MapParams(Map.of("level", "indices")), - ignored -> 4 + shards + ignored -> 4 + 2 * shards ); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java index 553dd2368e5bc..28476a0d8b839 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.reservedstate.TransformState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -673,10 +674,12 @@ public void testAddRemoveIndexTemplatesWithOverlap() throws Exception { } public void testHandlerCorrectness() { + final ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var putIndexAction = new TransportPutComposableIndexTemplateAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, null, mock(ActionFilters.class), null @@ -687,9 +690,9 @@ public void testHandlerCorrectness() { containsInAnyOrder(reservedComposableIndexName("aaa")) ); var delIndexAction = new TransportDeleteComposableIndexTemplateAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, null, mock(ActionFilters.class), null @@ -701,9 +704,9 @@ public void testHandlerCorrectness() { ); var putComponentAction = new TransportPutComponentTemplateAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, null, mock(ActionFilters.class), null, @@ -716,9 +719,9 @@ public void testHandlerCorrectness() { ); var delComponentAction = new TransportDeleteComponentTemplateAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, null, mock(ActionFilters.class), null @@ -921,10 +924,12 @@ public void testTemplatesWithReservedPrefix() throws Exception { PutComposableIndexTemplateAction.Request pr = new PutComposableIndexTemplateAction.Request(conflictingTemplateName); + final ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var putTemplateAction = new TransportPutComposableIndexTemplateAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, null, mock(ActionFilters.class), null diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java index ff96f4a00b883..1b0c24664be31 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AtomicArray; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexNotFoundException; @@ -32,6 +31,7 @@ import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -42,7 +42,6 @@ import java.util.function.Function; import static java.util.Collections.emptySet; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -107,9 +106,6 @@ private void indicesThatCannotBeCreatedTestCase( when(clusterService.localNode()).thenReturn(localNode); when(localNode.isIngestNode()).thenReturn(randomBoolean()); - final ThreadPool threadPool = mock(ThreadPool.class); - when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); - final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver( new ThreadContext(Settings.EMPTY), EmptySystemIndices.INSTANCE @@ -120,9 +116,11 @@ public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state) } }; + final ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportBulkAction action = new TransportBulkAction( threadPool, - mock(TransportService.class), + transportService, clusterService, null, null, diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index fd84a1e5fe816..85a74174b094f 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -183,6 +183,11 @@ public void setupAction() { MockitoAnnotations.openMocks(this); // setup services that will be called by action transportService = mock(TransportService.class); + + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + clusterService = mock(ClusterService.class); localIngest = true; // setup nodes for local and remote diff --git a/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java b/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java index f4cbfac402a5a..593cfa92877ba 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java @@ -41,9 +41,11 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.sameInstance; public class WriteableIngestDocumentTests extends AbstractXContentTestCase { + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99403") public void testEqualsAndHashcode() throws Exception { Map sourceAndMetadata = RandomDocumentPicks.randomSource(random()); int numFields = randomIntBetween(1, IngestDocument.Metadata.values().length); @@ -183,6 +185,14 @@ public void testXContentHashSetSerialization() throws Exception { } } + public void testCopiesTheIngestDocument() { + IngestDocument document = createRandomIngestDoc(); + WriteableIngestDocument wid = new WriteableIngestDocument(document); + + assertThat(wid.getIngestDocument(), equalTo(document)); + assertThat(wid.getIngestDocument(), not(sameInstance(document))); + } + static IngestDocument createRandomIngestDoc() { XContentType xContentType = randomFrom(XContentType.values()); BytesReference sourceBytes = RandomObjects.randomSource(random(), xContentType); diff --git a/server/src/test/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportActionTests.java b/server/src/test/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportActionTests.java index 93c4ea98adcd3..c2ff59f0ccbe0 100644 --- a/server/src/test/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportActionTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import java.io.IOException; @@ -47,7 +48,8 @@ public void testRejectImmutableConflictClusterStateUpdate() { ClusterService clusterService = mock(ClusterService.class); doReturn(clusterState).when(clusterService).state(); - Action handler = new Action("internal:testAction", clusterService, mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + Action handler = new Action("internal:testAction", clusterService, transportService, mock(ActionFilters.class)); // nothing should happen here, since the request doesn't touch any of the immutable state keys var future = new PlainActionFuture(); @@ -61,7 +63,7 @@ public void testRejectImmutableConflictClusterStateUpdate() { FakeReservedStateAwareAction action = new FakeReservedStateAwareAction( "internal:testClusterSettings", clusterService, - mock(TransportService.class), + transportService, mock(ActionFilters.class), null ); diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/PostWriteRefreshTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/PostWriteRefreshTests.java index e45fc51cf6bdd..d14429647c7d3 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/PostWriteRefreshTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/PostWriteRefreshTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.engine.DocIdSeqNoAndSource; @@ -31,7 +32,6 @@ import org.elasticsearch.index.shard.ReplicationGroup; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.transport.MockTransportService; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.io.IOException; @@ -67,7 +67,7 @@ public void setUp() throws Exception { transportService.acceptIncomingRequests(); transportService.registerRequestHandler( TransportUnpromotableShardRefreshAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, UnpromotableShardRefreshRequest::new, (request, channel, task) -> { unpromotableRefreshRequestReceived.set(true); diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java index 8e95277bdc84f..7726feaa30868 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java @@ -424,7 +424,7 @@ protected TestAction(boolean withDocumentFailureOnPrimary, boolean withDocumentF ), TransportWriteActionTests.this.clusterService, null, - null, + TransportWriteActionTests.threadPool, null, new ActionFilters(new HashSet<>()), TestRequest::new, diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index e3c5865333b94..b96a362b2a867 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -208,12 +208,6 @@ public void testToXContent() throws IOException { "max_index_version":%s } }, - "transport_versions" : [ - { - "node_id" : "nodeId1", - "transport_version" : "%s" - } - ], "nodes_versions" : [ { "node_id" : "nodeId1", @@ -373,7 +367,6 @@ public void testToXContent() throws IOException { IndexVersion.MINIMUM_COMPATIBLE, IndexVersion.current(), TransportVersion.current(), - TransportVersion.current(), IndexVersion.current(), IndexVersion.current(), allocationId, @@ -470,12 +463,6 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti "max_index_version" : %s } }, - "transport_versions" : [ - { - "node_id" : "nodeId1", - "transport_version" : "%s" - } - ], "nodes_versions" : [ { "node_id" : "nodeId1", @@ -631,7 +618,6 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti IndexVersion.MINIMUM_COMPATIBLE, IndexVersion.current(), TransportVersion.current(), - TransportVersion.current(), IndexVersion.current(), IndexVersion.current(), allocationId, @@ -728,12 +714,6 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti "max_index_version" : %s } }, - "transport_versions" : [ - { - "node_id" : "nodeId1", - "transport_version" : "%s" - } - ], "nodes_versions" : [ { "node_id" : "nodeId1", @@ -895,7 +875,6 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti IndexVersion.MINIMUM_COMPATIBLE, IndexVersion.current(), TransportVersion.current(), - TransportVersion.current(), IndexVersion.current(), IndexVersion.current(), allocationId, @@ -953,7 +932,6 @@ public void testToXContentSameTypeName() throws IOException { "master_node" : null, "blocks" : { }, "nodes" : { }, - "transport_versions" : [ ], "nodes_versions" : [ ], "metadata" : { "cluster_uuid" : "clusterUUID", @@ -1185,9 +1163,9 @@ public static int expectedChunkCount(ToXContent.Params params, ClusterState clus chunkCount += 2 + clusterState.blocks().indices().size(); } - // nodes, transport_versions, nodes_versions + // nodes, nodes_versions if (metrics.contains(ClusterState.Metric.NODES)) { - chunkCount += 6 + clusterState.nodes().size() + 2 * clusterState.compatibilityVersions().size(); + chunkCount += 4 + clusterState.nodes().size() + clusterState.compatibilityVersions().size(); } // metadata diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index 248d94cb67a63..8ff319134b964 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -174,7 +174,7 @@ protected Parameter[] getParameters() { @Override public FieldMapper build(MapperBuilderContext context) { - return new TestMapper(name(), context.buildFullName(name), multiFieldsBuilder.build(this, context), copyTo.build(), this); + return new TestMapper(name(), context.buildFullName(name), multiFieldsBuilder.build(this, context), copyTo, this); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 726ce0aa043c1..c2762d859f266 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -320,7 +320,7 @@ public void testDotProductWithInvalidNorm() throws Exception { b -> b.field("type", "dense_vector").field("dims", 3).field("index", true).field("similarity", VectorSimilarity.DOT_PRODUCT) ) ); - float[] vector = { 0f, 0f, 0f }; + float[] vector = { -12.1f, 2.7f, -4 }; DocumentParsingException e = expectThrows( DocumentParsingException.class, () -> mapper.parse(source(b -> b.array("field", vector))) @@ -329,7 +329,23 @@ public void testDotProductWithInvalidNorm() throws Exception { assertThat( e.getCause().getMessage(), containsString( - "The [dot_product] similarity does not support vectors with zero magnitude. Preview of invalid vector: [0.0, 0.0, 0.0]" + "The [dot_product] similarity can only be used with unit-length vectors. Preview of invalid vector: [-12.1, 2.7, -4.0]" + ) + ); + + DocumentMapper mapperWithLargerDim = createDocumentMapper( + fieldMapping( + b -> b.field("type", "dense_vector").field("dims", 6).field("index", true).field("similarity", VectorSimilarity.DOT_PRODUCT) + ) + ); + float[] largerVector = { -12.1f, 2.7f, -4, 1.05f, 10.0f, 29.9f }; + e = expectThrows(DocumentParsingException.class, () -> mapperWithLargerDim.parse(source(b -> b.array("field", largerVector)))); + assertNotNull(e.getCause()); + assertThat( + e.getCause().getMessage(), + containsString( + "The [dot_product] similarity can only be used with unit-length vectors. " + + "Preview of invalid vector: [-12.1, 2.7, -4.0, 1.05, 10.0, ...]" ) ); } @@ -499,7 +515,7 @@ public void testDefaultParamsBeforeIndexByDefault() throws Exception { assertNull(denseVectorFieldType.getSimilarity()); } - public void testParamsBeforeIndexByDefault() throws Exception { + public void testtParamsBeforeIndexByDefault() throws Exception { DocumentMapper documentMapper = createDocumentMapper(INDEXED_BY_DEFAULT_PREVIOUS_INDEX_VERSION, fieldMapping(b -> { b.field("type", "dense_vector").field("dims", 3).field("index", true).field("similarity", "dot_product"); })); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 448d6aff0f4e8..d22056d49beb5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -137,9 +137,9 @@ public void testFloatCreateKnnQuery() { ); e = expectThrows( IllegalArgumentException.class, - () -> dotProductField.createKnnQuery(new float[] { 0.0f, 0.0f, 0.0f }, 10, null, null) + () -> dotProductField.createKnnQuery(new float[] { 0.3f, 0.1f, 1.0f }, 10, null, null) ); - assertThat(e.getMessage(), containsString("The [dot_product] similarity does not support vectors with zero magnitude.")); + assertThat(e.getMessage(), containsString("The [dot_product] similarity can only be used with unit-length vectors.")); DenseVectorFieldType cosineField = new DenseVectorFieldType( "f", diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java index 66503bf4cea79..a18e7e8ce46f9 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.indices.recovery.plan.RecoveryPlannerService; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.NodeRoles; import org.elasticsearch.transport.TransportService; @@ -38,8 +39,9 @@ public void testDuplicateRecoveries() throws IOException { final ClusterService clusterService = mock(ClusterService.class); when(clusterService.getSettings()).thenReturn(NodeRoles.dataNode()); when(indicesService.clusterService()).thenReturn(clusterService); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); PeerRecoverySourceService peerRecoverySourceService = new PeerRecoverySourceService( - mock(TransportService.class), + transportService, indicesService, new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), mock(RecoveryPlannerService.class) diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java index 924ca1fc7a1e9..ae46668352048 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java @@ -1069,6 +1069,17 @@ public void testCopyConstructor() { assertThat(ingestDocument.getFieldValue("_id", String.class), equalTo("bar1")); assertThat(ingestDocument.getFieldValue("hello", String.class), equalTo("world1")); } + + { + // the copy constructor rejects self-references + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); + List someList = new ArrayList<>(); + someList.add("some string"); + someList.add(someList); // the list contains itself + ingestDocument.setFieldValue("someList", someList); + Exception e = expectThrows(IllegalArgumentException.class, () -> new IngestDocument(ingestDocument)); + assertThat(e.getMessage(), equalTo("Iterable object is self-referencing itself")); + } } public void testCopyConstructorWithZonedDateTime() { diff --git a/server/src/test/java/org/elasticsearch/node/NodeTests.java b/server/src/test/java/org/elasticsearch/node/NodeTests.java index c2c6671ee8875..ce54b00cec9ef 100644 --- a/server/src/test/java/org/elasticsearch/node/NodeTests.java +++ b/server/src/test/java/org/elasticsearch/node/NodeTests.java @@ -16,12 +16,14 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.version.CompatibilityVersions; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.core.RestApiVersion; @@ -633,15 +635,33 @@ private static class BaseTestClusterCoordinationPlugin extends Plugin implements @Override public Optional getPersistedClusterStateServiceFactory() { - return Optional.of( - (nodeEnvironment, namedXContentRegistry, clusterSettings, threadPool) -> persistedClusterStateService = - new PersistedClusterStateService( + return Optional.of(new PersistedClusterStateServiceFactory() { + @Override + public PersistedClusterStateService newPersistedClusterStateService( + NodeEnvironment nodeEnvironment, + NamedXContentRegistry xContentRegistry, + ClusterSettings clusterSettings, + ThreadPool threadPool + ) { + throw new AssertionError("not called"); + } + + @Override + public PersistedClusterStateService newPersistedClusterStateService( + NodeEnvironment nodeEnvironment, + NamedXContentRegistry namedXContentRegistry, + ClusterSettings clusterSettings, + ThreadPool threadPool, + CompatibilityVersions compatibilityVersions + ) { + return persistedClusterStateService = new PersistedClusterStateService( nodeEnvironment, namedXContentRegistry, clusterSettings, threadPool::relativeTimeInMillis - ) - ); + ); + } + }); } } diff --git a/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java b/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java index 4ffd1e3ed94eb..56e72c25802e3 100644 --- a/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java +++ b/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java @@ -132,7 +132,7 @@ private void runTest( childTransportService.getTaskManager().setTaskCancellationService(new TaskCancellationService(childTransportService)); childTransportService.registerRequestHandler( "internal:testAction[c]", - ThreadPool.Names.MANAGEMENT, // busy-wait for cancellation but not on a transport thread + threadPool.executor(ThreadPool.Names.MANAGEMENT), // busy-wait for cancellation but not on a transport thread (StreamInput in) -> new TransportRequest.Empty(in) { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 5635e07000029..2d0956fae5499 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.SuppressForbidden; @@ -125,7 +126,7 @@ public static MockTransportService startTransport( try { newService.registerRequestHandler( SearchShardsAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, SearchShardsRequest::new, (request, channel, task) -> { if ("index_not_found".equals(request.preference())) { @@ -135,41 +136,50 @@ public static MockTransportService startTransport( } } ); - newService.registerRequestHandler(SearchAction.NAME, ThreadPool.Names.SAME, SearchRequest::new, (request, channel, task) -> { - if ("index_not_found".equals(request.preference())) { - channel.sendResponse(new IndexNotFoundException("index")); - return; - } - SearchHits searchHits; - if ("null_target".equals(request.preference())) { - searchHits = new SearchHits(new SearchHit[] { new SearchHit(0) }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1F); - } else { - searchHits = new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN); + newService.registerRequestHandler( + SearchAction.NAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SearchRequest::new, + (request, channel, task) -> { + if ("index_not_found".equals(request.preference())) { + channel.sendResponse(new IndexNotFoundException("index")); + return; + } + SearchHits searchHits; + if ("null_target".equals(request.preference())) { + searchHits = new SearchHits( + new SearchHit[] { new SearchHit(0) }, + new TotalHits(1, TotalHits.Relation.EQUAL_TO), + 1F + ); + } else { + searchHits = new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN); + } + InternalSearchResponse response = new InternalSearchResponse( + searchHits, + InternalAggregations.EMPTY, + null, + null, + false, + null, + 1 + ); + SearchResponse searchResponse = new SearchResponse( + response, + null, + 1, + 1, + 0, + 100, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); + channel.sendResponse(searchResponse); } - InternalSearchResponse response = new InternalSearchResponse( - searchHits, - InternalAggregations.EMPTY, - null, - null, - false, - null, - 1 - ); - SearchResponse searchResponse = new SearchResponse( - response, - null, - 1, - 1, - 0, - 100, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ); - channel.sendResponse(searchResponse); - }); + ); newService.registerRequestHandler( ClusterStateAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ClusterStateRequest::new, (request, channel, task) -> { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); @@ -183,7 +193,7 @@ public static MockTransportService startTransport( if (RemoteClusterPortSettings.REMOTE_CLUSTER_SERVER_ENABLED.get(s)) { newService.registerRequestHandler( RemoteClusterNodesAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, RemoteClusterNodesAction.Request::new, (request, channel, task) -> channel.sendResponse(new RemoteClusterNodesAction.Response(knownNodes)) ); diff --git a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java index 577f5d2c7dc18..84e4249bb0ccb 100644 --- a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; @@ -120,7 +121,7 @@ public MockTransportService startTransport( try { newService.registerRequestHandler( ClusterStateAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ClusterStateRequest::new, (request, channel, task) -> { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); @@ -134,7 +135,7 @@ public MockTransportService startTransport( if (hasClusterCredentials) { newService.registerRequestHandler( RemoteClusterNodesAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, RemoteClusterNodesAction.Request::new, (request, channel, task) -> channel.sendResponse(new RemoteClusterNodesAction.Response(knownNodes)) ); diff --git a/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java b/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java index 704de4fb2276a..32a5b5dec9597 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.RefCounted; @@ -108,32 +109,47 @@ private MockTransportService buildService(VersionInformation version, TransportV } public void testSendMessage() throws InterruptedException { - serviceA.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { - assertEquals(request.sourceNode, "TS_A"); - final SimpleTestResponse response = new SimpleTestResponse("TS_A"); - channel.sendResponse(response); - assertThat(response.hasReferences(), equalTo(false)); - }); + serviceA.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SimpleTestRequest::new, + (request, channel, task) -> { + assertEquals(request.sourceNode, "TS_A"); + final SimpleTestResponse response = new SimpleTestResponse("TS_A"); + channel.sendResponse(response); + assertThat(response.hasReferences(), equalTo(false)); + } + ); final boolean cancellable = randomBoolean(); TransportActionProxy.registerProxyAction(serviceA, "internal:test", cancellable, SimpleTestResponse::new); AbstractSimpleTransportTestCase.connectToNode(serviceA, nodeB); - serviceB.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { - assertThat(task instanceof CancellableTask, equalTo(cancellable)); - assertEquals(request.sourceNode, "TS_A"); - final SimpleTestResponse response = new SimpleTestResponse("TS_B"); - channel.sendResponse(response); - assertThat(response.hasReferences(), equalTo(false)); - }); + serviceB.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SimpleTestRequest::new, + (request, channel, task) -> { + assertThat(task instanceof CancellableTask, equalTo(cancellable)); + assertEquals(request.sourceNode, "TS_A"); + final SimpleTestResponse response = new SimpleTestResponse("TS_B"); + channel.sendResponse(response); + assertThat(response.hasReferences(), equalTo(false)); + } + ); TransportActionProxy.registerProxyAction(serviceB, "internal:test", cancellable, SimpleTestResponse::new); AbstractSimpleTransportTestCase.connectToNode(serviceB, nodeC); - serviceC.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { - assertThat(task instanceof CancellableTask, equalTo(cancellable)); - assertEquals(request.sourceNode, "TS_A"); - final SimpleTestResponse response = new SimpleTestResponse("TS_C"); - channel.sendResponse(response); - assertThat(response.hasReferences(), equalTo(false)); - }); + serviceC.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SimpleTestRequest::new, + (request, channel, task) -> { + assertThat(task instanceof CancellableTask, equalTo(cancellable)); + assertEquals(request.sourceNode, "TS_A"); + final SimpleTestResponse response = new SimpleTestResponse("TS_C"); + channel.sendResponse(response); + assertThat(response.hasReferences(), equalTo(false)); + } + ); TransportActionProxy.registerProxyAction(serviceC, "internal:test", cancellable, SimpleTestResponse::new); // Node A -> Node B -> Node C: different versions - serialize the response @@ -248,7 +264,7 @@ public void testSendLocalRequest() throws Exception { final boolean cancellable = randomBoolean(); serviceB.registerRequestHandler( "internal:test", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), SimpleTestRequest::new, (request, channel, task) -> { try { @@ -308,24 +324,39 @@ public void handleException(TransportException exp) { public void testException() throws InterruptedException { boolean cancellable = randomBoolean(); - serviceA.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { - assertEquals(request.sourceNode, "TS_A"); - SimpleTestResponse response = new SimpleTestResponse("TS_A"); - channel.sendResponse(response); - }); + serviceA.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SimpleTestRequest::new, + (request, channel, task) -> { + assertEquals(request.sourceNode, "TS_A"); + SimpleTestResponse response = new SimpleTestResponse("TS_A"); + channel.sendResponse(response); + } + ); TransportActionProxy.registerProxyAction(serviceA, "internal:test", cancellable, SimpleTestResponse::new); AbstractSimpleTransportTestCase.connectToNode(serviceA, nodeB); - serviceB.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { - assertEquals(request.sourceNode, "TS_A"); - SimpleTestResponse response = new SimpleTestResponse("TS_B"); - channel.sendResponse(response); - }); + serviceB.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SimpleTestRequest::new, + (request, channel, task) -> { + assertEquals(request.sourceNode, "TS_A"); + SimpleTestResponse response = new SimpleTestResponse("TS_B"); + channel.sendResponse(response); + } + ); TransportActionProxy.registerProxyAction(serviceB, "internal:test", cancellable, SimpleTestResponse::new); AbstractSimpleTransportTestCase.connectToNode(serviceB, nodeC); - serviceC.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { - throw new ElasticsearchException("greetings from TS_C"); - }); + serviceC.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SimpleTestRequest::new, + (request, channel, task) -> { + throw new ElasticsearchException("greetings from TS_C"); + } + ); TransportActionProxy.registerProxyAction(serviceC, "internal:test", cancellable, SimpleTestResponse::new); CountDownLatch latch = new CountDownLatch(1); diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java index b6d3ab5455fd6..2eb77c706a3a2 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; @@ -68,7 +69,7 @@ protected void onSendRequest(long requestId, String action, TransportRequest req transportService.registerRequestHandler( testActionName, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, TransportRequest.Empty::new, (request, channel, task) -> channel.sendResponse(TransportResponse.Empty.INSTANCE) ); diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java index 6f422190e9870..c5aa918fdc0e2 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java @@ -155,7 +155,7 @@ public ExecutorService executor(String name) { for (final var executor : EXECUTOR_NAMES) { transportService.registerRequestHandler( ACTION_NAME_PREFIX + executor, - executor, + threadPool.executor(executor), TransportRequest.Empty::new, (request, channel, task) -> { if (randomBoolean()) { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index 0034be7100df6..cb028c746a8cd 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -11,6 +11,7 @@ import org.elasticsearch.index.query.SearchExecutionContext; import java.util.Collections; +import java.util.List; // this sucks how much must be overridden just do get a dummy field mapper... public class MockFieldMapper extends FieldMapper { @@ -80,14 +81,14 @@ public Builder addMultiField(Builder builder) { } public Builder copyTo(String field) { - this.copyTo.add(field); + this.copyTo = copyTo.withAddedFields(List.of(field)); return this; } @Override public MockFieldMapper build(MapperBuilderContext context) { MultiFields multiFields = multiFieldsBuilder.build(this, context); - return new MockFieldMapper(name(), fieldType, multiFields, copyTo.build()); + return new MockFieldMapper(name(), fieldType, multiFields, copyTo); } } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 49f5ab1e8c6fa..0b917c90b69eb 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -1849,7 +1849,7 @@ private static int getWorkerBasePort(int effectiveWorkerId) { return MIN_PRIVATE_PORT + PORTS_PER_WORKER + effectiveWorkerId * PORTS_PER_WORKER; } - protected static InetAddress randomIp(boolean v4) { + public static InetAddress randomIp(boolean v4) { try { if (v4) { byte[] ipv4 = new byte[4]; diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockUtils.java b/test/framework/src/main/java/org/elasticsearch/test/MockUtils.java new file mode 100644 index 0000000000000..82487d6e60bb7 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/MockUtils.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.test; + +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockingDetails; +import static org.mockito.Mockito.when; + +/** + * Utilities for setting up Mockito mocks. + */ +public class MockUtils { + + /** + * Sets up a mock TransportService that can answer calls to TransportService.getThreadPool().executor(String). + * + * @return A mocked TransportService instance + */ + public static TransportService setupTransportServiceWithThreadpoolExecutor() { + return setMockReturns(mock(TransportService.class), mock(ThreadPool.class)); + } + + /** + * Sets up a mock TransportService that can answer calls to TransportService.getThreadPool().executor(String), using the given + * threadPool in TransportService. + * + * @param threadPool A mock ThreadPool + * @return A mocked TransportService instance + */ + public static TransportService setupTransportServiceWithThreadpoolExecutor(ThreadPool threadPool) { + return setMockReturns(mock(TransportService.class), threadPool); + } + + private static TransportService setMockReturns(TransportService transportService, ThreadPool threadPool) { + assert mockingDetails(threadPool).isMock(); + assert mockingDetails(transportService).isMock(); + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + when(threadPool.generic()).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + return transportService; + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java index 412cc310458e1..94c9dc338496a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; /** * Holds an object and allows extraction of specific values from it, given their path @@ -171,4 +172,14 @@ public XContentBuilder toXContentBuilder(XContent xContent) throws IOException { public String toString() { return "ObjectPath[" + object + "]"; } + + public int evaluateArraySize(String path) throws IOException { + final List list = evaluate(path); + return list.size(); + } + + public Set evaluateMapKeys(String path) throws IOException { + final Map map = evaluate(path); + return map.keySet(); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java index 4a683c3f7dd57..2a80b4e2e7e45 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java @@ -313,7 +313,7 @@ public void assertNoPendingHandshakes(Transport transport) { public void testHelloWorld() { serviceA.registerRequestHandler( "internal:sayHello", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -397,7 +397,7 @@ public void testThreadContext() throws ExecutionException, InterruptedException serviceA.registerRequestHandler( "internal:ping_pong", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { assertEquals("ping_user", threadPool.getThreadContext().getHeader("test.ping.user")); @@ -460,7 +460,7 @@ public void testLocalNodeConnection() throws InterruptedException { final AtomicReference exception = new AtomicReference<>(); serviceA.registerRequestHandler( "internal:localNode", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { try { @@ -519,8 +519,8 @@ public void testMessageListeners() throws Exception { } }; final String ACTION = "internal:action"; - serviceA.registerRequestHandler(ACTION, ThreadPool.Names.GENERIC, TransportRequest.Empty::new, requestHandler); - serviceB.registerRequestHandler(ACTION, ThreadPool.Names.GENERIC, TransportRequest.Empty::new, requestHandler); + serviceA.registerRequestHandler(ACTION, threadPool.executor(ThreadPool.Names.GENERIC), TransportRequest.Empty::new, requestHandler); + serviceB.registerRequestHandler(ACTION, threadPool.executor(ThreadPool.Names.GENERIC), TransportRequest.Empty::new, requestHandler); class CountingListener implements TransportMessageListener { AtomicInteger requestsReceived = new AtomicInteger(); @@ -639,7 +639,7 @@ public void testVoidMessageCompressed() throws Exception { try (MockTransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY)) { serviceA.registerRequestHandler( "internal:sayHello", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), TransportRequest.Empty::new, (request, channel, task) -> { try { @@ -695,7 +695,7 @@ public void testHelloWorldCompressed() throws Exception { try (MockTransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY)) { serviceA.registerRequestHandler( "internal:sayHello", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -765,8 +765,18 @@ public void testIndexingDataCompression() throws Exception { fail(e.getMessage()); } }; - serviceA.registerRequestHandler("internal:sayHello", ThreadPool.Names.GENERIC, StringMessageRequest::new, handler); - serviceC.registerRequestHandler("internal:sayHello", ThreadPool.Names.GENERIC, StringMessageRequest::new, handler); + serviceA.registerRequestHandler( + "internal:sayHello", + threadPool.executor(ThreadPool.Names.GENERIC), + StringMessageRequest::new, + handler + ); + serviceC.registerRequestHandler( + "internal:sayHello", + threadPool.executor(ThreadPool.Names.GENERIC), + StringMessageRequest::new, + handler + ); Settings settingsWithCompress = Settings.builder() .put(TransportSettings.TRANSPORT_COMPRESS.getKey(), Compression.Enabled.INDEXING_DATA) @@ -830,7 +840,7 @@ public void handleException(TransportException exp) { public void testErrorMessage() throws InterruptedException { serviceA.registerRequestHandler( "internal:sayHelloException", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -893,7 +903,7 @@ public void testConcurrentSendRespondAndDisconnect() throws BrokenBarrierExcepti Set responseErrors = ConcurrentCollections.newConcurrentSet(); serviceA.registerRequestHandler( "internal:test", - randomBoolean() ? ThreadPool.Names.SAME : ThreadPool.Names.GENERIC, + threadPool.executor(randomBoolean() ? ThreadPool.Names.SAME : ThreadPool.Names.GENERIC), TestRequest::new, (request, channel, task) -> { try { @@ -912,7 +922,7 @@ public void testConcurrentSendRespondAndDisconnect() throws BrokenBarrierExcepti logger.trace("caught exception while responding from node B", e); } }; - serviceB.registerRequestHandler("internal:test", ThreadPool.Names.SAME, TestRequest::new, ignoringRequestHandler); + serviceB.registerRequestHandler("internal:test", EsExecutors.DIRECT_EXECUTOR_SERVICE, TestRequest::new, ignoringRequestHandler); int halfSenders = scaledRandomIntBetween(3, 10); final CyclicBarrier go = new CyclicBarrier(halfSenders * 2 + 1); @@ -1003,7 +1013,12 @@ public void onAfter() { // simulate restart of nodeB serviceB.close(); MockTransportService newService = buildService("TS_B_" + i, version1, transportVersion1, Settings.EMPTY); - newService.registerRequestHandler("internal:test", ThreadPool.Names.SAME, TestRequest::new, ignoringRequestHandler); + newService.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + ignoringRequestHandler + ); serviceB = newService; nodeB = newService.getLocalDiscoNode(); connectToNode(serviceB, nodeA); @@ -1027,7 +1042,7 @@ public void testNotifyOnShutdown() throws Exception { try { serviceA.registerRequestHandler( "internal:foobar", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { try { @@ -1060,7 +1075,7 @@ public void testNotifyOnShutdown() throws Exception { public void testTimeoutSendExceptionWithNeverSendingBackResponse() throws Exception { serviceA.registerRequestHandler( "internal:sayHelloTimeoutNoResponse", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> assertThat("moshe", equalTo(request.message)) ); // don't send back a response @@ -1105,7 +1120,7 @@ public void testTimeoutSendExceptionWithDelayedResponse() throws Exception { Semaphore inFlight = new Semaphore(Integer.MAX_VALUE); serviceA.registerRequestHandler( "internal:sayHelloTimeoutDelayedResponse", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { String message = request.message; @@ -1250,12 +1265,22 @@ public void handleResponse(StringMessageResponse response) {} public void handleException(TransportException exp) {} }; - serviceA.registerRequestHandler("internal:test", ThreadPool.Names.SAME, StringMessageRequest::new, handler); - serviceA.registerRequestHandler("internal:testNotSeen", ThreadPool.Names.SAME, StringMessageRequest::new, handler); - serviceA.registerRequestHandler("internal:testError", ThreadPool.Names.SAME, StringMessageRequest::new, handlerWithError); - serviceB.registerRequestHandler("internal:test", ThreadPool.Names.SAME, StringMessageRequest::new, handler); - serviceB.registerRequestHandler("internal:testNotSeen", ThreadPool.Names.SAME, StringMessageRequest::new, handler); - serviceB.registerRequestHandler("internal:testError", ThreadPool.Names.SAME, StringMessageRequest::new, handlerWithError); + serviceA.registerRequestHandler("internal:test", EsExecutors.DIRECT_EXECUTOR_SERVICE, StringMessageRequest::new, handler); + serviceA.registerRequestHandler("internal:testNotSeen", EsExecutors.DIRECT_EXECUTOR_SERVICE, StringMessageRequest::new, handler); + serviceA.registerRequestHandler( + "internal:testError", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + StringMessageRequest::new, + handlerWithError + ); + serviceB.registerRequestHandler("internal:test", EsExecutors.DIRECT_EXECUTOR_SERVICE, StringMessageRequest::new, handler); + serviceB.registerRequestHandler("internal:testNotSeen", EsExecutors.DIRECT_EXECUTOR_SERVICE, StringMessageRequest::new, handler); + serviceB.registerRequestHandler( + "internal:testError", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + StringMessageRequest::new, + handlerWithError + ); String includeSettings; String excludeSettings; @@ -1570,13 +1595,18 @@ public void writeTo(StreamOutput out) throws IOException { } public void testVersionFrom0to1() throws Exception { - serviceB.registerRequestHandler("internal:version", ThreadPool.Names.SAME, Version1Request::new, (request, channel, task) -> { - assertThat(request.value1, equalTo(1)); - assertThat(request.value2, equalTo(0)); // not set, coming from service A - Version1Response response = new Version1Response(1, 2); - channel.sendResponse(response); - assertEquals(transportVersion0, channel.getVersion()); - }); + serviceB.registerRequestHandler( + "internal:version", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + Version1Request::new, + (request, channel, task) -> { + assertThat(request.value1, equalTo(1)); + assertThat(request.value2, equalTo(0)); // not set, coming from service A + Version1Response response = new Version1Response(1, 2); + channel.sendResponse(response); + assertEquals(transportVersion0, channel.getVersion()); + } + ); Version0Request version0Request = new Version0Request(); version0Request.value1 = 1; @@ -1613,12 +1643,17 @@ public void handleException(TransportException exp) { } public void testVersionFrom1to0() throws Exception { - serviceA.registerRequestHandler("internal:version", ThreadPool.Names.SAME, Version0Request::new, (request, channel, task) -> { - assertThat(request.value1, equalTo(1)); - Version0Response response = new Version0Response(1); - channel.sendResponse(response); - assertEquals(transportVersion0, channel.getVersion()); - }); + serviceA.registerRequestHandler( + "internal:version", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + Version0Request::new, + (request, channel, task) -> { + assertThat(request.value1, equalTo(1)); + Version0Response response = new Version0Response(1); + channel.sendResponse(response); + assertEquals(transportVersion0, channel.getVersion()); + } + ); Version1Request version1Request = new Version1Request(); version1Request.value1 = 1; @@ -1658,15 +1693,20 @@ public void handleException(TransportException exp) { } public void testVersionFrom1to1() throws Exception { - serviceB.registerRequestHandler("internal:version", ThreadPool.Names.SAME, Version1Request::new, (request, channel, task) -> { - assertThat(request.value1, equalTo(1)); - assertThat(request.value2, equalTo(2)); - Version1Response response = new Version1Response(1, 2); - channel.sendResponse(response); - // channel versions don't make sense on DirectResponseChannel - assertThat(channel, instanceOf(TaskTransportChannel.class)); - assertThat(((TaskTransportChannel) channel).getChannel(), instanceOf(TransportService.DirectResponseChannel.class)); - }); + serviceB.registerRequestHandler( + "internal:version", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + Version1Request::new, + (request, channel, task) -> { + assertThat(request.value1, equalTo(1)); + assertThat(request.value2, equalTo(2)); + Version1Response response = new Version1Response(1, 2); + channel.sendResponse(response); + // channel versions don't make sense on DirectResponseChannel + assertThat(channel, instanceOf(TaskTransportChannel.class)); + assertThat(((TaskTransportChannel) channel).getChannel(), instanceOf(TransportService.DirectResponseChannel.class)); + } + ); Version1Request version1Request = new Version1Request(); version1Request.value1 = 1; @@ -1706,14 +1746,19 @@ public void handleException(TransportException exp) { } public void testVersionFrom0to0() throws Exception { - serviceA.registerRequestHandler("internal:version", ThreadPool.Names.SAME, Version0Request::new, (request, channel, task) -> { - assertThat(request.value1, equalTo(1)); - Version0Response response = new Version0Response(1); - channel.sendResponse(response); - // channel versions don't make sense on DirectResponseChannel - assertThat(channel, instanceOf(TaskTransportChannel.class)); - assertThat(((TaskTransportChannel) channel).getChannel(), instanceOf(TransportService.DirectResponseChannel.class)); - }); + serviceA.registerRequestHandler( + "internal:version", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + Version0Request::new, + (request, channel, task) -> { + assertThat(request.value1, equalTo(1)); + Version0Response response = new Version0Response(1); + channel.sendResponse(response); + // channel versions don't make sense on DirectResponseChannel + assertThat(channel, instanceOf(TaskTransportChannel.class)); + assertThat(((TaskTransportChannel) channel).getChannel(), instanceOf(TransportService.DirectResponseChannel.class)); + } + ); Version0Request version0Request = new Version0Request(); version0Request.value1 = 1; @@ -1752,7 +1797,7 @@ public void handleException(TransportException exp) { public void testMockFailToSendNoConnectRule() throws Exception { serviceA.registerRequestHandler( "internal:sayHello", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -1808,7 +1853,7 @@ public void handleException(TransportException exp) { public void testMockUnresponsiveRule() throws InterruptedException { serviceA.registerRequestHandler( "internal:sayHello", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -1860,11 +1905,16 @@ public void testHostOnMessages() throws InterruptedException { final CountDownLatch latch = new CountDownLatch(2); final AtomicReference addressA = new AtomicReference<>(); final AtomicReference addressB = new AtomicReference<>(); - serviceB.registerRequestHandler("internal:action1", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - addressA.set(request.remoteAddress()); - channel.sendResponse(new TestResponse((String) null)); - latch.countDown(); - }); + serviceB.registerRequestHandler( + "internal:action1", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { + addressA.set(request.remoteAddress()); + channel.sendResponse(new TestResponse((String) null)); + latch.countDown(); + } + ); serviceA.sendRequest(nodeB, "internal:action1", new TestRequest(), new TransportResponseHandler() { @Override public TestResponse read(StreamInput in) throws IOException { @@ -1902,10 +1952,15 @@ public void handleException(TransportException exp) { public void testRejectEarlyIncomingRequests() throws Exception { try (TransportService service = buildService("TS_TEST", version0, transportVersion0, null, Settings.EMPTY, false, false)) { AtomicBoolean requestProcessed = new AtomicBoolean(false); - service.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - requestProcessed.set(true); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - }); + service.registerRequestHandler( + "internal:action", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { + requestProcessed.set(true); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + ); DiscoveryNode node = service.getLocalNode(); serviceA.close(); @@ -2145,19 +2200,19 @@ public Executor executor(ThreadPool threadPool) { } serviceB.registerRequestHandler( "internal:action1", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), TestRequest::new, new TestRequestHandler(serviceB) ); serviceC.registerRequestHandler( "internal:action1", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), TestRequest::new, new TestRequestHandler(serviceC) ); serviceA.registerRequestHandler( "internal:action1", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), TestRequest::new, new TestRequestHandler(serviceA) ); @@ -2231,7 +2286,7 @@ public Executor executor(ThreadPool threadPool) { public void testRegisterHandlerTwice() { serviceB.registerRequestHandler( "internal:action1", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), TestRequest::new, (request, message, task) -> { throw new AssertionError("boom"); @@ -2241,7 +2296,7 @@ public void testRegisterHandlerTwice() { IllegalArgumentException.class, () -> serviceB.registerRequestHandler( "internal:action1", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), TestRequest::new, (request, message, task) -> { throw new AssertionError("boom"); @@ -2251,7 +2306,7 @@ public void testRegisterHandlerTwice() { serviceA.registerRequestHandler( "internal:action1", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), TestRequest::new, (request, message, task) -> { throw new AssertionError("boom"); @@ -2449,16 +2504,21 @@ public void run() { public void testResponseHeadersArePreserved() throws InterruptedException { List executors = new ArrayList<>(ThreadPool.THREAD_POOL_TYPES.keySet()); CollectionUtil.timSort(executors); // makes sure it's reproducible - serviceA.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { + serviceA.registerRequestHandler( + "internal:action", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { - threadPool.getThreadContext().putTransient("boom", new Object()); - threadPool.getThreadContext().addResponseHeader("foo.bar", "baz"); - if ("fail".equals(request.info)) { - throw new RuntimeException("boom"); - } else { - channel.sendResponse(TransportResponse.Empty.INSTANCE); + threadPool.getThreadContext().putTransient("boom", new Object()); + threadPool.getThreadContext().addResponseHeader("foo.bar", "baz"); + if ("fail".equals(request.info)) { + throw new RuntimeException("boom"); + } else { + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } } - }); + ); CountDownLatch latch = new CountDownLatch(2); @@ -2512,9 +2572,14 @@ public void testHandlerIsInvokedOnConnectionClose() throws IOException, Interrup List executors = new ArrayList<>(ThreadPool.THREAD_POOL_TYPES.keySet()); CollectionUtil.timSort(executors); // makes sure it's reproducible TransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY); - serviceC.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - // do nothing - }); + serviceC.registerRequestHandler( + "internal:action", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { + // do nothing + } + ); CountDownLatch latch = new CountDownLatch(1); TransportResponseHandler transportResponseHandler = new TransportResponseHandler() { @Override @@ -2576,26 +2641,31 @@ public void testConcurrentDisconnectOnNonPublishedConnection() throws IOExceptio MockTransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY); CountDownLatch receivedLatch = new CountDownLatch(1); CountDownLatch sendResponseLatch = new CountDownLatch(1); - serviceC.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - // don't block on a network thread here - threadPool.generic().execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - try { - channel.sendResponse(e); - } catch (IOException e1) { - throw new UncheckedIOException(e1); + serviceC.registerRequestHandler( + "internal:action", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { + // don't block on a network thread here + threadPool.generic().execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + try { + channel.sendResponse(e); + } catch (IOException e1) { + throw new UncheckedIOException(e1); + } } - } - @Override - protected void doRun() throws Exception { - receivedLatch.countDown(); - sendResponseLatch.await(); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - } - }); - }); + @Override + protected void doRun() throws Exception { + receivedLatch.countDown(); + sendResponseLatch.await(); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + }); + } + ); CountDownLatch responseLatch = new CountDownLatch(1); TransportResponseHandler transportResponseHandler = new TransportResponseHandler.Empty() { @Override @@ -2643,26 +2713,31 @@ public void testTransportStats() throws Exception { MockTransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY); CountDownLatch receivedLatch = new CountDownLatch(1); CountDownLatch sendResponseLatch = new CountDownLatch(1); - serviceB.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - // don't block on a network thread here - threadPool.generic().execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - try { - channel.sendResponse(e); - } catch (IOException e1) { - throw new UncheckedIOException(e1); + serviceB.registerRequestHandler( + "internal:action", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { + // don't block on a network thread here + threadPool.generic().execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + try { + channel.sendResponse(e); + } catch (IOException e1) { + throw new UncheckedIOException(e1); + } } - } - @Override - protected void doRun() throws Exception { - receivedLatch.countDown(); - sendResponseLatch.await(); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - } - }); - }); + @Override + protected void doRun() throws Exception { + receivedLatch.countDown(); + sendResponseLatch.await(); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + }); + } + ); CountDownLatch responseLatch = new CountDownLatch(1); TransportResponseHandler transportResponseHandler = new TransportResponseHandler.Empty() { @Override @@ -2755,26 +2830,31 @@ public void testTransportStatsWithException() throws Exception { CountDownLatch sendResponseLatch = new CountDownLatch(1); Exception ex = new RuntimeException("boom"); ex.setStackTrace(new StackTraceElement[0]); - serviceB.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - // don't block on a network thread here - threadPool.generic().execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - try { - channel.sendResponse(e); - } catch (IOException e1) { - throw new UncheckedIOException(e1); + serviceB.registerRequestHandler( + "internal:action", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { + // don't block on a network thread here + threadPool.generic().execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + try { + channel.sendResponse(e); + } catch (IOException e1) { + throw new UncheckedIOException(e1); + } } - } - @Override - protected void doRun() throws Exception { - receivedLatch.countDown(); - sendResponseLatch.await(); - onFailure(ex); - } - }); - }); + @Override + protected void doRun() throws Exception { + receivedLatch.countDown(); + sendResponseLatch.await(); + onFailure(ex); + } + }); + } + ); CountDownLatch responseLatch = new CountDownLatch(1); AtomicReference receivedException = new AtomicReference<>(null); TransportResponseHandler transportResponseHandler = new TransportResponseHandler.Empty() { @@ -3120,31 +3200,41 @@ public void testFailToSendIllegalStateException() throws InterruptedException { public void testChannelToString() { final String ACTION = "internal:action"; - serviceA.registerRequestHandler(ACTION, ThreadPool.Names.SAME, TransportRequest.Empty::new, (request, channel, task) -> { - assertThat( - channel.toString(), - allOf( - containsString("DirectResponseChannel"), - containsString('{' + ACTION + '}'), - containsString("TaskTransportChannel{task=" + task.getId() + '}') - ) - ); - assertThat(new ChannelActionListener<>(channel).toString(), containsString(channel.toString())); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - }); - serviceB.registerRequestHandler(ACTION, ThreadPool.Names.SAME, TransportRequest.Empty::new, (request, channel, task) -> { - assertThat( - channel.toString(), - allOf( - containsString("TcpTransportChannel"), - containsString('{' + ACTION + '}'), - containsString("TaskTransportChannel{task=" + task.getId() + '}'), - containsString("localAddress="), - containsString(serviceB.getLocalNode().getAddress().toString()) - ) - ); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - }); + serviceA.registerRequestHandler( + ACTION, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TransportRequest.Empty::new, + (request, channel, task) -> { + assertThat( + channel.toString(), + allOf( + containsString("DirectResponseChannel"), + containsString('{' + ACTION + '}'), + containsString("TaskTransportChannel{task=" + task.getId() + '}') + ) + ); + assertThat(new ChannelActionListener<>(channel).toString(), containsString(channel.toString())); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + ); + serviceB.registerRequestHandler( + ACTION, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TransportRequest.Empty::new, + (request, channel, task) -> { + assertThat( + channel.toString(), + allOf( + containsString("TcpTransportChannel"), + containsString('{' + ACTION + '}'), + containsString("TaskTransportChannel{task=" + task.getId() + '}'), + containsString("localAddress="), + containsString(serviceB.getLocalNode().getAddress().toString()) + ) + ); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + ); PlainActionFuture.get( f -> submitRequest( @@ -3233,7 +3323,7 @@ public void writeTo(StreamOutput out) throws IOException { final var requestSize = between(0, ByteSizeUnit.MB.toIntBytes(1)); final var responseSize = between(0, ByteSizeUnit.MB.toIntBytes(1)); - serviceB.registerRequestHandler(ACTION, ThreadPool.Names.SAME, Request::new, (request, channel, task) -> { + serviceB.registerRequestHandler(ACTION, EsExecutors.DIRECT_EXECUTOR_SERVICE, Request::new, (request, channel, task) -> { assertEquals(requestSize, request.refSize); channel.sendResponse(new Response(responseSize)); }); diff --git a/test/framework/src/test/java/org/elasticsearch/common/util/concurrent/DeterministicTaskQueueTests.java b/test/framework/src/test/java/org/elasticsearch/common/util/concurrent/DeterministicTaskQueueTests.java index fb90b6b87f219..dc51a01da844f 100644 --- a/test/framework/src/test/java/org/elasticsearch/common/util/concurrent/DeterministicTaskQueueTests.java +++ b/test/framework/src/test/java/org/elasticsearch/common/util/concurrent/DeterministicTaskQueueTests.java @@ -398,9 +398,11 @@ public void testSameExecutor() { final AtomicBoolean executed = new AtomicBoolean(false); final AtomicBoolean executedNested = new AtomicBoolean(false); threadPool.generic().execute(() -> { - threadPool.executor(ThreadPool.Names.SAME).execute(() -> executedNested.set(true)); + final var executor = threadPool.executor(ThreadPool.Names.SAME); + assertSame(EsExecutors.DIRECT_EXECUTOR_SERVICE, executor); + executor.execute(() -> assertTrue(executedNested.compareAndSet(false, true))); assertThat(executedNested.get(), is(true)); - executed.set(true); + assertTrue(executed.compareAndSet(false, true)); }); taskQueue.runAllRunnableTasks(); assertThat(executed.get(), is(true)); diff --git a/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java b/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java index 2925254ea992b..6d5542db5e529 100644 --- a/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java +++ b/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java @@ -322,7 +322,12 @@ public void handleException(TransportException exp) { } private void registerRequestHandler(TransportService transportService, TransportRequestHandler handler) { - transportService.registerRequestHandler(TEST_ACTION, ThreadPool.Names.GENERIC, TestRequest::new, handler); + transportService.registerRequestHandler( + TEST_ACTION, + transportService.getThreadPool().executor(ThreadPool.Names.GENERIC), + TestRequest::new, + handler + ); } private void send( diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/CloseToAssertion.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/CloseToAssertion.java index cff3deb2c1068..557556abc743e 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/CloseToAssertion.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/CloseToAssertion.java @@ -36,14 +36,14 @@ public static CloseToAssertion parse(XContentParser parser) throws IOException { throw new IllegalArgumentException("expected a map with value and error but got a map with " + map.size() + " fields"); } Object valObj = map.get("value"); - if (valObj == null) { - throw new IllegalArgumentException("value is missing"); + if (valObj instanceof Number == false) { + throw new IllegalArgumentException("value is missing or not a number"); } Object errObj = map.get("error"); if (errObj instanceof Number == false) { throw new IllegalArgumentException("error is missing or not a number"); } - return new CloseToAssertion(location, fieldValueTuple.v1(), valObj, ((Number) errObj).doubleValue()); + return new CloseToAssertion(location, fieldValueTuple.v1(), ((Number) valObj).doubleValue(), ((Number) errObj).doubleValue()); } else { throw new IllegalArgumentException( "expected a map with value and error but got " + fieldValueTuple.v2().getClass().getSimpleName() @@ -56,7 +56,7 @@ public static CloseToAssertion parse(XContentParser parser) throws IOException { private final double error; - public CloseToAssertion(XContentLocation location, String field, Object expectedValue, Double error) { + public CloseToAssertion(XContentLocation location, String field, Double expectedValue, Double error) { super(location, field, expectedValue); this.error = error; } @@ -69,9 +69,9 @@ public final double getError() { protected void doAssert(Object actualValue, Object expectedValue) { logger.trace("assert that [{}] is close to [{}] with error [{}] (field [{}])", actualValue, expectedValue, error, getField()); if (actualValue instanceof Number actualValueNumber) { - assertThat(actualValueNumber.doubleValue(), closeTo(((Number) expectedValue).doubleValue(), error)); + assertThat(actualValueNumber.doubleValue(), closeTo((Double) expectedValue, error)); } else { - throw new AssertionError("expected a value close to " + expectedValue + " but got " + actualValue + ", which is not a number"); + throw new AssertionError("excpected a value close to " + expectedValue + " but got " + actualValue + ", which is not a number"); } } } diff --git a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/ObjectPathTests.java b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/ObjectPathTests.java index 94c459b773d08..bca1521b94753 100644 --- a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/ObjectPathTests.java +++ b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/ObjectPathTests.java @@ -336,4 +336,43 @@ public void testEvaluateArrayAsRoot() throws Exception { assertThat(object, instanceOf(String.class)); assertThat(object, equalTo("test2")); } + + public void testEvaluateArraySize() throws Exception { + XContentBuilder xContentBuilder = randomXContentBuilder(); + xContentBuilder.startObject(); + xContentBuilder.startArray("test-array"); + xContentBuilder.startObject(); + xContentBuilder.field("foo", "bar"); + xContentBuilder.endObject(); + xContentBuilder.value(1); + xContentBuilder.value("test-string"); + xContentBuilder.endArray(); + xContentBuilder.endObject(); + + ObjectPath objectPath = ObjectPath.createFromXContent( + XContentFactory.xContent(xContentBuilder.contentType()), + BytesReference.bytes(xContentBuilder) + ); + + assertEquals(3, objectPath.evaluateArraySize("test-array")); + } + + public void testEvaluateMapKeys() throws Exception { + XContentBuilder xContentBuilder = randomXContentBuilder(); + xContentBuilder.startObject(); + xContentBuilder.startObject("test-object"); + xContentBuilder.field("key1", "bar"); + xContentBuilder.field("key2", 42); + xContentBuilder.startObject("key3"); + xContentBuilder.endObject(); + xContentBuilder.endObject(); + xContentBuilder.endObject(); + + ObjectPath objectPath = ObjectPath.createFromXContent( + XContentFactory.xContent(xContentBuilder.contentType()), + BytesReference.bytes(xContentBuilder) + ); + + assertEquals(Set.of("key1", "key2", "key3"), objectPath.evaluateMapKeys("test-object")); + } } diff --git a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java index 705f403632f62..1ac0bff285b96 100644 --- a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java +++ b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java @@ -169,7 +169,7 @@ public void testInvalidCloseTo() throws Exception { parser = createParser(YamlXContent.yamlXContent, "{ field: { foo: 13, bar: 15 } }"); exception = expectThrows(IllegalArgumentException.class, () -> CloseToAssertion.parse(parser)); - assertThat(exception.getMessage(), equalTo("value is missing")); + assertThat(exception.getMessage(), equalTo("value is missing or not a number")); } public void testExists() throws IOException { diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 08969b7a779dc..08fad5dd3b83c 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -96,7 +96,7 @@ public HistogramFieldMapper build(MapperBuilderContext context) { name, new HistogramFieldType(context.buildFullName(name), meta.getValue()), multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, this ); } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java index ee7924428dd0b..eb7cc9c51c62e 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; @@ -59,13 +60,15 @@ public void init() { } public void testAvailable() throws Exception { - AnalyticsInfoTransportAction featureSet = new AnalyticsInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + AnalyticsInfoTransportAction featureSet = new AnalyticsInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); Client client = mockClient(); AnalyticsUsageTransportAction usageAction = new AnalyticsUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, client @@ -84,14 +87,16 @@ public void testAvailable() throws Exception { } public void testEnabled() throws Exception { - AnalyticsInfoTransportAction featureSet = new AnalyticsInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + AnalyticsInfoTransportAction featureSet = new AnalyticsInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); assertTrue(featureSet.enabled()); Client client = mockClient(); AnalyticsUsageTransportAction usageAction = new AnalyticsUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, client diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java index 4f9d803c50c25..6b0b53aedd12d 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; @@ -36,6 +37,7 @@ import static java.util.stream.Collectors.toList; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -44,6 +46,10 @@ public TransportAnalyticsStatsAction action(AnalyticsUsage usage) { TransportService transportService = mock(TransportService.class); ThreadPool threadPool = mock(ThreadPool.class); + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + ClusterService clusterService = mock(ClusterService.class); DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("nodeId"); when(clusterService.localNode()).thenReturn(discoveryNode); diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java index 93ac8bdc85896..91dc4f7d9d479 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata; @@ -39,10 +40,12 @@ public class TransportDeleteAutoscalingPolicyActionTests extends AutoscalingTestCase { public void testWriteBlock() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportDeleteAutoscalingPolicyAction action = new TransportDeleteAutoscalingPolicyAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); @@ -61,10 +64,12 @@ public void testWriteBlock() { } public void testNoWriteBlock() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportDeleteAutoscalingPolicyAction action = new TransportDeleteAutoscalingPolicyAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyActionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyActionTests.java index 17f97e2ddc050..12d9fcee8bd10 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyActionTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.autoscaling.AutoscalingLicenseChecker; @@ -37,10 +38,12 @@ public class TransportGetAutoscalingPolicyActionTests extends AutoscalingTestCase { public void testReadBlock() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportGetAutoscalingPolicyAction action = new TransportGetAutoscalingPolicyAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), new AutoscalingLicenseChecker(() -> true) @@ -64,10 +67,12 @@ public void testReadBlock() { } public void testNoReadBlock() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportGetAutoscalingPolicyAction action = new TransportGetAutoscalingPolicyAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), new AutoscalingLicenseChecker(() -> true) diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionTests.java index a39b3113a96ca..bca2fc4ab96af 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.autoscaling.AutoscalingLicenseChecker; @@ -41,10 +42,12 @@ public class TransportPutAutoscalingPolicyActionTests extends AutoscalingTestCas private static final PolicyValidator NO_VALIDATION = policy -> {}; public void testWriteBlock() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportPutAutoscalingPolicyAction action = new TransportPutAutoscalingPolicyAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), NO_VALIDATION, @@ -65,10 +68,12 @@ public void testWriteBlock() { } public void testNoWriteBlock() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportPutAutoscalingPolicyAction action = new TransportPutAutoscalingPolicyAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), NO_VALIDATION, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java index 7a7d375781e16..1d81549f7adec 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; @@ -48,8 +49,9 @@ public void init() { } public void testAvailable() { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); CCRInfoTransportAction featureSet = new CCRInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), Settings.EMPTY, licenseState @@ -64,8 +66,9 @@ public void testAvailable() { public void testEnabled() { Settings.Builder settings = Settings.builder().put("xpack.ccr.enabled", false); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); CCRInfoTransportAction featureSet = new CCRInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), settings.build(), licenseState @@ -73,13 +76,14 @@ public void testEnabled() { assertThat(featureSet.enabled(), equalTo(false)); settings = Settings.builder().put("xpack.ccr.enabled", true); - featureSet = new CCRInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState); + featureSet = new CCRInfoTransportAction(transportService, mock(ActionFilters.class), settings.build(), licenseState); assertThat(featureSet.enabled(), equalTo(true)); } public void testName() { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); CCRInfoTransportAction featureSet = new CCRInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), Settings.EMPTY, licenseState @@ -137,10 +141,12 @@ public void testUsageStats() throws Exception { ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metadata(metadata).build(); Mockito.when(clusterService.state()).thenReturn(clusterState); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var usageAction = new CCRUsageTransportAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, Settings.EMPTY, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionActionTests.java index 95ab1c2323f7d..540a6a8f7bcb5 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionActionTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.ccr.repository.CcrRestoreSourceService; @@ -54,9 +55,8 @@ public void testPrivilegeForActions() { public void testActionNames() { final ActionFilters actionFilters = mock(ActionFilters.class); - final TransportService transportService = mock(TransportService.class); + final TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final CcrRestoreSourceService ccrRestoreSourceService = mock(CcrRestoreSourceService.class); - final var action = new ClearCcrRestoreSessionAction.TransportAction(actionFilters, transportService, ccrRestoreSourceService); assertThat(action.actionName, equalTo(ClearCcrRestoreSessionAction.NAME)); @@ -70,7 +70,7 @@ public void testActionNames() { public void testRequestedShardIdMustBeConsistentWithSessionShardId() { final ActionFilters actionFilters = mock(ActionFilters.class); - final TransportService transportService = mock(TransportService.class); + final TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final CcrRestoreSourceService ccrRestoreSourceService = mock(CcrRestoreSourceService.class); final ShardId expectedShardId = mock(ShardId.class); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkActionTests.java index 7357873bba0f2..629b98a25ba4b 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.ccr.repository.CcrRestoreSourceService; @@ -59,7 +60,7 @@ public void testPrivilegeForActions() { public void testActionNames() { final ActionFilters actionFilters = mock(ActionFilters.class); final BigArrays bigArrays = mock(BigArrays.class); - final TransportService transportService = mock(TransportService.class); + final TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final CcrRestoreSourceService ccrRestoreSourceService = mock(CcrRestoreSourceService.class); final var action = new GetCcrRestoreFileChunkAction.TransportAction( @@ -82,7 +83,7 @@ public void testActionNames() { public void testRequestedShardIdMustBeConsistentWithSessionShardId() { final ActionFilters actionFilters = mock(ActionFilters.class); final BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), ByteSizeValue.ofBytes(1024)); - final TransportService transportService = mock(TransportService.class); + final TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final CcrRestoreSourceService ccrRestoreSourceService = mock(CcrRestoreSourceService.class); final String sessionUUID = UUIDs.randomBase64UUID(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index 0245e559496c8..18c72ea8d42cb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -281,13 +281,12 @@ static RoleDescriptor kibanaSystem(String name) { RoleDescriptor.IndicesPrivileges.builder() .indices("logs-ti_*_latest.*") .privileges( - // Require "create_index", "delete_index", "read", "index", "delete", IndicesAliasesAction.NAME, and - // UpdateSettingsAction.NAME for transform "create_index", "delete_index", "read", "index", "delete", + "manage", IndicesAliasesAction.NAME, UpdateSettingsAction.NAME ) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java index 7eca64b82cbb4..1c09fcb746f49 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java @@ -135,7 +135,7 @@ public TransportTermsEnumAction( transportService.registerRequestHandler( transportShardAction, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, NodeTermsEnumRequest::new, new NodeTransportHandler() ); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java index 0801ae1834954..b457a77e38f01 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo.FeatureSet; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import java.util.ArrayList; @@ -59,8 +60,9 @@ public void testDoExecute() throws Exception { }); } + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); TransportXPackInfoAction action = new TransportXPackInfoAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), licenseService, client diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ecs-dynamic-mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ecs-dynamic-mappings.json index 3df1c1bd7928a..fc29fc98dca96 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ecs-dynamic-mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ecs-dynamic-mappings.json @@ -83,24 +83,6 @@ }, { "ecs_path_match_keyword_and_match_only_text": { - "mapping": { - "fields": { - "text": { - "type": "match_only_text" - } - }, - "type": "keyword" - }, - "path_match": [ - "*file.path", - "*file.target_path", - "*os.full", - "user_agent.original" - ] - } - }, - { - "ecs_match_keyword_and_match_only_text": { "mapping": { "fields": { "text": { @@ -114,7 +96,13 @@ "*.executable", "*.name", "*.working_directory", - "*.full_name" + "*.full_name", + "*file.path", + "*file.target_path", + "*os.full", + "email.subject", + "vulnerability.description", + "user_agent.original" ] } }, diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java index e60f8c1ccae6e..190ac8d388bcf 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ESTestCase; @@ -33,6 +34,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; public class TransportNodeDeprecationCheckActionTests extends ESTestCase { @@ -62,6 +64,11 @@ public void testNodeOperation() { DiscoveryNode node = Mockito.mock(DiscoveryNode.class); when(node.getId()).thenReturn("mock-node"); TransportService transportService = Mockito.mock(TransportService.class); + + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + when(transportService.getLocalNode()).thenReturn(node); PluginsService pluginsService = Mockito.mock(PluginsService.class); ActionFilters actionFilters = Mockito.mock(ActionFilters.class); diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java index b122b59813b43..84b55a5fa8009 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java @@ -209,7 +209,6 @@ public boolean validateClusterForming() { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98485") public void testDownsampleIndexWithRollingRestart() throws Exception { try (InternalTestCluster cluster = internalCluster()) { final List masterNodes = cluster.startMasterOnlyNodes(1); @@ -296,11 +295,10 @@ private void startDownsampleTaskDuringDisruption( } catch (Exception e) { throw new AssertionError(e); } - }, 60, TimeUnit.SECONDS); + }, 120, TimeUnit.SECONDS); disruptionEnd.await(); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98485") public void testDownsampleIndexWithFullClusterRestart() throws Exception { try (InternalTestCluster cluster = internalCluster()) { final List masterNodes = cluster.startMasterOnlyNodes(1); @@ -428,7 +426,7 @@ private void downsample(final String sourceIndex, final String downsampleIndex, assertAcked( internalCluster().client() .execute(DownsampleAction.INSTANCE, new DownsampleAction.Request(sourceIndex, downsampleIndex, TIMEOUT, config)) - .actionGet() + .actionGet(TIMEOUT.millis()) ); } diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java index 89ff330051418..76577b12cd8d9 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.junit.Before; @@ -28,7 +29,8 @@ public class InternalExecutePolicyActionTests extends ESTestCase { @Before public void instantiateTransportAction() { - transportAction = new InternalExecutePolicyAction.Transport(mock(TransportService.class), mock(ActionFilters.class), null, null); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + transportAction = new InternalExecutePolicyAction.Transport(transportService, mock(ActionFilters.class), null, null); } public void testSelectNodeForPolicyExecution() { diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java index c350465f0f725..c9d2f97eb5624 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ObjectPath; @@ -54,12 +55,14 @@ public void init() throws Exception { } public void testAvailable() { - EqlInfoTransportAction featureSet = new EqlInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + EqlInfoTransportAction featureSet = new EqlInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testEnabled() { - EqlInfoTransportAction featureSet = new EqlInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + EqlInfoTransportAction featureSet = new EqlInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); } @@ -93,10 +96,12 @@ public void testUsageStats() throws Exception { when(mockNode.getId()).thenReturn("mocknode"); when(clusterService.localNode()).thenReturn(mockNode); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var usageAction = new EqlUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, client diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/ConvertEvaluatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/ConvertEvaluatorImplementer.java index a7dce57348fb6..256dd690a7fb6 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/ConvertEvaluatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/ConvertEvaluatorImplementer.java @@ -40,6 +40,7 @@ public class ConvertEvaluatorImplementer { private final TypeElement declarationType; private final ExecutableElement processFunction; + private final String extraName; private final ClassName implementation; private final TypeName argumentType; private final TypeName resultType; @@ -50,6 +51,7 @@ public ConvertEvaluatorImplementer(Elements elements, ExecutableElement processF if (processFunction.getParameters().size() != 1) { throw new IllegalArgumentException("processing function should have exactly one parameter"); } + this.extraName = extraName; this.argumentType = TypeName.get(processFunction.getParameters().get(0).asType()); this.resultType = TypeName.get(processFunction.getReturnType()); @@ -96,7 +98,7 @@ private MethodSpec ctor() { private MethodSpec name() { MethodSpec.Builder builder = MethodSpec.methodBuilder("name").addModifiers(Modifier.PUBLIC); builder.addAnnotation(Override.class).returns(String.class); - builder.addStatement("return $S", declarationType.getSimpleName()); + builder.addStatement("return $S", declarationType.getSimpleName() + extraName); return builder.build(); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java index 9106508f7e262..49ec715604507 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java @@ -62,12 +62,21 @@ public abstract sealed class BlockHash implements Releasable, SeenGroupIds // * Creates a specialized hash table that maps one or more {@link Block}s to ids. * @param emitBatchSize maximum batch size to be emitted when handling combinatorial * explosion of groups caused by multivalued fields + * @param allowBrokenOptimizations true ot allow optimizations with bad null handling. We will fix their + * null handling and remove this flag, but we need to disable these in + * production until we can. And this lets us continue to compile and + * test them. */ - public static BlockHash build(List groups, BigArrays bigArrays, int emitBatchSize) { + public static BlockHash build( + List groups, + BigArrays bigArrays, + int emitBatchSize, + boolean allowBrokenOptimizations + ) { if (groups.size() == 1) { return newForElementType(groups.get(0).channel(), groups.get(0).elementType(), bigArrays); } - if (groups.size() == 2) { + if (allowBrokenOptimizations && groups.size() == 2) { var g1 = groups.get(0); var g2 = groups.get(1); if (g1.elementType() == ElementType.LONG && g2.elementType() == ElementType.LONG) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverRunner.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverRunner.java index 9ab40b15e4623..c687ce7f864f1 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverRunner.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverRunner.java @@ -9,14 +9,11 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.util.concurrent.CountDown; import org.elasticsearch.core.Releasables; import org.elasticsearch.tasks.TaskCancelledException; -import org.elasticsearch.threadpool.ThreadPool; import java.util.List; -import java.util.Map; import java.util.concurrent.atomic.AtomicReference; /** @@ -90,33 +87,4 @@ private void done() { start(driver, driverListener); } } - - /** - * Run all the of the listed drivers in the supplier {@linkplain ThreadPool}. - * @return the headers added to the context while running the drivers - */ - public static Map> runToCompletion(ThreadPool threadPool, int maxIterations, List drivers) { - DriverRunner runner = new DriverRunner() { - @Override - protected void start(Driver driver, ActionListener driverListener) { - Driver.start(threadPool.executor("esql"), driver, maxIterations, driverListener); - } - }; - AtomicReference>> responseHeaders = new AtomicReference<>(); - PlainActionFuture future = new PlainActionFuture<>(); - runner.runToCompletion(drivers, new ActionListener<>() { - @Override - public void onResponse(Void unused) { - responseHeaders.set(threadPool.getThreadContext().getResponseHeaders()); - future.onResponse(null); - } - - @Override - public void onFailure(Exception e) { - future.onFailure(e); - } - }); - future.actionGet(); - return responseHeaders.get(); - } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java index 8a39bc5110c93..53d5a66de7b66 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java @@ -36,9 +36,9 @@ public class DriverTaskRunner { public static final String ACTION_NAME = "internal:data/read/esql/compute"; private final TransportService transportService; - public DriverTaskRunner(TransportService transportService, String executorName) { + public DriverTaskRunner(TransportService transportService, Executor executor) { this.transportService = transportService; - transportService.registerRequestHandler(ACTION_NAME, executorName, DriverRequest::new, new DriverRequestHandler()); + transportService.registerRequestHandler(ACTION_NAME, executor, DriverRequest::new, new DriverRequestHandler()); } public void executeDrivers(Task parentTask, List drivers, Executor executor, ActionListener listener) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashAggregationOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashAggregationOperator.java index 5ca3c854c00b2..585ab18c75e2a 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashAggregationOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashAggregationOperator.java @@ -39,7 +39,7 @@ public record HashAggregationOperatorFactory( ) implements OperatorFactory { @Override public Operator get(DriverContext driverContext) { - return new HashAggregationOperator(aggregators, () -> BlockHash.build(groups, bigArrays, maxPageSize), driverContext); + return new HashAggregationOperator(aggregators, () -> BlockHash.build(groups, bigArrays, maxPageSize, false), driverContext); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java index bc44a3a6d305c..996561121df8f 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java @@ -422,7 +422,7 @@ private static class ValuesAggregator implements Releasable { this.extractor = new ValuesSourceReaderOperator(sources, docChannel, groupingField); this.aggregator = new HashAggregationOperator( aggregatorFactories, - () -> BlockHash.build(List.of(new GroupSpec(channelIndex, sources.get(0).elementType())), bigArrays, maxPageSize), + () -> BlockHash.build(List.of(new GroupSpec(channelIndex, sources.get(0).elementType())), bigArrays, maxPageSize, false), driverContext ); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java index c3efd67579d66..6db4a8c4fe37d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java @@ -79,13 +79,13 @@ public ExchangeService(Settings settings, ThreadPool threadPool, String executor public void registerTransportHandler(TransportService transportService) { transportService.registerRequestHandler( EXCHANGE_ACTION_NAME, - requestExecutorName, + threadPool.executor(requestExecutorName), ExchangeRequest::new, new ExchangeTransportAction() ); transportService.registerRequestHandler( OPEN_EXCHANGE_ACTION_NAME, - requestExecutorName, + threadPool.executor(requestExecutorName), OpenExchangeRequest::new, new OpenExchangeRequestHandler() ); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java index 639e8e401a726..135877e4f5405 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java @@ -247,7 +247,8 @@ public String toString() { () -> BlockHash.build( List.of(new HashAggregationOperator.GroupSpec(0, ElementType.BYTES_REF)), bigArrays, - randomPageSize() + randomPageSize(), + false ), driverContext ) diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashRandomizedTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashRandomizedTests.java index c585108a89fd0..f00c8d6d3cb1f 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashRandomizedTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashRandomizedTests.java @@ -168,7 +168,7 @@ private BlockHash newBlockHash(int emitBatchSize, List types) { MockBigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, new NoneCircuitBreakerService()); return forcePackedHash ? new PackedValuesBlockHash(specs, bigArrays, emitBatchSize) - : BlockHash.build(specs, bigArrays, emitBatchSize); + : BlockHash.build(specs, bigArrays, emitBatchSize, true); } private static class KeyComparator implements Comparator> { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java index e2c848f2cd7ef..553ce83d8002c 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java @@ -1032,7 +1032,7 @@ private void hash(Consumer callback, int emitBatchSize, Block... va try ( BlockHash blockHash = forcePackedHash ? new PackedValuesBlockHash(specs, bigArrays, emitBatchSize) - : BlockHash.build(specs, bigArrays, emitBatchSize) + : BlockHash.build(specs, bigArrays, emitBatchSize, true) ) { hash(true, blockHash, callback, values); } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java index f32ef67ab766c..0f47b5c29cc3b 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java @@ -7,6 +7,8 @@ package org.elasticsearch.compute.operator; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.settings.Settings; @@ -17,6 +19,7 @@ import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.compute.data.Page; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.indices.CrankyCircuitBreakerService; import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.threadpool.TestThreadPool; @@ -178,8 +181,16 @@ public static void runDriver(List drivers) { getTestClass().getSimpleName(), new FixedExecutorBuilder(Settings.EMPTY, "esql", numThreads, 1024, "esql", EsExecutors.TaskTrackingConfig.DEFAULT) ); + var driverRunner = new DriverRunner() { + @Override + protected void start(Driver driver, ActionListener driverListener) { + Driver.start(threadPool.executor("esql"), driver, between(1, 10000), driverListener); + } + }; + PlainActionFuture future = new PlainActionFuture<>(); try { - DriverRunner.runToCompletion(threadPool, between(1, 10000), drivers); + driverRunner.runToCompletion(drivers, future); + future.actionGet(TimeValue.timeValueSeconds(30)); } finally { terminate(threadPool); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec index ba89685716059..9108b0d6eedb6 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec @@ -108,3 +108,61 @@ from employees | eval true_bool = null is null, false_bool = null is not null, n true_bool:boolean | negated_true:boolean | false_bool:boolean | negated_false:boolean | first_name:keyword | last_name:keyword true | false | false | true | Georgi | Facello ; + + +repetitiveEval +from employees | sort emp_no | keep emp_no | eval sum = emp_no + 1 +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| limit 3 +; + +emp_no:i | sum:i +10001 | 3230324 +10002 | 3230647 +10003 | 3230970 +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec index aa4bcac3fc9be..93b7ebe2156ed 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec @@ -266,38 +266,30 @@ d: double | s:double ; log10ofNegative -row d = -1.0 | eval s = is_nan(log10(d)); +row d = -1.0 | eval s = log10(d); +warning:Line 1:25: evaluation of [log10(d)] failed, treating result as null. Only first 20 failures recorded. +warning:java.lang.ArithmeticException: Log of non-positive number -d:double | s:boolean --1.0 | true -; - -log10ofNan -row d = 0.0/0.0 | eval s = is_nan(log10(d)); - -d:double | s:boolean -NaN | true +d:double | s:double +-1.0 | null ; log10ofZero -row d = 0.0 |eval s = is_infinite(log10(d)); +row d = 0.0 | eval s = log10(d); +warning:Line 1:24: evaluation of [log10(d)] failed, treating result as null. Only first 20 failures recorded. +warning:java.lang.ArithmeticException: Log of non-positive number -d:double | s:boolean -0.0 | true +d:double | s:double +0.0 | null ; log10ofNegativeZero -row d = -0.0 |eval s = is_infinite(log10(d)); +row d = -0.0 | eval s = log10(d); +warning:Line 1:25: evaluation of [log10(d)] failed, treating result as null. Only first 20 failures recorded. +warning:java.lang.ArithmeticException: Log of non-positive number -d:double | s:boolean --0.0 | true -; - -log10ofInfinite -row d = 1/0.0 | eval s = is_infinite(log10(d)); - -d:double | s:boolean -Infinity | true +d:double | s:double +-0.0 | null ; log10ofLong @@ -995,8 +987,9 @@ i:ul | c:ul | f:ul sqrt // tag::sqrt[] ROW d = 100.0 -| EVAL s = SQRT(d); +| EVAL s = SQRT(d) // end::sqrt[] +; // tag::sqrt-result[] d: double | s:double @@ -1063,8 +1056,9 @@ Infinity | true least // tag::least[] ROW a = 10, b = 20 -| EVAL l = LEAST(a, b); +| EVAL l = LEAST(a, b) // end::least[] +; // tag::least-result[] a:integer | b:integer | l:integer @@ -1090,8 +1084,9 @@ l:integer greatest // tag::greatest[] ROW a = 10, b = 20 -| EVAL g = GREATEST(a, b); +| EVAL g = GREATEST(a, b) // end::greatest[] +; // tag::greatest-result[] a:integer | b:integer | g:integer diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec index efd1e3591e2b8..cea3490f156bc 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec @@ -81,13 +81,13 @@ to_integer |to_integer(arg1) to_ip |to_ip(arg1) to_long |to_long(arg1) to_radians |to_radians(arg1) -to_str |to_str(arg1) -to_string |to_string(arg1) +to_str |to_str(v) +to_string |to_string(v) to_ul |to_ul(arg1) to_ulong |to_ulong(arg1) to_unsigned_long |to_unsigned_long(arg1) -to_ver |to_ver(arg1) -to_version |to_version(arg1) +to_ver |to_ver(v) +to_version |to_version(v) trim |trim(arg1) ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index db1800fabeed7..c1f623fda251d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -290,23 +290,35 @@ null ; byStringAndLong -from employees | eval trunk_worked_seconds = avg_worked_seconds / 100000000 * 100000000 | stats c = count(gender) by gender, trunk_worked_seconds | sort c desc; +FROM employees +| EVAL trunk_worked_seconds = avg_worked_seconds / 100000000 * 100000000 +| STATS c = COUNT(gender) by gender, trunk_worked_seconds +| SORT c desc; c:long | gender:keyword | trunk_worked_seconds:long -30 | M | 300000000 -27 | M | 200000000 -22 | F | 300000000 -11 | F | 200000000 +30 | M | 300000000 +27 | M | 200000000 +22 | F | 300000000 +11 | F | 200000000 + 0 | null | 200000000 + 0 | null | 300000000 ; byStringAndLongWithAlias -from employees | eval trunk_worked_seconds = avg_worked_seconds / 100000000 * 100000000 | rename gender as g, trunk_worked_seconds as tws | keep g, tws | stats c = count(g) by g, tws | sort c desc; +FROM employees +| EVAL trunk_worked_seconds = avg_worked_seconds / 100000000 * 100000000 +| RENAME gender as g, trunk_worked_seconds as tws +| KEEP g, tws +| STATS c = count(g) by g, tws +| SORT c desc; c:long | g:keyword | tws:long -30 | M | 300000000 -27 | M | 200000000 -22 | F | 300000000 -11 | F | 200000000 +30 | M | 300000000 +27 | M | 200000000 +22 | F | 300000000 +11 | F | 200000000 + 0 | null | 200000000 + 0 | null | 300000000 ; byStringAndString @@ -324,35 +336,45 @@ c:long | gender:keyword | hire_year_str:keyword ; byLongAndLong -from employees | eval trunk_worked_seconds = avg_worked_seconds / 100000000 * 100000000 | stats c = count(languages.long) by languages.long, trunk_worked_seconds | sort c desc, languages.long, trunk_worked_seconds; +FROM employees +| EVAL trunk_worked_seconds = avg_worked_seconds / 100000000 * 100000000 +| STATS c = COUNT(languages.long) BY languages.long, trunk_worked_seconds +| SORT c DESC, languages.long, trunk_worked_seconds; c:long | languages.long:long | trunk_worked_seconds:long -15 |5 |300000000 -11 |2 |300000000 -10 |4 |300000000 -9 |3 |200000000 -8 |1 |200000000 -8 |2 |200000000 -8 |3 |300000000 -8 |4 |200000000 -7 |1 |300000000 -6 |5 |200000000 +15 |5 |300000000 +11 |2 |300000000 +10 |4 |300000000 +9 |3 |200000000 +8 |1 |200000000 +8 |2 |200000000 +8 |3 |300000000 +8 |4 |200000000 +7 |1 |300000000 +6 |5 |200000000 +0 |null |200000000 +0 |null |300000000 ; byUnmentionedLongAndLong -from employees | eval trunk_worked_seconds = avg_worked_seconds / 100000000 * 100000000 | stats c = count(gender) by languages.long, trunk_worked_seconds | sort c desc, trunk_worked_seconds; +FROM employees +| EVAL trunk_worked_seconds = avg_worked_seconds / 100000000 * 100000000 +| STATS c = count(gender) by languages.long, trunk_worked_seconds +| SORT c desc, trunk_worked_seconds; c:long | languages.long:long | trunk_worked_seconds:long -13 |5 |300000000 -10 |2 |300000000 -9 |3 |200000000 -9 |4 |300000000 -8 |4 |200000000 -8 |3 |300000000 -7 |1 |200000000 -6 |2 |200000000 -6 |1 |300000000 -4 |5 |200000000 +13 |5 |300000000 +10 |2 |300000000 + 9 |3 |200000000 + 9 |4 |300000000 + 8 |4 |200000000 + 8 |3 |300000000 + 7 |1 |200000000 + 6 |2 |200000000 + 6 |null |300000000 + 6 |1 |300000000 + 4 |null |200000000 + 4 |5 |200000000 ; byUnmentionedIntAndLong diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/lookup/EnrichLookupIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/lookup/EnrichLookupIT.java index 8f15c6b07f5e3..86d082f1051ab 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/lookup/EnrichLookupIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/lookup/EnrichLookupIT.java @@ -8,6 +8,8 @@ package org.elasticsearch.xpack.esql.lookup; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Settings; @@ -22,12 +24,14 @@ import org.elasticsearch.compute.operator.DriverRunner; import org.elasticsearch.compute.operator.OutputOperator; import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; import org.elasticsearch.xpack.esql.enrich.EnrichLookupOperator; +import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.elasticsearch.xpack.esql.plugin.TransportEsqlQueryAction; import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.NamedExpression; @@ -38,6 +42,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; @@ -84,7 +89,7 @@ public void testSimple() { ); DiscoveryNode clientNode = randomFrom(clusterService().state().nodes().stream().toList()); - var lookupService = internalCluster().getInstance(TransportEsqlQueryAction.class, clientNode.getName()).enrichLookupService(); + TransportEsqlQueryAction queryAction = internalCluster().getInstance(TransportEsqlQueryAction.class, clientNode.getName()); TransportService transportService = internalCluster().getInstance(TransportService.class, clientNode.getName()); EsqlQueryRequest parentRequest = new EsqlQueryRequest(); @@ -95,7 +100,7 @@ public void testSimple() { parentTask, randomIntBetween(1, 3), 0, - lookupService, + queryAction.enrichLookupService(), "users", "match", "uid", @@ -132,11 +137,18 @@ public void testSimple() { DateFormatter dateFmt = DateFormatter.forPattern("yyyy-MM-dd"); - DriverRunner.runToCompletion( - internalCluster().getInstance(TransportService.class).getThreadPool(), - between(1, 10_000), - List.of(new Driver(new DriverContext(), sourceOperator, List.of(enrichOperator), outputOperator, () -> {})) - ); + var runner = new DriverRunner() { + final Executor executor = transportService.getThreadPool().executor(EsqlPlugin.ESQL_THREAD_POOL_NAME); + + @Override + protected void start(Driver driver, ActionListener listener) { + Driver.start(executor, driver, between(1, 1000), listener); + } + }; + Driver driver = new Driver(new DriverContext(), sourceOperator, List.of(enrichOperator), outputOperator, () -> {}); + PlainActionFuture future = new PlainActionFuture<>(); + runner.runToCompletion(List.of(driver), future); + future.actionGet(TimeValue.timeValueSeconds(30)); transportService.getTaskManager().unregister(parentTask); Page output = outputPage.get(); assertThat(output.getBlockCount(), equalTo(4)); diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromDoubleEvaluator.java index ee95635e52b0b..dba115f4f7c29 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromDoubleEvaluator.java @@ -29,7 +29,7 @@ public ToBooleanFromDoubleEvaluator(EvalOperator.ExpressionEvaluator field, Sour @Override public String name() { - return "ToBoolean"; + return "ToBooleanFromDouble"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromIntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromIntEvaluator.java index 6005ffe9be356..d20179fd7baed 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromIntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromIntEvaluator.java @@ -29,7 +29,7 @@ public ToBooleanFromIntEvaluator(EvalOperator.ExpressionEvaluator field, Source @Override public String name() { - return "ToBoolean"; + return "ToBooleanFromInt"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromLongEvaluator.java index 06306a9a2acab..7ab2d656a59cb 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromLongEvaluator.java @@ -29,7 +29,7 @@ public ToBooleanFromLongEvaluator(EvalOperator.ExpressionEvaluator field, Source @Override public String name() { - return "ToBoolean"; + return "ToBooleanFromLong"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromStringEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromStringEvaluator.java index b71ee10036822..d70d0365aaf4d 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromStringEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromStringEvaluator.java @@ -30,7 +30,7 @@ public ToBooleanFromStringEvaluator(EvalOperator.ExpressionEvaluator field, Sour @Override public String name() { - return "ToBoolean"; + return "ToBooleanFromString"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromUnsignedLongEvaluator.java index 9d391b5730d05..d2cf4b41770ce 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromUnsignedLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanFromUnsignedLongEvaluator.java @@ -29,7 +29,7 @@ public ToBooleanFromUnsignedLongEvaluator(EvalOperator.ExpressionEvaluator field @Override public String name() { - return "ToBoolean"; + return "ToBooleanFromUnsignedLong"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeFromStringEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeFromStringEvaluator.java index 56e3f7c52fe39..98310bb390392 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeFromStringEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeFromStringEvaluator.java @@ -30,7 +30,7 @@ public ToDatetimeFromStringEvaluator(EvalOperator.ExpressionEvaluator field, Sou @Override public String name() { - return "ToDatetime"; + return "ToDatetimeFromString"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromBooleanEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromBooleanEvaluator.java index c3ab6f87db2a1..d2b16e4b722cb 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromBooleanEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromBooleanEvaluator.java @@ -29,7 +29,7 @@ public ToDoubleFromBooleanEvaluator(EvalOperator.ExpressionEvaluator field, Sour @Override public String name() { - return "ToDouble"; + return "ToDoubleFromBoolean"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromIntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromIntEvaluator.java index 57bb11132d1bc..53e8edac3c5b3 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromIntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromIntEvaluator.java @@ -29,7 +29,7 @@ public ToDoubleFromIntEvaluator(EvalOperator.ExpressionEvaluator field, Source s @Override public String name() { - return "ToDouble"; + return "ToDoubleFromInt"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromLongEvaluator.java index ee5defac00278..9be5f1f2456b1 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromLongEvaluator.java @@ -29,7 +29,7 @@ public ToDoubleFromLongEvaluator(EvalOperator.ExpressionEvaluator field, Source @Override public String name() { - return "ToDouble"; + return "ToDoubleFromLong"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromStringEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromStringEvaluator.java index 3c3dc8f7eb298..653034f0c3bc9 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromStringEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromStringEvaluator.java @@ -30,7 +30,7 @@ public ToDoubleFromStringEvaluator(EvalOperator.ExpressionEvaluator field, Sourc @Override public String name() { - return "ToDouble"; + return "ToDoubleFromString"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromUnsignedLongEvaluator.java index 3d0cb711e9dde..54cc374c758fb 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromUnsignedLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleFromUnsignedLongEvaluator.java @@ -29,7 +29,7 @@ public ToDoubleFromUnsignedLongEvaluator(EvalOperator.ExpressionEvaluator field, @Override public String name() { - return "ToDouble"; + return "ToDoubleFromUnsignedLong"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPFromStringEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPFromStringEvaluator.java index 6277d89a7f918..7befa47c6009c 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPFromStringEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPFromStringEvaluator.java @@ -31,7 +31,7 @@ public ToIPFromStringEvaluator(EvalOperator.ExpressionEvaluator field, Source so @Override public String name() { - return "ToIP"; + return "ToIPFromString"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromBooleanEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromBooleanEvaluator.java index 997ba90fc8043..49f79cd0bcd3e 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromBooleanEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromBooleanEvaluator.java @@ -29,7 +29,7 @@ public ToIntegerFromBooleanEvaluator(EvalOperator.ExpressionEvaluator field, Sou @Override public String name() { - return "ToInteger"; + return "ToIntegerFromBoolean"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromDoubleEvaluator.java index 2589c116dc106..e1b0db72ad7d9 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromDoubleEvaluator.java @@ -29,7 +29,7 @@ public ToIntegerFromDoubleEvaluator(EvalOperator.ExpressionEvaluator field, Sour @Override public String name() { - return "ToInteger"; + return "ToIntegerFromDouble"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromLongEvaluator.java index f79fd470ae76c..9a1394b9c02cf 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromLongEvaluator.java @@ -29,7 +29,7 @@ public ToIntegerFromLongEvaluator(EvalOperator.ExpressionEvaluator field, Source @Override public String name() { - return "ToInteger"; + return "ToIntegerFromLong"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromStringEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromStringEvaluator.java index feb0c565290b2..180e64f97e63b 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromStringEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromStringEvaluator.java @@ -30,7 +30,7 @@ public ToIntegerFromStringEvaluator(EvalOperator.ExpressionEvaluator field, Sour @Override public String name() { - return "ToInteger"; + return "ToIntegerFromString"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromUnsignedLongEvaluator.java index d5b02c0d87889..698db22c0ecc6 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromUnsignedLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromUnsignedLongEvaluator.java @@ -29,7 +29,7 @@ public ToIntegerFromUnsignedLongEvaluator(EvalOperator.ExpressionEvaluator field @Override public String name() { - return "ToInteger"; + return "ToIntegerFromUnsignedLong"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromBooleanEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromBooleanEvaluator.java index 87879b26c7d23..bf76fb0eb8a59 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromBooleanEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromBooleanEvaluator.java @@ -29,7 +29,7 @@ public ToLongFromBooleanEvaluator(EvalOperator.ExpressionEvaluator field, Source @Override public String name() { - return "ToLong"; + return "ToLongFromBoolean"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromDoubleEvaluator.java index 23900011be8ae..116be245e3191 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromDoubleEvaluator.java @@ -29,7 +29,7 @@ public ToLongFromDoubleEvaluator(EvalOperator.ExpressionEvaluator field, Source @Override public String name() { - return "ToLong"; + return "ToLongFromDouble"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromIntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromIntEvaluator.java index a043af09a47cf..02d043c641cb0 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromIntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromIntEvaluator.java @@ -29,7 +29,7 @@ public ToLongFromIntEvaluator(EvalOperator.ExpressionEvaluator field, Source sou @Override public String name() { - return "ToLong"; + return "ToLongFromInt"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromStringEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromStringEvaluator.java index 3629c0d0d046f..cc825664cc331 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromStringEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromStringEvaluator.java @@ -30,7 +30,7 @@ public ToLongFromStringEvaluator(EvalOperator.ExpressionEvaluator field, Source @Override public String name() { - return "ToLong"; + return "ToLongFromString"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromUnsignedLongEvaluator.java index 4f6e2363a676b..02bef2f9f9c2d 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromUnsignedLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongFromUnsignedLongEvaluator.java @@ -28,7 +28,7 @@ public ToLongFromUnsignedLongEvaluator(EvalOperator.ExpressionEvaluator field, S @Override public String name() { - return "ToLong"; + return "ToLongFromUnsignedLong"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromBooleanEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromBooleanEvaluator.java index 7f8244391ba90..c95d64ee0f8f5 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromBooleanEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromBooleanEvaluator.java @@ -32,7 +32,7 @@ public ToStringFromBooleanEvaluator(EvalOperator.ExpressionEvaluator field, Sour @Override public String name() { - return "ToString"; + return "ToStringFromBoolean"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDatetimeEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDatetimeEvaluator.java index a681fa818cf8f..d5f7454fc0d12 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDatetimeEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDatetimeEvaluator.java @@ -32,7 +32,7 @@ public ToStringFromDatetimeEvaluator(EvalOperator.ExpressionEvaluator field, Sou @Override public String name() { - return "ToString"; + return "ToStringFromDatetime"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDoubleEvaluator.java index 2eb67c1739d9c..deb897a4cf170 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDoubleEvaluator.java @@ -32,7 +32,7 @@ public ToStringFromDoubleEvaluator(EvalOperator.ExpressionEvaluator field, Sourc @Override public String name() { - return "ToString"; + return "ToStringFromDouble"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromIPEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromIPEvaluator.java index d774668f7c654..dbb54eb4dbb59 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromIPEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromIPEvaluator.java @@ -31,7 +31,7 @@ public ToStringFromIPEvaluator(EvalOperator.ExpressionEvaluator field, Source so @Override public String name() { - return "ToString"; + return "ToStringFromIP"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromIntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromIntEvaluator.java index b004b9974b18e..6c4969948dbf0 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromIntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromIntEvaluator.java @@ -32,7 +32,7 @@ public ToStringFromIntEvaluator(EvalOperator.ExpressionEvaluator field, Source s @Override public String name() { - return "ToString"; + return "ToStringFromInt"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromLongEvaluator.java index 4ea62a864f894..d8fd6b0ebd57d 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromLongEvaluator.java @@ -32,7 +32,7 @@ public ToStringFromLongEvaluator(EvalOperator.ExpressionEvaluator field, Source @Override public String name() { - return "ToString"; + return "ToStringFromLong"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromUnsignedLongEvaluator.java index 057ff28f0c32d..f1a78b375ce7e 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromUnsignedLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromUnsignedLongEvaluator.java @@ -32,7 +32,7 @@ public ToStringFromUnsignedLongEvaluator(EvalOperator.ExpressionEvaluator field, @Override public String name() { - return "ToString"; + return "ToStringFromUnsignedLong"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromVersionEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromVersionEvaluator.java index 832984e467fa3..7f3361587a38c 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromVersionEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromVersionEvaluator.java @@ -31,7 +31,7 @@ public ToStringFromVersionEvaluator(EvalOperator.ExpressionEvaluator field, Sour @Override public String name() { - return "ToString"; + return "ToStringFromVersion"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromBooleanEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromBooleanEvaluator.java index 379cf1b80f4d6..ec8b16568c380 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromBooleanEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromBooleanEvaluator.java @@ -29,7 +29,7 @@ public ToUnsignedLongFromBooleanEvaluator(EvalOperator.ExpressionEvaluator field @Override public String name() { - return "ToUnsignedLong"; + return "ToUnsignedLongFromBoolean"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromDoubleEvaluator.java index 80ec90c02d2d4..2ada365ce848e 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromDoubleEvaluator.java @@ -29,7 +29,7 @@ public ToUnsignedLongFromDoubleEvaluator(EvalOperator.ExpressionEvaluator field, @Override public String name() { - return "ToUnsignedLong"; + return "ToUnsignedLongFromDouble"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromIntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromIntEvaluator.java index c8e370296aa9f..9acad2f9481a6 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromIntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromIntEvaluator.java @@ -29,7 +29,7 @@ public ToUnsignedLongFromIntEvaluator(EvalOperator.ExpressionEvaluator field, So @Override public String name() { - return "ToUnsignedLong"; + return "ToUnsignedLongFromInt"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromLongEvaluator.java index af2b02fb3d73a..0cb7da2ed230f 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromLongEvaluator.java @@ -28,7 +28,7 @@ public ToUnsignedLongFromLongEvaluator(EvalOperator.ExpressionEvaluator field, S @Override public String name() { - return "ToUnsignedLong"; + return "ToUnsignedLongFromLong"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromStringEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromStringEvaluator.java index 201c1fbc4157e..3297fcffbe73b 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromStringEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromStringEvaluator.java @@ -30,7 +30,7 @@ public ToUnsignedLongFromStringEvaluator(EvalOperator.ExpressionEvaluator field, @Override public String name() { - return "ToUnsignedLong"; + return "ToUnsignedLongFromString"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionFromStringEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionFromStringEvaluator.java index 8630c6fa98d17..d8136b4ec4127 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionFromStringEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionFromStringEvaluator.java @@ -31,7 +31,7 @@ public ToVersionFromStringEvaluator(EvalOperator.ExpressionEvaluator field, Sour @Override public String name() { - return "ToVersion"; + return "ToVersionFromString"; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10DoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10DoubleEvaluator.java index 5303aaed79e0a..15eaa7388944f 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10DoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10DoubleEvaluator.java @@ -4,6 +4,7 @@ // 2.0. package org.elasticsearch.xpack.esql.expression.function.scalar.math; +import java.lang.ArithmeticException; import java.lang.Override; import java.lang.String; import org.elasticsearch.compute.data.Block; @@ -11,15 +12,20 @@ import org.elasticsearch.compute.data.DoubleVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.expression.function.Warnings; +import org.elasticsearch.xpack.ql.tree.Source; /** * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Log10}. * This class is generated. Do not edit it. */ public final class Log10DoubleEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + private final EvalOperator.ExpressionEvaluator val; - public Log10DoubleEvaluator(EvalOperator.ExpressionEvaluator val) { + public Log10DoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator val) { + this.warnings = new Warnings(source); this.val = val; } @@ -34,7 +40,7 @@ public Block eval(Page page) { if (valVector == null) { return eval(page.getPositionCount(), valBlock); } - return eval(page.getPositionCount(), valVector).asBlock(); + return eval(page.getPositionCount(), valVector); } public DoubleBlock eval(int positionCount, DoubleBlock valBlock) { @@ -44,15 +50,25 @@ public DoubleBlock eval(int positionCount, DoubleBlock valBlock) { result.appendNull(); continue position; } - result.appendDouble(Log10.process(valBlock.getDouble(valBlock.getFirstValueIndex(p)))); + try { + result.appendDouble(Log10.process(valBlock.getDouble(valBlock.getFirstValueIndex(p)))); + } catch (ArithmeticException e) { + warnings.registerException(e); + result.appendNull(); + } } return result.build(); } - public DoubleVector eval(int positionCount, DoubleVector valVector) { - DoubleVector.Builder result = DoubleVector.newVectorBuilder(positionCount); + public DoubleBlock eval(int positionCount, DoubleVector valVector) { + DoubleBlock.Builder result = DoubleBlock.newBlockBuilder(positionCount); position: for (int p = 0; p < positionCount; p++) { - result.appendDouble(Log10.process(valVector.getDouble(p))); + try { + result.appendDouble(Log10.process(valVector.getDouble(p))); + } catch (ArithmeticException e) { + warnings.registerException(e); + result.appendNull(); + } } return result.build(); } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10IntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10IntEvaluator.java index 86dcad75c1c9d..c3c95930836cd 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10IntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10IntEvaluator.java @@ -4,24 +4,29 @@ // 2.0. package org.elasticsearch.xpack.esql.expression.function.scalar.math; +import java.lang.ArithmeticException; import java.lang.Override; import java.lang.String; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.DoubleBlock; -import org.elasticsearch.compute.data.DoubleVector; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.expression.function.Warnings; +import org.elasticsearch.xpack.ql.tree.Source; /** * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Log10}. * This class is generated. Do not edit it. */ public final class Log10IntEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + private final EvalOperator.ExpressionEvaluator val; - public Log10IntEvaluator(EvalOperator.ExpressionEvaluator val) { + public Log10IntEvaluator(Source source, EvalOperator.ExpressionEvaluator val) { + this.warnings = new Warnings(source); this.val = val; } @@ -36,7 +41,7 @@ public Block eval(Page page) { if (valVector == null) { return eval(page.getPositionCount(), valBlock); } - return eval(page.getPositionCount(), valVector).asBlock(); + return eval(page.getPositionCount(), valVector); } public DoubleBlock eval(int positionCount, IntBlock valBlock) { @@ -46,15 +51,25 @@ public DoubleBlock eval(int positionCount, IntBlock valBlock) { result.appendNull(); continue position; } - result.appendDouble(Log10.process(valBlock.getInt(valBlock.getFirstValueIndex(p)))); + try { + result.appendDouble(Log10.process(valBlock.getInt(valBlock.getFirstValueIndex(p)))); + } catch (ArithmeticException e) { + warnings.registerException(e); + result.appendNull(); + } } return result.build(); } - public DoubleVector eval(int positionCount, IntVector valVector) { - DoubleVector.Builder result = DoubleVector.newVectorBuilder(positionCount); + public DoubleBlock eval(int positionCount, IntVector valVector) { + DoubleBlock.Builder result = DoubleBlock.newBlockBuilder(positionCount); position: for (int p = 0; p < positionCount; p++) { - result.appendDouble(Log10.process(valVector.getInt(p))); + try { + result.appendDouble(Log10.process(valVector.getInt(p))); + } catch (ArithmeticException e) { + warnings.registerException(e); + result.appendNull(); + } } return result.build(); } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10LongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10LongEvaluator.java index fe74f2d34b776..e8c1e590b9084 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10LongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10LongEvaluator.java @@ -4,24 +4,29 @@ // 2.0. package org.elasticsearch.xpack.esql.expression.function.scalar.math; +import java.lang.ArithmeticException; import java.lang.Override; import java.lang.String; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.DoubleBlock; -import org.elasticsearch.compute.data.DoubleVector; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.LongVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.expression.function.Warnings; +import org.elasticsearch.xpack.ql.tree.Source; /** * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Log10}. * This class is generated. Do not edit it. */ public final class Log10LongEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + private final EvalOperator.ExpressionEvaluator val; - public Log10LongEvaluator(EvalOperator.ExpressionEvaluator val) { + public Log10LongEvaluator(Source source, EvalOperator.ExpressionEvaluator val) { + this.warnings = new Warnings(source); this.val = val; } @@ -36,7 +41,7 @@ public Block eval(Page page) { if (valVector == null) { return eval(page.getPositionCount(), valBlock); } - return eval(page.getPositionCount(), valVector).asBlock(); + return eval(page.getPositionCount(), valVector); } public DoubleBlock eval(int positionCount, LongBlock valBlock) { @@ -46,15 +51,25 @@ public DoubleBlock eval(int positionCount, LongBlock valBlock) { result.appendNull(); continue position; } - result.appendDouble(Log10.process(valBlock.getLong(valBlock.getFirstValueIndex(p)))); + try { + result.appendDouble(Log10.process(valBlock.getLong(valBlock.getFirstValueIndex(p)))); + } catch (ArithmeticException e) { + warnings.registerException(e); + result.appendNull(); + } } return result.build(); } - public DoubleVector eval(int positionCount, LongVector valVector) { - DoubleVector.Builder result = DoubleVector.newVectorBuilder(positionCount); + public DoubleBlock eval(int positionCount, LongVector valVector) { + DoubleBlock.Builder result = DoubleBlock.newBlockBuilder(positionCount); position: for (int p = 0; p < positionCount; p++) { - result.appendDouble(Log10.process(valVector.getLong(p))); + try { + result.appendDouble(Log10.process(valVector.getLong(p))); + } catch (ArithmeticException e) { + warnings.registerException(e); + result.appendNull(); + } } return result.build(); } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10UnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10UnsignedLongEvaluator.java index 9e1fc7ed44f57..faa75ee571eb9 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10UnsignedLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10UnsignedLongEvaluator.java @@ -4,24 +4,29 @@ // 2.0. package org.elasticsearch.xpack.esql.expression.function.scalar.math; +import java.lang.ArithmeticException; import java.lang.Override; import java.lang.String; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.DoubleBlock; -import org.elasticsearch.compute.data.DoubleVector; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.LongVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.expression.function.Warnings; +import org.elasticsearch.xpack.ql.tree.Source; /** * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Log10}. * This class is generated. Do not edit it. */ public final class Log10UnsignedLongEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + private final EvalOperator.ExpressionEvaluator val; - public Log10UnsignedLongEvaluator(EvalOperator.ExpressionEvaluator val) { + public Log10UnsignedLongEvaluator(Source source, EvalOperator.ExpressionEvaluator val) { + this.warnings = new Warnings(source); this.val = val; } @@ -36,7 +41,7 @@ public Block eval(Page page) { if (valVector == null) { return eval(page.getPositionCount(), valBlock); } - return eval(page.getPositionCount(), valVector).asBlock(); + return eval(page.getPositionCount(), valVector); } public DoubleBlock eval(int positionCount, LongBlock valBlock) { @@ -46,15 +51,25 @@ public DoubleBlock eval(int positionCount, LongBlock valBlock) { result.appendNull(); continue position; } - result.appendDouble(Log10.processUnsignedLong(valBlock.getLong(valBlock.getFirstValueIndex(p)))); + try { + result.appendDouble(Log10.processUnsignedLong(valBlock.getLong(valBlock.getFirstValueIndex(p)))); + } catch (ArithmeticException e) { + warnings.registerException(e); + result.appendNull(); + } } return result.build(); } - public DoubleVector eval(int positionCount, LongVector valVector) { - DoubleVector.Builder result = DoubleVector.newVectorBuilder(positionCount); + public DoubleBlock eval(int positionCount, LongVector valVector) { + DoubleBlock.Builder result = DoubleBlock.newBlockBuilder(positionCount); position: for (int p = 0; p < positionCount; p++) { - result.appendDouble(Log10.processUnsignedLong(valVector.getLong(p))); + try { + result.appendDouble(Log10.processUnsignedLong(valVector.getLong(p))); + } catch (ArithmeticException e) { + warnings.registerException(e); + result.appendNull(); + } } return result.build(); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java index 8aff85d088d6f..4d783b9a1012c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java @@ -102,12 +102,7 @@ public EnrichLookupService(ClusterService clusterService, SearchService searchSe this.searchService = searchService; this.transportService = transportService; this.executor = transportService.getThreadPool().executor(EsqlPlugin.ESQL_THREAD_POOL_NAME); - transportService.registerRequestHandler( - LOOKUP_ACTION_NAME, - EsqlPlugin.ESQL_THREAD_POOL_NAME, - LookupRequest::new, - new TransportHandler() - ); + transportService.registerRequestHandler(LOOKUP_ACTION_NAME, this.executor, LookupRequest::new, new TransportHandler()); } public void lookupAsync( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java index 189fac27a0c73..0360397e419b0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java @@ -117,7 +117,7 @@ protected final void registerException(Exception exception) { @Override public final String toString() { - return name() + "[field=" + fieldEvaluator + "]"; + return name() + "Evaluator[field=" + fieldEvaluator + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java index dad7da17086ba..428c1f32b1fc7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java @@ -12,6 +12,7 @@ import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; +import org.elasticsearch.xpack.esql.expression.function.Named; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; @@ -29,6 +30,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.IP; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.LONG; +import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.VERSION; import static org.elasticsearch.xpack.ql.util.DateUtils.UTC_DATE_TIME_FORMATTER; @@ -52,14 +54,16 @@ public class ToString extends AbstractConvertFunction implements EvaluatorMapper ToStringFromLongEvaluator::new, INTEGER, ToStringFromIntEvaluator::new, + TEXT, + (fieldEval, source) -> fieldEval, VERSION, ToStringFromVersionEvaluator::new, UNSIGNED_LONG, ToStringFromUnsignedLongEvaluator::new ); - public ToString(Source source, Expression field) { - super(source, field); + public ToString(Source source, @Named("v") Expression v) { + super(source, v); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java index c22a773542a94..0051bee45eead 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java @@ -10,6 +10,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.ConvertEvaluator; import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.expression.function.Named; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; @@ -21,15 +22,20 @@ import java.util.function.BiFunction; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; +import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; import static org.elasticsearch.xpack.ql.type.DataTypes.VERSION; public class ToVersion extends AbstractConvertFunction { private static final Map> EVALUATORS = - Map.of(VERSION, (fieldEval, source) -> fieldEval, KEYWORD, ToVersionFromStringEvaluator::new); + Map.ofEntries( + Map.entry(VERSION, (fieldEval, source) -> fieldEval), + Map.entry(KEYWORD, ToVersionFromStringEvaluator::new), + Map.entry(TEXT, ToVersionFromStringEvaluator::new) + ); - public ToVersion(Source source, Expression field) { - super(source, field); + public ToVersion(Source source, @Named("v") Expression v) { + super(source, v); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10.java index 4541fb16580bf..616172f165d0b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10.java @@ -41,38 +41,50 @@ public Supplier toEvaluator( var eval = field.get(); if (fieldType == DataTypes.DOUBLE) { - return () -> new Log10DoubleEvaluator(eval); + return () -> new Log10DoubleEvaluator(source(), eval); } if (fieldType == DataTypes.INTEGER) { - return () -> new Log10IntEvaluator(eval); + return () -> new Log10IntEvaluator(source(), eval); } if (fieldType == DataTypes.LONG) { - return () -> new Log10LongEvaluator(eval); + return () -> new Log10LongEvaluator(source(), eval); } if (fieldType == DataTypes.UNSIGNED_LONG) { - return () -> new Log10UnsignedLongEvaluator(eval); + return () -> new Log10UnsignedLongEvaluator(source(), eval); } throw EsqlIllegalArgumentException.illegalDataType(fieldType); } - @Evaluator(extraName = "Double") + @Evaluator(extraName = "Double", warnExceptions = ArithmeticException.class) static double process(double val) { + if (val <= 0d) { + throw new ArithmeticException("Log of non-positive number"); + } return Math.log10(val); } - @Evaluator(extraName = "Long") + @Evaluator(extraName = "Long", warnExceptions = ArithmeticException.class) static double process(long val) { + if (val <= 0L) { + throw new ArithmeticException("Log of non-positive number"); + } return Math.log10(val); } - @Evaluator(extraName = "UnsignedLong") + @Evaluator(extraName = "UnsignedLong", warnExceptions = ArithmeticException.class) static double processUnsignedLong(long val) { + if (val == NumericUtils.ZERO_AS_UNSIGNED_LONG) { + throw new ArithmeticException("Log of non-positive number"); + } return Math.log10(NumericUtils.unsignedLongToDouble(val)); } - @Evaluator(extraName = "Int") + @Evaluator(extraName = "Int", warnExceptions = ArithmeticException.class) static double process(int val) { + if (val <= 0) { + throw new ArithmeticException("Log of non-positive number"); + } return Math.log10(val); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index 4f6f523c53bbb..2e14c33c701cb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -98,6 +98,7 @@ protected static List> rules() { var operators = new Batch<>( "Operator Optimization", new CombineProjections(), + new CombineEvals(), new PruneEmptyPlans(), new PropagateEmptyRelation(), new ConvertStringToByteRef(), @@ -312,6 +313,26 @@ private static Expression trimAliases(Expression e) { } } + /** + * Combine multiple Evals into one in order to reduce the number of nodes in a plan. + * TODO: eliminate unnecessary fields inside the eval as well + */ + static class CombineEvals extends OptimizerRules.OptimizerRule { + + CombineEvals() { + super(TransformDirection.UP); + } + + @Override + protected LogicalPlan rule(Eval eval) { + LogicalPlan plan = eval; + if (eval.child() instanceof Eval subEval) { + plan = new Eval(eval.source(), subEval.child(), CollectionUtils.combine(subEval.fields(), eval.fields())); + } + return plan; + } + } + // // Replace any reference attribute with its source, if it does not affect the result. // This avoids ulterior look-ups between attributes and its source across nodes. @@ -584,7 +605,6 @@ protected static class PushDownEval extends OptimizerRules.OptimizerRule { protected LogicalPlan rule(Eval eval) { LogicalPlan child = eval.child(); - // TODO: combine with CombineEval from https://github.com/elastic/elasticsearch-internal/pull/511 when merged if (child instanceof OrderBy orderBy) { return orderBy.replaceChild(eval.replaceChild(orderBy.child())); } else if (child instanceof Project) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java index b057dd8023031..3f05fa90ac8ab 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java @@ -20,8 +20,8 @@ import org.elasticsearch.xpack.esql.planner.PhysicalVerifier; import org.elasticsearch.xpack.ql.common.Failure; import org.elasticsearch.xpack.ql.expression.Alias; -import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.AttributeMap; +import org.elasticsearch.xpack.ql.expression.AttributeSet; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Expressions; import org.elasticsearch.xpack.ql.expression.Literal; @@ -36,7 +36,6 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.LinkedHashSet; import java.util.List; import static java.lang.Boolean.FALSE; @@ -91,7 +90,7 @@ static class ProjectAwayColumns extends Rule { public PhysicalPlan apply(PhysicalPlan plan) { var projectAll = new Holder<>(TRUE); var keepCollecting = new Holder<>(TRUE); - var attributes = new LinkedHashSet(); + var attributes = new AttributeSet(); var aliases = new AttributeMap(); return plan.transformDown(UnaryExec.class, p -> { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java index f5000b4394e21..cb20d3a549915 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java @@ -23,6 +23,7 @@ public class Eval extends UnaryPlan { private final List fields; + private List lazyOutput; public Eval(Source source, LogicalPlan child, List fields) { super(source, child); @@ -35,7 +36,11 @@ public List fields() { @Override public List output() { - return mergeOutputAttributes(fields, child().output()); + if (lazyOutput == null) { + lazyOutput = mergeOutputAttributes(fields, child().output()); + } + + return lazyOutput; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index f725e4fe60175..27634bf0d4eaa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -103,14 +103,9 @@ public ComputeService( this.searchService = searchService; this.transportService = transportService; this.bigArrays = bigArrays.withCircuitBreaking(); - transportService.registerRequestHandler( - DATA_ACTION_NAME, - ESQL_THREAD_POOL_NAME, - DataNodeRequest::new, - new DataNodeRequestHandler() - ); this.esqlExecutor = threadPool.executor(ESQL_THREAD_POOL_NAME); - this.driverRunner = new DriverTaskRunner(transportService, ESQL_THREAD_POOL_NAME); + transportService.registerRequestHandler(DATA_ACTION_NAME, this.esqlExecutor, DataNodeRequest::new, new DataNodeRequestHandler()); + this.driverRunner = new DriverTaskRunner(transportService, this.esqlExecutor); this.exchangeService = exchangeService; this.enrichLookupService = enrichLookupService; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index fa14501edc50d..8a4a53f7b986d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -9,6 +9,8 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.settings.Settings; @@ -16,9 +18,11 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.Driver; +import org.elasticsearch.compute.operator.DriverRunner; import org.elasticsearch.compute.operator.exchange.ExchangeSinkHandler; import org.elasticsearch.compute.operator.exchange.ExchangeSourceHandler; import org.elasticsearch.core.Releasables; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; @@ -89,7 +93,6 @@ import java.util.TreeMap; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.compute.operator.DriverRunner.runToCompletion; import static org.elasticsearch.test.ListMatcher.matchesList; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.xpack.esql.CsvTestUtils.ExpectedResults; @@ -357,7 +360,6 @@ private ActualResults executePlan() throws Exception { List drivers = new ArrayList<>(); List collectedPages = Collections.synchronizedList(new ArrayList<>()); - Map> responseHeaders; // replace fragment inside the coordinator plan try { @@ -375,11 +377,21 @@ private ActualResults executePlan() throws Exception { drivers.addAll(dataNodeExecutionPlan.createDrivers(sessionId)); Randomness.shuffle(drivers); } - responseHeaders = runToCompletion(threadPool, between(1, 10_000), drivers); + // Execute the driver + DriverRunner runner = new DriverRunner() { + @Override + protected void start(Driver driver, ActionListener driverListener) { + Driver.start(threadPool.executor(ESQL_THREAD_POOL_NAME), driver, between(1, 1000), driverListener); + } + }; + PlainActionFuture future = new PlainActionFuture<>(); + runner.runToCompletion(drivers, future); + future.actionGet(TimeValue.timeValueSeconds(30)); + var responseHeaders = threadPool.getThreadContext().getResponseHeaders(); + return new ActualResults(columnNames, columnTypes, dataTypes, collectedPages, responseHeaders); } finally { Releasables.close(() -> Releasables.close(drivers), exchangeSource::decRef); } - return new ActualResults(columnNames, columnTypes, dataTypes, collectedPages, responseHeaders); } private Throwable reworkException(Throwable th) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java index 7f0bdbab6add3..f0b3a89a4444b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java @@ -43,11 +43,17 @@ public class EsqlInfoTransportActionTests extends ESTestCase { private ThreadPool threadPool; + private TransportService transportService; private Client client; @Before public void init() { threadPool = new TestThreadPool(getTestName()); + + // TODO: temporary, remove in #97879 + transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); + client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); } @@ -58,12 +64,12 @@ public void shutdown() { } public void testAvailable() { - EsqlInfoTransportAction featureSet = new EsqlInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + EsqlInfoTransportAction featureSet = new EsqlInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testEnabled() { - EsqlInfoTransportAction featureSet = new EsqlInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + EsqlInfoTransportAction featureSet = new EsqlInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); } @@ -98,7 +104,7 @@ public void testUsageStats() throws Exception { when(clusterService.localNode()).thenReturn(mockNode); var usageAction = new EsqlUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, threadPool, mock(ActionFilters.class), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 0d0f57b02b9fe..06cbddac631c7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -1167,7 +1167,7 @@ public void testRegexOnInt() { public void testUnsupportedTypesWithToString() { // DATE_PERIOD and TIME_DURATION types have been added, but not really patched through the engine; i.e. supported. - final String supportedTypes = "boolean, datetime, double, integer, ip, keyword, long, unsigned_long or version"; + final String supportedTypes = "boolean, datetime, double, integer, ip, keyword, long, text, unsigned_long or version"; verifyUnsupported( "row period = 1 year | eval to_string(period)", "line 1:28: argument of [to_string(period)] must be [" + supportedTypes + "], found value [period] type [date_period]" diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index a1a5e1aa60d19..83900b5d1bd77 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.type.EsField; +import org.elasticsearch.xpack.ql.util.StringUtils; import org.elasticsearch.xpack.versionfield.Version; import org.junit.After; import org.junit.AfterClass; @@ -164,6 +165,9 @@ protected final void assertResolveTypeValid(Expression expression, DataType expe public final void testEvaluate() { assumeTrue("All test data types must be representable in order to build fields", testCase.allTypesAreRepresentable()); + logger.info( + "Test Values: " + testCase.getData().stream().map(TestCaseSupplier.TypedData::toString).collect(Collectors.joining(",")) + ); Expression expression = buildFieldExpression(testCase); if (testCase.getExpectedTypeError() != null) { assertTrue("expected unresolved", expression.typeResolved().unresolved()); @@ -462,7 +466,8 @@ private static String typeErrorMessage(List> validPerPosition, Lis private static final Map, String> NAMED_EXPECTED_TYPES = Map.ofEntries( Map.entry(Set.of(DataTypes.DOUBLE, DataTypes.NULL), "double"), Map.entry(Set.of(DataTypes.INTEGER, DataTypes.NULL), "integer"), - Map.entry(Set.of(DataTypes.LONG, DataTypes.INTEGER, DataTypes.UNSIGNED_LONG, DataTypes.DOUBLE, DataTypes.NULL), "numeric") + Map.entry(Set.of(DataTypes.LONG, DataTypes.INTEGER, DataTypes.UNSIGNED_LONG, DataTypes.DOUBLE, DataTypes.NULL), "numeric"), + Map.entry(Set.of(DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.VERSION, DataTypes.NULL), "keyword, text or version") ); private static String expectedType(Set validTypes) { @@ -581,7 +586,7 @@ private static FunctionDefinition definition() { } private static String functionName() { - return getTestClass().getSimpleName().replace("Tests", "").toLowerCase(Locale.ROOT); + return StringUtils.camelCaseToUnderscore(getTestClass().getSimpleName().replace("Tests", "")).toLowerCase(Locale.ROOT); } /** diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index f564cadff3477..3b6174bac5bf4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -7,6 +7,9 @@ package org.elasticsearch.xpack.esql.expression.function; +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.expression.Expression; @@ -15,9 +18,11 @@ import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.util.NumericUtils; +import org.elasticsearch.xpack.versionfield.Version; import org.hamcrest.Matcher; import java.math.BigInteger; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -40,7 +45,7 @@ public record TestCaseSupplier(String name, List types, Supplier { - public static final BigInteger MAX_UNSIGNED_LONG = BigInteger.valueOf(1 << 64).subtract(BigInteger.ONE); + public static final BigInteger MAX_UNSIGNED_LONG = NumericUtils.UNSIGNED_LONG_MAX; /** * Build a test case without types. * @@ -84,7 +89,13 @@ public String toString() { * Generate positive test cases for unary functions that operate on an {@code numeric} * fields by casting them to {@link DataTypes#DOUBLE}s. */ - public static List forUnaryCastingToDouble(String name, String argName, DoubleUnaryOperator expected) { + public static List forUnaryCastingToDouble( + String name, + String argName, + DoubleUnaryOperator expected, + Double min, + Double max + ) { String read = "Attribute[channel=0]"; String eval = name + "[" + argName + "="; List suppliers = new ArrayList<>(); @@ -93,33 +104,29 @@ public static List forUnaryCastingToDouble(String name, String eval + castToDoubleEvaluator(read, DataTypes.INTEGER) + "]", DataTypes.DOUBLE, i -> expected.applyAsDouble(i), - Integer.MIN_VALUE, - Integer.MAX_VALUE + min.intValue(), + max.intValue(), + List.of() ); forUnaryLong( suppliers, eval + castToDoubleEvaluator(read, DataTypes.LONG) + "]", DataTypes.DOUBLE, l -> expected.applyAsDouble(l), - Long.MIN_VALUE, - Long.MAX_VALUE + min.longValue(), + max.longValue(), + List.of() ); forUnaryUnsignedLong( suppliers, eval + castToDoubleEvaluator(read, DataTypes.UNSIGNED_LONG) + "]", DataTypes.DOUBLE, ul -> expected.applyAsDouble(ul.doubleValue()), - BigInteger.ZERO, - MAX_UNSIGNED_LONG - ); - forUnaryDouble( - suppliers, - eval + read + "]", - DataTypes.DOUBLE, - i -> expected.applyAsDouble(i), - Double.NEGATIVE_INFINITY, - Double.POSITIVE_INFINITY + BigInteger.valueOf((int) Math.ceil(min)), + BigInteger.valueOf((int) Math.floor(max)), + List.of() ); + forUnaryDouble(suppliers, eval + read + "]", DataTypes.DOUBLE, i -> expected.applyAsDouble(i), min, max, List.of()); return suppliers; } @@ -184,7 +191,8 @@ public static void forUnaryInt( DataType expectedType, IntFunction expectedValue, int lowerBound, - int upperBound + int upperBound, + List warnings ) { unaryNumeric( suppliers, @@ -192,7 +200,8 @@ public static void forUnaryInt( DataTypes.INTEGER, intCases(lowerBound, upperBound), expectedType, - n -> expectedValue.apply(n.intValue()) + n -> expectedValue.apply(n.intValue()), + warnings ); } @@ -205,7 +214,8 @@ public static void forUnaryLong( DataType expectedType, LongFunction expectedValue, long lowerBound, - long upperBound + long upperBound, + List warnings ) { unaryNumeric( suppliers, @@ -213,7 +223,8 @@ public static void forUnaryLong( DataTypes.LONG, longCases(lowerBound, upperBound), expectedType, - n -> expectedValue.apply(n.longValue()) + n -> expectedValue.apply(n.longValue()), + warnings ); } @@ -226,7 +237,8 @@ public static void forUnaryUnsignedLong( DataType expectedType, Function expectedValue, BigInteger lowerBound, - BigInteger upperBound + BigInteger upperBound, + List warnings ) { unaryNumeric( suppliers, @@ -234,7 +246,8 @@ public static void forUnaryUnsignedLong( DataTypes.UNSIGNED_LONG, ulongCases(lowerBound, upperBound), expectedType, - n -> expectedValue.apply((BigInteger) n) + n -> expectedValue.apply((BigInteger) n), + warnings ); } @@ -247,7 +260,8 @@ public static void forUnaryDouble( DataType expectedType, DoubleFunction expectedValue, double lowerBound, - double upperBound + double upperBound, + List warnings ) { unaryNumeric( suppliers, @@ -255,7 +269,115 @@ public static void forUnaryDouble( DataTypes.DOUBLE, doubleCases(lowerBound, upperBound), expectedType, - n -> expectedValue.apply(n.doubleValue()) + n -> expectedValue.apply(n.doubleValue()), + warnings + ); + } + + /** + * Generate positive test cases for a unary function operating on an {@link DataTypes#BOOLEAN}. + */ + public static void forUnaryBoolean( + List suppliers, + String expectedEvaluatorToString, + DataType expectedType, + Function expectedValue, + List warnings + ) { + unary( + suppliers, + expectedEvaluatorToString, + DataTypes.BOOLEAN, + booleanCases(), + expectedType, + v -> expectedValue.apply((Boolean) v), + warnings + ); + } + + /** + * Generate positive test cases for a unary function operating on an {@link DataTypes#DATETIME}. + */ + public static void forUnaryDatetime( + List suppliers, + String expectedEvaluatorToString, + DataType expectedType, + Function expectedValue, + List warnings + ) { + unaryNumeric( + suppliers, + expectedEvaluatorToString, + DataTypes.DATETIME, + dateCases(), + expectedType, + n -> expectedValue.apply(Instant.ofEpochMilli(n.longValue())), + warnings + ); + } + + /** + * Generate positive test cases for a unary function operating on an {@link DataTypes#IP}. + */ + public static void forUnaryIp( + List suppliers, + String expectedEvaluatorToString, + DataType expectedType, + Function expectedValue, + List warnings + ) { + unary( + suppliers, + expectedEvaluatorToString, + DataTypes.IP, + ipCases(), + expectedType, + v -> expectedValue.apply((BytesRef) v), + warnings + ); + } + + /** + * Generate positive test cases for a unary function operating on an {@link DataTypes#KEYWORD} and {@link DataTypes#TEXT}. + */ + public static void forUnaryStrings( + List suppliers, + String expectedEvaluatorToString, + DataType expectedType, + Function expectedValue, + List warnings + ) { + for (DataType type : EsqlDataTypes.types().stream().filter(EsqlDataTypes::isString).toList()) { + unary( + suppliers, + expectedEvaluatorToString, + type, + stringCases(type.typeName()), + expectedType, + v -> expectedValue.apply((BytesRef) v), + warnings + ); + } + } + + /** + * Generate positive test cases for a unary function operating on an {@link DataTypes#VERSION}. + */ + public static void forUnaryVersion( + List suppliers, + String expectedEvaluatorToString, + DataType expectedType, + Function expectedValue, + List warnings + ) { + unary( + suppliers, + expectedEvaluatorToString, + DataTypes.VERSION, + versionCases(""), + expectedType, + v -> expectedValue.apply(new Version((BytesRef) v)), + warnings ); } @@ -265,18 +387,48 @@ private static void unaryNumeric( DataType inputType, List>> valueSuppliers, DataType expectedOutputType, - Function expected + Function expected, + List warnings + ) { + unary( + suppliers, + expectedEvaluatorToString, + inputType, + valueSuppliers, + expectedOutputType, + v -> expected.apply((Number) v), + warnings + ); + } + + private static void unary( + List suppliers, + String expectedEvaluatorToString, + DataType inputType, + List>> valueSuppliers, + DataType expectedOutputType, + Function expected, + List warnings ) { for (Map.Entry> supplier : valueSuppliers) { suppliers.add(new TestCaseSupplier(supplier.getKey(), List.of(inputType), () -> { - Number value = (Number) supplier.getValue().get(); + Object value = supplier.getValue().get(); TypedData typed = new TypedData( // TODO there has to be a better way to handle unsigned long value instanceof BigInteger b ? NumericUtils.asLongUnsigned(b) : value, inputType, "value" ); - return new TestCase(List.of(typed), expectedEvaluatorToString, expectedOutputType, equalTo(expected.apply(value))); + TestCase testCase = new TestCase( + List.of(typed), + expectedEvaluatorToString, + expectedOutputType, + equalTo(expected.apply(value)) + ); + for (String warning : warnings) { + testCase = testCase.withWarning(warning); + } + return testCase; })); } } @@ -340,7 +492,7 @@ private static List>> ulongCases(BigInteger m // small values, less than Long.MAX_VALUE BigInteger lower1 = min.max(BigInteger.ONE); - BigInteger upper1 = max.min(BigInteger.valueOf(Integer.MAX_VALUE)); + BigInteger upper1 = max.min(BigInteger.valueOf(Long.MAX_VALUE)); if (lower1.compareTo(upper1) < 0) { cases.add( Map.entry( @@ -358,12 +510,12 @@ private static List>> ulongCases(BigInteger m if (lower2.compareTo(upper2) < 0) { cases.add( Map.entry( - "", + "", () -> BigInteger.valueOf(ESTestCase.randomLongBetween(lower2.longValue(), upper2.longValue())) ) ); } else if (lower2.compareTo(upper2) == 0) { - cases.add(Map.entry("", () -> lower2)); + cases.add(Map.entry("", () -> lower2)); } return cases; } @@ -416,6 +568,63 @@ private static List>> doubleCases(double min, return cases; } + private static List>> booleanCases() { + return List.of(Map.entry("", () -> true), Map.entry("", () -> false)); + } + + private static List>> dateCases() { + return List.of( + Map.entry("<1970-01-01T00:00:00Z>", () -> 0L), + Map.entry( + "", + () -> ESTestCase.randomLongBetween(0, 10 * (long) 10e11) // 1970-01-01T00:00:00Z - 2286-11-20T17:46:40Z + ), + Map.entry( + "", + // 2286-11-20T17:46:40Z - +292278994-08-17T07:12:55.807Z + () -> ESTestCase.randomLongBetween(10 * (long) 10e11, Long.MAX_VALUE) + ) + ); + } + + private static List>> ipCases() { + return List.of( + Map.entry("<127.0.0.1 ip>", () -> new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1")))), + Map.entry("", () -> new BytesRef(InetAddressPoint.encode(ESTestCase.randomIp(true)))), + Map.entry("", () -> new BytesRef(InetAddressPoint.encode(ESTestCase.randomIp(false)))) + ); + } + + private static List>> stringCases(String type) { + List>> result = new ArrayList<>(); + result.add(Map.entry("", () -> new BytesRef(""))); + result.add(Map.entry("", () -> new BytesRef(ESTestCase.randomAlphaOfLengthBetween(1, 30)))); + result.add(Map.entry("", () -> new BytesRef(ESTestCase.randomAlphaOfLengthBetween(300, 3000)))); + result.add(Map.entry("", () -> new BytesRef(ESTestCase.randomRealisticUnicodeOfLengthBetween(1, 30)))); + result.add( + Map.entry("", () -> new BytesRef(ESTestCase.randomRealisticUnicodeOfLengthBetween(300, 3000))) + ); + return result; + } + + /** + * Supplier test case data for {@link Version} fields. + */ + public static List>> versionCases(String prefix) { + return List.of( + Map.entry("<" + prefix + "version major>", () -> new Version(Integer.toString(ESTestCase.between(0, 100))).toBytesRef()), + Map.entry( + "<" + prefix + "version major.minor>", + () -> new Version(ESTestCase.between(0, 100) + "." + ESTestCase.between(0, 100)).toBytesRef() + ), + Map.entry( + "<" + prefix + "version major.minor.patch>", + () -> new Version(ESTestCase.between(0, 100) + "." + ESTestCase.between(0, 100) + "." + ESTestCase.between(0, 100)) + .toBytesRef() + ) + ); + } + private static final Map>>> RANDOM_VALUE_SUPPLIERS = Map.ofEntries( Map.entry( DataTypes.DOUBLE, @@ -586,5 +795,10 @@ public record TypedData(Object data, DataType type, String name) { public TypedData(Object data, String name) { this(data, EsqlDataTypes.fromJava(data), name); } + + @Override + public String toString() { + return type.toString() + "(" + (data == null ? "null" : data.toString()) + ")"; + } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java new file mode 100644 index 0000000000000..1b611c78f8e8a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +public class ToStringTests extends AbstractFunctionTestCase { + public ToStringTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + // TODO multivalue fields + String read = "Attribute[channel=0]"; + List suppliers = new ArrayList<>(); + TestCaseSupplier.forUnaryInt( + suppliers, + "ToStringFromIntEvaluator[field=" + read + "]", + DataTypes.KEYWORD, + i -> new BytesRef(Integer.toString(i)), + Integer.MIN_VALUE, + Integer.MAX_VALUE, + List.of() + ); + TestCaseSupplier.forUnaryLong( + suppliers, + "ToStringFromLongEvaluator[field=" + read + "]", + DataTypes.KEYWORD, + l -> new BytesRef(Long.toString(l)), + Long.MIN_VALUE, + Long.MAX_VALUE, + List.of() + ); + // TestCaseSupplier.forUnaryUnsignedLong( + // suppliers, + // "ToStringFromUnsignedLongEvaluator[field=" + read + "]", + // DataTypes.KEYWORD, + // ul -> new BytesRef(ul.toString()), + // BigInteger.ZERO, + // MAX_UNSIGNED_LONG, + // List.of() + // ); + TestCaseSupplier.forUnaryDouble( + suppliers, + "ToStringFromDoubleEvaluator[field=" + read + "]", + DataTypes.KEYWORD, + d -> new BytesRef(Double.toString(d)), + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY, + List.of() + ); + TestCaseSupplier.forUnaryBoolean( + suppliers, + "ToStringFromBooleanEvaluator[field=" + read + "]", + DataTypes.KEYWORD, + b -> new BytesRef(b.toString()), + List.of() + ); + TestCaseSupplier.forUnaryDatetime( + suppliers, + "ToStringFromDatetimeEvaluator[field=" + read + "]", + DataTypes.KEYWORD, + i -> new BytesRef(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(i.toEpochMilli())), + List.of() + ); + TestCaseSupplier.forUnaryIp( + suppliers, + "ToStringFromIPEvaluator[field=" + read + "]", + DataTypes.KEYWORD, + ip -> new BytesRef(DocValueFormat.IP.format(ip)), + List.of() + ); + TestCaseSupplier.forUnaryStrings(suppliers, read, DataTypes.KEYWORD, bytesRef -> bytesRef, List.of()); + TestCaseSupplier.forUnaryVersion( + suppliers, + "ToStringFromVersionEvaluator[field=" + read + "]", + DataTypes.KEYWORD, + v -> new BytesRef(v.toString()), + List.of() + ); + return parameterSuppliersFromTypedData( + // errorsForCasesWithoutExamples( + anyNullIsNull(true, suppliers) + // ) + ); + } + + @Override + protected Expression build(Source source, List args) { + return new ToString(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java new file mode 100644 index 0000000000000..c84f244b08df8 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; +import org.elasticsearch.xpack.versionfield.Version; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; + +public class ToVersionTests extends AbstractFunctionTestCase { + public ToVersionTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + // TODO multivalue fields + String read = "Attribute[channel=0]"; + String stringEvaluator = "ToVersionFromStringEvaluator[field=" + read + "]"; + List suppliers = new ArrayList<>(); + // Converting and IP to an IP doesn't change anything. Everything should succeed. + TestCaseSupplier.forUnaryVersion(suppliers, read, DataTypes.VERSION, v -> v.toBytesRef(), List.of()); + // None of the random strings ever look like versions so they should all become "invalid" versions + // TODO should this return null with warnings? they aren't version shaped at all. + TestCaseSupplier.forUnaryStrings( + suppliers, + stringEvaluator, + DataTypes.VERSION, + bytesRef -> new Version(bytesRef.utf8ToString()).toBytesRef(), + List.of() + ); + // But strings that are shaped like versions do parse to valid versions + for (DataType inputType : EsqlDataTypes.types().stream().filter(EsqlDataTypes::isString).toList()) { + for (Map.Entry> versionGen : TestCaseSupplier.versionCases(inputType.typeName() + " ")) { + suppliers.add(new TestCaseSupplier(versionGen.getKey(), List.of(inputType), () -> { + BytesRef encodedVersion = (BytesRef) versionGen.getValue().get(); + TestCaseSupplier.TypedData typed = new TestCaseSupplier.TypedData( + new BytesRef(new Version(encodedVersion).toString()), + inputType, + "value" + ); + return new TestCaseSupplier.TestCase(List.of(typed), stringEvaluator, DataTypes.VERSION, equalTo(encodedVersion)); + })); + } + } + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); + } + + @Override + protected Expression build(Source source, List args) { + return new ToVersion(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java index 0958fb67ee805..bf1eeacc9ed06 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java @@ -25,7 +25,13 @@ public AcosTests(@Name("TestCase") Supplier testCaseS @ParametersFactory public static Iterable parameters() { - List suppliers = TestCaseSupplier.forUnaryCastingToDouble("AcosEvaluator", "val", Math::acos); + List suppliers = TestCaseSupplier.forUnaryCastingToDouble( + "AcosEvaluator", + "val", + Math::acos, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY + ); return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java index a3f1fcee7a092..119683d2d94e1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java @@ -25,7 +25,13 @@ public AsinTests(@Name("TestCase") Supplier testCaseS @ParametersFactory public static Iterable parameters() { - List suppliers = TestCaseSupplier.forUnaryCastingToDouble("AsinEvaluator", "val", Math::asin); + List suppliers = TestCaseSupplier.forUnaryCastingToDouble( + "AsinEvaluator", + "val", + Math::asin, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY + ); return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanTests.java index f529b07071720..eb448b9402eea 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanTests.java @@ -25,7 +25,13 @@ public AtanTests(@Name("TestCase") Supplier testCaseS @ParametersFactory public static Iterable parameters() { - List suppliers = TestCaseSupplier.forUnaryCastingToDouble("AtanEvaluator", "val", Math::atan); + List suppliers = TestCaseSupplier.forUnaryCastingToDouble( + "AtanEvaluator", + "val", + Math::atan, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY + ); return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosTests.java index 70d3f7704a0dc..a024fbdb1d76f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosTests.java @@ -25,7 +25,13 @@ public CosTests(@Name("TestCase") Supplier testCaseSu @ParametersFactory public static Iterable parameters() { - List suppliers = TestCaseSupplier.forUnaryCastingToDouble("CosEvaluator", "val", Math::cos); + List suppliers = TestCaseSupplier.forUnaryCastingToDouble( + "CosEvaluator", + "val", + Math::cos, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY + ); return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshTests.java index 4e6add3ffe0b4..6d5393469422b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshTests.java @@ -25,7 +25,13 @@ public CoshTests(@Name("TestCase") Supplier testCaseS @ParametersFactory public static Iterable parameters() { - List suppliers = TestCaseSupplier.forUnaryCastingToDouble("CoshEvaluator", "val", Math::cosh); + List suppliers = TestCaseSupplier.forUnaryCastingToDouble( + "CoshEvaluator", + "val", + Math::cosh, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY + ); return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorTests.java index 9485aed23320a..ee63353f2637d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorTests.java @@ -33,15 +33,16 @@ public FloorTests(@Name("TestCase") Supplier testCase public static Iterable parameters() { String read = "Attribute[channel=0]"; List suppliers = new ArrayList<>(); - TestCaseSupplier.forUnaryInt(suppliers, read, DataTypes.INTEGER, i -> i, Integer.MIN_VALUE, Integer.MAX_VALUE); - TestCaseSupplier.forUnaryLong(suppliers, read, DataTypes.LONG, l -> l, Long.MIN_VALUE, Long.MAX_VALUE); + TestCaseSupplier.forUnaryInt(suppliers, read, DataTypes.INTEGER, i -> i, Integer.MIN_VALUE, Integer.MAX_VALUE, List.of()); + TestCaseSupplier.forUnaryLong(suppliers, read, DataTypes.LONG, l -> l, Long.MIN_VALUE, Long.MAX_VALUE, List.of()); TestCaseSupplier.forUnaryUnsignedLong( suppliers, read, DataTypes.UNSIGNED_LONG, ul -> NumericUtils.asLongUnsigned(ul), BigInteger.ZERO, - MAX_UNSIGNED_LONG + MAX_UNSIGNED_LONG, + List.of() ); TestCaseSupplier.forUnaryDouble( suppliers, @@ -49,7 +50,8 @@ public static Iterable parameters() { DataTypes.DOUBLE, Math::floor, Double.NEGATIVE_INFINITY, - Double.POSITIVE_INFINITY + Double.POSITIVE_INFINITY, + List.of() ); return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(false, suppliers))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java index fff62198f6e48..267c780ab4619 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; +import org.elasticsearch.xpack.ql.util.NumericUtils; import java.math.BigInteger; import java.util.ArrayList; @@ -32,39 +33,98 @@ public Log10Tests(@Name("TestCase") Supplier testCase public static Iterable parameters() { String read = "Attribute[channel=0]"; List suppliers = new ArrayList<>(); + // Cases in valid range TestCaseSupplier.forUnaryInt( suppliers, "Log10IntEvaluator[val=" + read + "]", DataTypes.DOUBLE, Math::log10, - Integer.MIN_VALUE, - Integer.MAX_VALUE + 1, + Integer.MAX_VALUE, + List.of() ); TestCaseSupplier.forUnaryLong( suppliers, "Log10LongEvaluator[val=" + read + "]", DataTypes.DOUBLE, Math::log10, + 1L, + Long.MAX_VALUE, + List.of() + ); + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + "Log10UnsignedLongEvaluator[val=" + read + "]", + DataTypes.DOUBLE, + ul -> Math.log10(ul == null ? null : NumericUtils.unsignedLongToDouble(NumericUtils.asLongUnsigned(ul))), + BigInteger.ONE, + MAX_UNSIGNED_LONG, + List.of() + ); + TestCaseSupplier.forUnaryDouble( + suppliers, + "Log10DoubleEvaluator[val=" + read + "]", + DataTypes.DOUBLE, + Math::log10, + Double.MIN_VALUE, + Double.POSITIVE_INFINITY, + List.of() + ); + + // Add in null cases here; the out of range cases won't set the right warnings on a null input. + suppliers = anyNullIsNull(true, suppliers); + + // Cases with invalid inputs + TestCaseSupplier.forUnaryInt( + suppliers, + "Log10IntEvaluator[val=" + read + "]", + DataTypes.DOUBLE, + k -> null, + Integer.MIN_VALUE, + 0, + List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "java.lang.ArithmeticException: Log of non-positive number" + ) + ); + TestCaseSupplier.forUnaryLong( + suppliers, + "Log10LongEvaluator[val=" + read + "]", + DataTypes.DOUBLE, + k -> null, Long.MIN_VALUE, - Long.MAX_VALUE + 0L, + List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "java.lang.ArithmeticException: Log of non-positive number" + ) ); TestCaseSupplier.forUnaryUnsignedLong( suppliers, "Log10UnsignedLongEvaluator[val=" + read + "]", DataTypes.DOUBLE, - ul -> Math.log10(ul.doubleValue()), + k -> null, BigInteger.ZERO, - MAX_UNSIGNED_LONG + BigInteger.ZERO, + List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "java.lang.ArithmeticException: Log of non-positive number" + ) ); TestCaseSupplier.forUnaryDouble( suppliers, "Log10DoubleEvaluator[val=" + read + "]", DataTypes.DOUBLE, - Math::log10, + k -> null, Double.NEGATIVE_INFINITY, - Double.POSITIVE_INFINITY + 0d, + List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "java.lang.ArithmeticException: Log of non-positive number" + ) ); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); + + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers)); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java index e04d1fb9b3c80..e9d0599842f98 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java @@ -25,7 +25,13 @@ public SinTests(@Name("TestCase") Supplier testCaseSu @ParametersFactory public static Iterable parameters() { - List suppliers = TestCaseSupplier.forUnaryCastingToDouble("SinEvaluator", "val", Math::sin); + List suppliers = TestCaseSupplier.forUnaryCastingToDouble( + "SinEvaluator", + "val", + Math::sin, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY + ); return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhTests.java index 4e51a424aa21f..0f53ea76e8462 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhTests.java @@ -25,7 +25,13 @@ public SinhTests(@Name("TestCase") Supplier testCaseS @ParametersFactory public static Iterable parameters() { - List suppliers = TestCaseSupplier.forUnaryCastingToDouble("SinhEvaluator", "val", Math::sinh); + List suppliers = TestCaseSupplier.forUnaryCastingToDouble( + "SinhEvaluator", + "val", + Math::sinh, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY + ); return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java index 348360d9a395f..4c13100b7c746 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; +import org.elasticsearch.xpack.ql.util.NumericUtils; import java.math.BigInteger; import java.util.ArrayList; @@ -38,7 +39,8 @@ public static Iterable parameters() { DataTypes.DOUBLE, Math::sqrt, Integer.MIN_VALUE, - Integer.MAX_VALUE + Integer.MAX_VALUE, + List.of() ); TestCaseSupplier.forUnaryLong( suppliers, @@ -46,15 +48,17 @@ public static Iterable parameters() { DataTypes.DOUBLE, Math::sqrt, Long.MIN_VALUE, - Long.MAX_VALUE + Long.MAX_VALUE, + List.of() ); TestCaseSupplier.forUnaryUnsignedLong( suppliers, "SqrtUnsignedLongEvaluator[val=" + read + "]", DataTypes.DOUBLE, - ul -> Math.sqrt(ul.doubleValue()), + ul -> Math.sqrt(ul == null ? null : NumericUtils.unsignedLongToDouble(NumericUtils.asLongUnsigned(ul))), BigInteger.ZERO, - MAX_UNSIGNED_LONG + MAX_UNSIGNED_LONG, + List.of() ); TestCaseSupplier.forUnaryDouble( suppliers, @@ -62,7 +66,8 @@ public static Iterable parameters() { DataTypes.DOUBLE, Math::sqrt, Double.NEGATIVE_INFINITY, - Double.POSITIVE_INFINITY + Double.POSITIVE_INFINITY, + List.of() ); return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanTests.java index afb7e4c695228..7e2bd466ffc10 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanTests.java @@ -25,7 +25,13 @@ public TanTests(@Name("TestCase") Supplier testCaseSu @ParametersFactory public static Iterable parameters() { - List suppliers = TestCaseSupplier.forUnaryCastingToDouble("TanEvaluator", "val", Math::tan); + List suppliers = TestCaseSupplier.forUnaryCastingToDouble( + "TanEvaluator", + "val", + Math::tan, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY + ); return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhTests.java index a86dfc0b6efee..1f6d5bd6e3b9f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhTests.java @@ -25,7 +25,13 @@ public TanhTests(@Name("TestCase") Supplier testCaseS @ParametersFactory public static Iterable parameters() { - List suppliers = TestCaseSupplier.forUnaryCastingToDouble("TanhEvaluator", "val", Math::tanh); + List suppliers = TestCaseSupplier.forUnaryCastingToDouble( + "TanhEvaluator", + "val", + Math::tanh, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY + ); return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java index 0f48da5d80f72..ad74e1896a951 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java @@ -119,7 +119,7 @@ protected static void bytesRefs( ) { cases.add( new TestCaseSupplier( - name + "(\"\")", + name + "(empty string)", () -> new TestCaseSupplier.TestCase( List.of(new TestCaseSupplier.TypedData(List.of(new BytesRef("")), DataTypes.KEYWORD, "field")), evaluatorName + "[field=Attribute[channel=0]]", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index b6d818038a72c..bd1615ef9e5f2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -255,6 +255,28 @@ public void testCombineProjectionWithPruning() { var from = as(agg.child(), EsRelation.class); } + /** + * Expects + * EsqlProject[[x{r}#3, y{r}#6]] + * \_Eval[[emp_no{f}#9 + 2[INTEGER] AS x, salary{f}#14 + 3[INTEGER] AS y]] + * \_Limit[10000[INTEGER]] + * \_EsRelation[test][_meta_field{f}#15, emp_no{f}#9, first_name{f}#10, g..] + */ + public void testCombineEvals() { + var plan = plan(""" + from test + | eval x = emp_no + 2 + | eval y = salary + 3 + | keep x, y + """); + + var project = as(plan, Project.class); + var eval = as(project.child(), Eval.class); + assertThat(Expressions.names(eval.fields()), contains("x", "y")); + var limit = as(eval.child(), Limit.class); + var source = as(limit.child(), EsRelation.class); + } + public void testCombineLimits() { var limitValues = new int[] { randomIntBetween(10, 99), randomIntBetween(100, 1000) }; var firstLimit = randomBoolean() ? 0 : 1; @@ -777,9 +799,7 @@ public void testCombineOrderByThroughEvalWithTwoDefs() { var topN = as(plan, TopN.class); assertThat(orderNames(topN), contains("z", "emp_no")); var eval = as(topN.child(), Eval.class); - assertThat(Expressions.names(eval.fields()), contains("z")); - eval = as(eval.child(), Eval.class); - assertThat(Expressions.names(eval.fields()), contains("x", "y")); + assertThat(Expressions.names(eval.fields()), contains("x", "y", "z")); as(eval.child(), EsRelation.class); } @@ -1599,8 +1619,6 @@ public void testNoPruningWhenDealingJustWithEvals() { """); var eval = as(plan, Eval.class); - eval = as(eval.child(), Eval.class); - eval = as(eval.child(), Eval.class); var limit = as(eval.child(), Limit.class); var agg = as(limit.child(), Aggregate.class); } @@ -1630,9 +1648,7 @@ public void testPruningDuplicateEvals() { var project = as(plan, Project.class); var eval = as(project.child(), Eval.class); - assertThat(Expressions.names(eval.fields()), contains("y")); - eval = as(eval.child(), Eval.class); - assertThat(Expressions.names(eval.fields()), contains("x")); + assertThat(Expressions.names(eval.fields()), contains("x", "y")); var limit = as(eval.child(), Limit.class); var source = as(limit.child(), EsRelation.class); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 83d71d2cc3ed1..87693bffc4433 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -366,14 +366,13 @@ public void testExtractorForField() { /** * Expected - * - * EvalExec[[emp_no{f}#538 + 1[INTEGER] AS emp_no]] - * \_EvalExec[[emp_no{f}#538 + 1[INTEGER] AS e]] - * \_LimitExec[10000[INTEGER]] - * \_ExchangeExec[GATHER,SINGLE_DISTRIBUTION] - * \_ProjectExec[[_meta_field{f}#537, emp_no{f}#538, first_name{f}#539, languages{f}#540, last_name{f}#541, salary{f}#542]] - * \_FieldExtractExec[_meta_field{f}#537, emp_no{f}#538, first_name{f}#53..] - * \_EsQueryExec[test], query[][_doc{f}#543], limit[10000] + * EvalExec[[emp_no{f}#7 + 1[INTEGER] AS e, emp_no{f}#7 + 1[INTEGER] AS emp_no]] + * \_LimitExec[10000[INTEGER]] + * \_ExchangeExec[[],false] + * \_ProjectExec[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, job{f}#14, job.raw{f}#15, languages{f}#10, last + * _name{f}#11, salary{f}#12]] + * \_FieldExtractExec[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] + * \_EsQueryExec[test], query[][_doc{f}#16], limit[10000], sort[] estimatedRowSize[324] */ public void testExtractorMultiEvalWithDifferentNames() { var plan = physicalPlan(""" @@ -383,8 +382,8 @@ public void testExtractorMultiEvalWithDifferentNames() { """); var optimized = optimizedPlan(plan); + var eval = as(optimized, EvalExec.class); - eval = as(eval.child(), EvalExec.class); var topLimit = as(eval.child(), LimitExec.class); var exchange = asRemoteExchange(topLimit.child()); var project = as(exchange.child(), ProjectExec.class); @@ -397,13 +396,13 @@ public void testExtractorMultiEvalWithDifferentNames() { /** * Expected - * EvalExec[[emp_no{r}#120 + 1[INTEGER] AS emp_no]] - * \_EvalExec[[emp_no{f}#125 + 1[INTEGER] AS emp_no]] - * \_LimitExec[10000[INTEGER]] - * \_ExchangeExec[GATHER,SINGLE_DISTRIBUTION] - * \_ProjectExec[[_meta_field{f}#124, emp_no{f}#125, first_name{f}#126, languages{f}#127, last_name{f}#128, salary{f}#129]] - * \_FieldExtractExec[_meta_field{f}#124, emp_no{f}#125, first_name{f}#12..] - * \_EsQueryExec[test], query[][_doc{f}#130], limit[10000] + * EvalExec[[emp_no{f}#7 + 1[INTEGER] AS emp_no, emp_no{r}#3 + 1[INTEGER] AS emp_no]] + * \_LimitExec[10000[INTEGER]] + * \_ExchangeExec[[],false] + * \_ProjectExec[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, job{f}#14, job.raw{f}#15, languages{f}#10, last + * _name{f}#11, salary{f}#12]] + * \_FieldExtractExec[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] + * \_EsQueryExec[test], query[][_doc{f}#16], limit[10000], sort[] estimatedRowSize[324] */ public void testExtractorMultiEvalWithSameName() { var plan = physicalPlan(""" @@ -413,8 +412,8 @@ public void testExtractorMultiEvalWithSameName() { """); var optimized = optimizedPlan(plan); + var eval = as(optimized, EvalExec.class); - eval = as(eval.child(), EvalExec.class); var topLimit = as(eval.child(), LimitExec.class); var exchange = asRemoteExchange(topLimit.child()); var project = as(exchange.child(), ProjectExec.class); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java index df163250e6e7a..24c7612cd1326 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java @@ -256,7 +256,8 @@ public Operator get(DriverContext driverContext) { () -> BlockHash.build( List.of(new HashAggregationOperator.GroupSpec(groupByChannel, groupElementType)), bigArrays, - pageSize + pageSize, + false ), columnName, driverContext diff --git a/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java b/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java index f1bad49abda68..90034c9620923 100644 --- a/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java +++ b/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; @@ -33,8 +34,9 @@ public void init() throws Exception { } public void testAvailable() throws Exception { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); GraphInfoTransportAction featureSet = new GraphInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), Settings.EMPTY, licenseState @@ -44,7 +46,7 @@ public void testAvailable() throws Exception { assertThat(featureSet.available(), is(available)); var usageAction = new GraphUsageTransportAction( - mock(TransportService.class), + transportService, null, mock(ThreadPool.class), mock(ActionFilters.class), @@ -73,8 +75,10 @@ public void testEnabled() throws Exception { } else { settings.put("xpack.graph.enabled", enabled); } + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); GraphInfoTransportAction featureSet = new GraphInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), settings.build(), licenseState @@ -82,7 +86,7 @@ public void testEnabled() throws Exception { assertThat(featureSet.enabled(), is(enabled)); GraphUsageTransportAction usageAction = new GraphUsageTransportAction( - mock(TransportService.class), + transportService, null, mock(ThreadPool.class), mock(ActionFilters.class), diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java index ae031a050d3d5..0ea85bab36a4e 100644 --- a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java +++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider; import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderDocument; @@ -57,10 +58,10 @@ public void setupMocks() { index = mock(SamlServiceProviderIndex.class); idp = mock(SamlIdentityProvider.class); when(idp.getAllowedNameIdFormats()).thenReturn(Set.of(TRANSIENT)); - now = Instant.ofEpochMilli(System.currentTimeMillis() + randomLongBetween(-500_000, 500_000)); final Clock clock = Clock.fixed(now, randomZone()); - action = new TransportPutSamlServiceProviderAction(mock(TransportService.class), mock(ActionFilters.class), index, idp, clock); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + action = new TransportPutSamlServiceProviderAction(transportService, mock(ActionFilters.class), index, idp, clock); } public void testRegisterNewServiceProvider() throws Exception { diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportActionTests.java index 107421ce12a31..d81faf6a398d7 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportActionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; @@ -52,18 +53,14 @@ public void init() throws Exception { } public void testAvailable() { - IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class) - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), equalTo(true)); } public void testName() { - IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class) - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.name(), equalTo("ilm")); } @@ -94,13 +91,9 @@ public void testUsageStats() throws Exception { ClusterState clusterState = buildClusterState(policies, indexPolicies); Mockito.when(clusterService.state()).thenReturn(clusterState); - var usageAction = new IndexLifecycleUsageTransportAction( - mock(TransportService.class), - null, - mock(ThreadPool.class), - mock(ActionFilters.class), - null - ); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + var usageAction = new IndexLifecycleUsageTransportAction(transportService, null, threadPool, mock(ActionFilters.class), null); PlainActionFuture future = new PlainActionFuture<>(); usageAction.masterOperation(null, null, clusterState, future); IndexLifecycleFeatureSetUsage ilmUsage = (IndexLifecycleFeatureSetUsage) future.get().getUsage(); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java index feb66c12c2613..3e0f19246dfe3 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ilm.action.DeleteLifecycleAction; @@ -20,10 +21,12 @@ public class TransportDeleteLifecycleActionTests extends ESTestCase { public void testReservedStateHandler() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportDeleteLifecycleAction putAction = new TransportDeleteLifecycleAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java index ef1d65f2e4950..8467e46630876 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -46,10 +47,12 @@ public void testIsNoop() { } public void testReservedStateHandler() throws Exception { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportPutLifecycleAction putAction = new TransportPutLifecycleAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), mock(NamedXContentRegistry.class), diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java index cdc50f28fc306..2c19a79ac5af0 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ilm.StopILMRequest; @@ -34,10 +35,12 @@ public class TransportStopILMActionTests extends ESTestCase { public void testStopILMClusterStatePriorityIsImmediate() { ClusterService clusterService = mock(ClusterService.class); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportStopILMAction transportStopILMAction = new TransportStopILMAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineActionTests.java b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineActionTests.java index d72d079d4ae19..56b7965be3686 100644 --- a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineActionTests.java +++ b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineActionTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.transport.TransportService; @@ -27,8 +28,9 @@ public class TransportDeletePipelineActionTests extends ESTestCase { public void testDeletePipelineWithMissingIndex() throws Exception { try (Client client = getFailureClient(new IndexNotFoundException("missing .logstash"))) { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final TransportDeletePipelineAction action = new TransportDeletePipelineAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), client ); diff --git a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineActionTests.java b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineActionTests.java index d8b4b167bcc46..d8a4d048f1fe4 100644 --- a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineActionTests.java +++ b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineActionTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.MockLogAppender; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.transport.TransportService; @@ -100,11 +101,8 @@ public void onFailure(Exception e) { try (Client client = getMockClient(multiGetResponse)) { Loggers.addAppender(logger, mockLogAppender); - TransportGetPipelineAction action = new TransportGetPipelineAction( - mock(TransportService.class), - mock(ActionFilters.class), - client - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + TransportGetPipelineAction action = new TransportGetPipelineAction(transportService, mock(ActionFilters.class), client); action.doExecute(null, request, testActionListener); } finally { Loggers.removeAppender(logger, mockLogAppender); @@ -152,8 +150,9 @@ public void onFailure(Exception e) { } }; + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); try (Client client = getMockClient(searchResponse)) { - new TransportGetPipelineAction(mock(TransportService.class), mock(ActionFilters.class), client).doExecute( + new TransportGetPipelineAction(transportService, mock(ActionFilters.class), client).doExecute( null, request, testActionListener @@ -165,8 +164,9 @@ public void onFailure(Exception e) { public void testMissingIndexHandling() throws Exception { try (Client failureClient = getFailureClient(new IndexNotFoundException("foo"))) { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final TransportGetPipelineAction action = new TransportGetPipelineAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), failureClient ); diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 1c5e8d05ccf38..4e04f85383def 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -203,7 +203,7 @@ public UnsignedLongFieldMapper build(MapperBuilderContext context) { metric.getValue(), indexMode ); - return new UnsignedLongFieldMapper(name, fieldType, multiFieldsBuilder.build(this, context), copyTo.build(), this); + return new UnsignedLongFieldMapper(name, fieldType, multiFieldsBuilder.build(this, context), copyTo, this); } } diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index f0d86da6efe8d..9e88c516576c2 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -86,13 +86,14 @@ public class VersionStringFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "version"; public static class Defaults { - public static final FieldType FIELD_TYPE = new FieldType(); + public static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setTokenized(false); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - FIELD_TYPE.freeze(); + FieldType ft = new FieldType(); + ft.setTokenized(false); + ft.setOmitNorms(true); + ft.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } } @@ -116,7 +117,7 @@ public VersionStringFieldMapper build(MapperBuilderContext context) { fieldtype, buildFieldType(context, fieldtype), multiFieldsBuilder.build(this, context), - copyTo.build() + copyTo ); } @@ -355,7 +356,7 @@ private VersionStringFieldMapper( CopyTo copyTo ) { super(simpleName, mappedFieldType, multiFields, copyTo); - this.fieldType = fieldType; + this.fieldType = freezeAndDeduplicateFieldType(fieldType); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index 2a9b133cb11dd..7a60dff726909 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.Releasable; @@ -289,7 +290,7 @@ public void triggerDeleteJobsInStateDeletingWithoutDeletionTask(ActionListener> chainTaskExecutor = new TypedChainTaskExecutor<>( - threadPool.executor(ThreadPool.Names.SAME), + EsExecutors.DIRECT_EXECUTOR_SERVICE, unused -> true, unused -> true ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java index 43307b05f63f7..17db1cbc5a234 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.Randomness; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Tuple; import org.elasticsearch.license.License; import org.elasticsearch.license.LicenseUtils; @@ -23,7 +24,6 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction; @@ -171,7 +171,7 @@ private void getModelAndInfer( ) { ActionListener getModelListener = ActionListener.wrap(model -> { TypedChainTaskExecutor typedChainTaskExecutor = new TypedChainTaskExecutor<>( - client.threadPool().executor(ThreadPool.Names.SAME), + EsExecutors.DIRECT_EXECUTOR_SERVICE, // run through all tasks r -> true, // Always fail immediately and return an error diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java index 418d277977c16..2ff9bb2906a5c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.ingest.IngestStats; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -145,10 +146,12 @@ private MachineLearningUsageTransportAction newUsageAction( boolean isDataFrameAnalyticsEnabled, boolean isNlpEnabled ) { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); return new MachineLearningUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), TestEnvironment.newEnvironment(settings), @@ -162,8 +165,9 @@ private MachineLearningUsageTransportAction newUsageAction( } public void testAvailable() throws Exception { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); MachineLearningInfoTransportAction featureSet = new MachineLearningInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), commonSettings, licenseState @@ -191,9 +195,11 @@ public void testEnabled() throws Exception { enabled = randomBoolean(); settings.put("xpack.ml.enabled", enabled); } + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); boolean expected = enabled; MachineLearningInfoTransportAction featureSet = new MachineLearningInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), settings.build(), licenseState diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java index c55b78c07939a..67da449af850f 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.MlTasks; @@ -337,10 +338,12 @@ public static void addTask( } private TransportCloseJobAction createAction() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); return new TransportCloseJobAction( - mock(TransportService.class), + transportService, client, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), clusterService, mock(AnomalyDetectionAuditor.class), diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java index f85a13fcb3935..26409b1ada849 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java @@ -38,6 +38,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; public class TransportDeleteExpiredDataActionTests extends ESTestCase { @@ -59,6 +60,10 @@ public void remove(float requestsPerSec, ActionListener listener, Boole public void setup() { threadPool = new TestThreadPool("TransportDeleteExpiredDataActionTests thread pool"); TransportService transportService = mock(TransportService.class); + + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + Client client = mock(Client.class); ClusterService clusterService = mock(ClusterService.class); auditor = mock(AnomalyDetectionAuditor.class); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java index 1c0243fcab098..c8096a7b1f384 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java @@ -81,8 +81,12 @@ public void testOperation() { } private TransportFinalizeJobExecutionAction createAction(ClusterService clusterService) { + // TODO: temporary, remove in #97879 + TransportService transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); + return new TransportFinalizeJobExecutionAction( - mock(TransportService.class), + transportService, clusterService, threadPool, mock(ActionFilters.class), diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java index 79a376b90040f..4d242db394d10 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.TransportVersionUtils; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; @@ -44,18 +45,14 @@ public class MonitoringInfoTransportActionTests extends ESTestCase { private final Exporters exporters = mock(Exporters.class); public void testAvailable() { - MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class) - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testMonitoringEnabledByDefault() { - MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class) - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); } @@ -94,10 +91,12 @@ public void testUsage() throws Exception { when(exporters.getEnabledExporters()).thenReturn(exporterList); when(monitoring.isMonitoringActive()).thenReturn(collectionEnabled); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var usageAction = new MonitoringUsageTransportAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, new MonitoringUsageServices(monitoring, exporters) diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java index 3676fc5d2e7a0..d2a694c04ae34 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; @@ -98,6 +99,10 @@ public void setUpMocks() { transportService = mock(TransportService.class); filters = mock(ActionFilters.class); + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + when(transportService.getTaskManager()).thenReturn(taskManager); when(taskManager.register(anyString(), eq(MonitoringBulkAction.NAME), any(TaskAwareRequest.class))).thenReturn(mock(Task.class)); when(filters.filters()).thenReturn(new ActionFilter[0]); diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java index 40a04115173fc..504ff930a8097 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java @@ -768,7 +768,6 @@ public void testToXContent() throws IOException { "max_index_version":%s } }, - "transport_versions": [], "nodes_versions": [] }, "cluster_settings": { diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java index dd78eb95e7f3d..9fe634a178179 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.BoostingQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; @@ -49,7 +50,6 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportRequestHandler; import org.elasticsearch.transport.TransportService; @@ -105,7 +105,7 @@ public TransportRollupSearchAction( transportService.registerRequestHandler( actionName, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, false, true, SearchRequest::new, diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java index ab7420f0b50ee..b0881eb350d5a 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; @@ -25,23 +26,21 @@ public class RollupInfoTransportActionTests extends ESTestCase { public void testAvailable() { - RollupInfoTransportAction featureSet = new RollupInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + RollupInfoTransportAction featureSet = new RollupInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testEnabledDefault() { - RollupInfoTransportAction featureSet = new RollupInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + RollupInfoTransportAction featureSet = new RollupInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); } public void testUsage() throws ExecutionException, InterruptedException, IOException { - var usageAction = new RollupUsageTransportAction( - mock(TransportService.class), - null, - mock(ThreadPool.class), - mock(ActionFilters.class), - null - ); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + var usageAction = new RollupUsageTransportAction(transportService, null, threadPool, mock(ActionFilters.class), null); PlainActionFuture future = new PlainActionFuture<>(); usageAction.masterOperation(null, null, null, future); XPackFeatureSet.Usage rollupUsage = future.get().getUsage(); diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java index ddba0395382f3..985bec3ba8c77 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Tuple; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -1108,7 +1109,7 @@ private static MockTransportService startTransport( try { service.registerRequestHandler( ClusterStateAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ClusterStateRequest::new, (request, channel, task) -> { capturedHeaders.add( @@ -1121,7 +1122,7 @@ private static MockTransportService startTransport( ); service.registerRequestHandler( RemoteClusterNodesAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, RemoteClusterNodesAction.Request::new, (request, channel, task) -> { capturedHeaders.add( @@ -1132,7 +1133,7 @@ private static MockTransportService startTransport( ); service.registerRequestHandler( SearchShardsAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, SearchShardsRequest::new, (request, channel, task) -> { capturedHeaders.add( @@ -1141,31 +1142,36 @@ private static MockTransportService startTransport( channel.sendResponse(new SearchShardsResponse(List.of(), List.of(), Collections.emptyMap())); } ); - service.registerRequestHandler(SearchAction.NAME, ThreadPool.Names.SAME, SearchRequest::new, (request, channel, task) -> { - capturedHeaders.add( - new CapturedActionWithHeaders(task.getAction(), Map.copyOf(threadPool.getThreadContext().getHeaders())) - ); - channel.sendResponse( - new SearchResponse( - new InternalSearchResponse( - new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), - InternalAggregations.EMPTY, - null, - null, - false, + service.registerRequestHandler( + SearchAction.NAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SearchRequest::new, + (request, channel, task) -> { + capturedHeaders.add( + new CapturedActionWithHeaders(task.getAction(), Map.copyOf(threadPool.getThreadContext().getHeaders())) + ); + channel.sendResponse( + new SearchResponse( + new InternalSearchResponse( + new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), + InternalAggregations.EMPTY, + null, + null, + false, + null, + 1 + ), null, - 1 - ), - null, - 1, - 1, - 0, - 100, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ) - ); - }); + 1, + 1, + 0, + 100, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ) + ); + } + ); service.start(); service.acceptIncomingRequests(); success = true; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java index 4b750fb7d2ab6..88f233087e1dd 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ToXContent; @@ -78,24 +79,18 @@ public void init() throws Exception { } public void testAvailable() { - SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class), - settings - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction(transportService, mock(ActionFilters.class), settings); assertThat(featureSet.available(), is(true)); } public void testEnabled() { - SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class), - settings - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction(transportService, mock(ActionFilters.class), settings); assertThat(featureSet.enabled(), is(true)); Settings disabled = Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), false).build(); - featureSet = new SecurityInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class), disabled); + featureSet = new SecurityInfoTransportAction(transportService, mock(ActionFilters.class), disabled); assertThat(featureSet.enabled(), is(false)); } @@ -371,10 +366,12 @@ private void configureRoleMappingStoreUsage(boolean roleMappingStoreEnabled) { } private SecurityUsageTransportAction newUsageAction(Settings settings) { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); return new SecurityUsageTransportAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, settings, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyActionTests.java index e2d1fc3a24568..f94acab50b6b5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyResponse; @@ -42,12 +43,8 @@ public void setUp() throws Exception { super.setUp(); apiKeyService = mock(ApiKeyService.class); securityContext = mock(SecurityContext.class); - action = new TransportCreateCrossClusterApiKeyAction( - mock(TransportService.class), - mock(ActionFilters.class), - apiKeyService, - securityContext - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + action = new TransportCreateCrossClusterApiKeyAction(transportService, mock(ActionFilters.class), apiKeyService, securityContext); } public void testApiKeyWillBeCreatedWithEmptyUserRoleDescriptors() throws IOException { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java index 5d68dea7a146e..f9e219f168718 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java @@ -61,6 +61,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; public class TransportGrantApiKeyActionTests extends ESTestCase { @@ -69,6 +70,7 @@ public class TransportGrantApiKeyActionTests extends ESTestCase { private ApiKeyUserRoleDescriptorResolver resolver; private AuthenticationService authenticationService; private ThreadPool threadPool; + private TransportService transportService; private AuthorizationService authorizationService; @Before @@ -81,8 +83,12 @@ public void setupMocks() throws Exception { threadPool = new TestThreadPool("TP-" + getTestName()); final ThreadContext threadContext = threadPool.getThreadContext(); + // TODO: temporary, remove in #97879 + TransportService transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); + action = new TransportGrantApiKeyAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), threadContext, authenticationService, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyActionTests.java index 711c604206823..7ce920506d7d1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyActionTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.apikey.ApiKey; @@ -55,8 +56,9 @@ public void testExecute() throws IOException { () -> AuthenticationTestHelper.builder().build() ); when(securityContext.getAuthentication()).thenReturn(authentication); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final var action = new TransportUpdateCrossClusterApiKeyAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), apiKeyService, securityContext @@ -120,8 +122,9 @@ public void testExecute() throws IOException { public void testAuthenticationCheck() { final SecurityContext securityContext = mock(SecurityContext.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final var action = new TransportUpdateCrossClusterApiKeyAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), mock(ApiKeyService.class), securityContext diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java index ed01d0ca6fc40..dca8633d7c1eb 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java @@ -64,6 +64,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TransportProfileHasPrivilegesActionTests extends ESTestCase { @@ -80,6 +81,10 @@ public class TransportProfileHasPrivilegesActionTests extends ESTestCase { public void setup() { threadPool = new TestThreadPool(TransportProfileHasPrivilegesActionTests.class.getSimpleName()); transportService = mock(TransportService.class); + + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + actionFilters = mock(ActionFilters.class); authorizationService = mock(AuthorizationService.class); nativePrivilegeStore = mock(NativePrivilegeStore.class); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java index df414207da8ee..92bef206a9d6a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRequest; @@ -50,10 +51,13 @@ public void setup() { fileRealm = mockRealm("file"); final Realms realms = mockRealms(List.of(nativeRealm, fileRealm)); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + action = new TransportClearRealmCacheAction( - mock(ThreadPool.class), + threadPool, mockClusterService(), - mock(TransportService.class), + transportService, mock(ActionFilters.class), realms, authenticationService diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenActionTests.java index ac7680ea2b574..5194ff032c5ba 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenActionTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.service.CreateServiceAccountTokenRequest; @@ -40,8 +41,10 @@ public class TransportCreateServiceAccountTokenActionTests extends ESTestCase { public void init() throws IOException { serviceAccountService = mock(ServiceAccountService.class); securityContext = mock(SecurityContext.class); + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); transportCreateServiceAccountTokenAction = new TransportCreateServiceAccountTokenAction( - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), serviceAccountService, securityContext diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenActionTests.java index 8b0530e48c560..2050d31638fe9 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.service.DeleteServiceAccountTokenRequest; import org.elasticsearch.xpack.core.security.action.service.DeleteServiceAccountTokenResponse; @@ -32,8 +33,9 @@ public class TransportDeleteServiceAccountTokenActionTests extends ESTestCase { @Before public void init() { serviceAccountService = mock(ServiceAccountService.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); transportDeleteServiceAccountTokenAction = new TransportDeleteServiceAccountTokenAction( - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), serviceAccountService ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java index 87ef1ea24559e..e8d178efe6b4a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.service.GetServiceAccountRequest; import org.elasticsearch.xpack.core.security.action.service.GetServiceAccountResponse; @@ -31,8 +32,9 @@ public class TransportGetServiceAccountActionTests extends ESTestCase { @Before public void init() { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); transportGetServiceAccountAction = new TransportGetServiceAccountAction( - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()) ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsActionTests.java index da6508ac79ef5..ec68ed7a2776f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsActionTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.service.GetServiceAccountCredentialsRequest; @@ -58,8 +59,10 @@ public void init() throws UnknownHostException { when(transport.boundAddress()).thenReturn(new BoundTransportAddress(new TransportAddress[] { transportAddress }, transportAddress)); final Settings settings = builder.build(); serviceAccountService = mock(ServiceAccountService.class); + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); transportGetServiceAccountCredentialsAction = new TransportGetServiceAccountCredentialsAction( - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), serviceAccountService ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java index bd63dd65824bd..dcdeace6b06b6 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java @@ -88,6 +88,7 @@ public class TransportCreateTokenActionTests extends ESTestCase { .build(); private ThreadPool threadPool; + private TransportService transportService; private Client client; private SecurityIndexManager securityIndex; private ClusterService clusterService; @@ -99,6 +100,11 @@ public class TransportCreateTokenActionTests extends ESTestCase { @Before public void setupClient() { threadPool = new TestThreadPool(getTestName()); + + // TODO: temporary, remove in #97879 + transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); + client = mock(Client.class); idxReqReference = new AtomicReference<>(); authenticationService = mock(AuthenticationService.class); @@ -221,7 +227,7 @@ public void testClientCredentialsCreatesWithoutRefreshToken() throws Exception { final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService, authenticationService, @@ -262,7 +268,7 @@ public void testPasswordGrantTypeCreatesWithRefreshToken() throws Exception { final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService, authenticationService, @@ -305,7 +311,7 @@ public void testKerberosGrantTypeCreatesWithRefreshToken() throws Exception { final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService, authenticationService, @@ -358,7 +364,7 @@ public void testKerberosGrantTypeWillFailOnBase64DecodeError() throws Exception final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService, authenticationService, @@ -397,7 +403,7 @@ public void testServiceAccountCannotCreateOAuthToken() throws Exception { final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService, authenticationService, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java index a24361078ff26..f07bb7dc365ae 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java @@ -56,6 +56,7 @@ public class TransportInvalidateTokenActionTests extends ESTestCase { .build(); private ThreadPool threadPool; + private TransportService transportService; private Client client; private SecurityIndexManager securityIndex; private ClusterService clusterService; @@ -65,6 +66,11 @@ public class TransportInvalidateTokenActionTests extends ESTestCase { @Before public void setup() { threadPool = new TestThreadPool(getTestName()); + + // TODO: temporary, remove in #97879 + transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); + securityContext = new SecurityContext(Settings.EMPTY, threadPool.getThreadContext()); client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); @@ -91,7 +97,7 @@ public void testInvalidateTokensWhenIndexUnavailable() throws Exception { clusterService ); final TransportInvalidateTokenAction action = new TransportInvalidateTokenAction( - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService ); @@ -137,7 +143,7 @@ public void testInvalidateTokensWhenIndexClosed() throws Exception { clusterService ); final TransportInvalidateTokenAction action = new TransportInvalidateTokenAction( - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java index b4a79e070e927..9f7815a1c9891 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest; @@ -57,8 +58,10 @@ public void testHasPrivilegesRequestDoesNotAllowDLSRoleQueryBasedIndicesPrivileg .build(false); when(context.getAuthentication()).thenReturn(authentication); threadContext.putTransient(AuthenticationField.AUTHENTICATION_KEY, authentication); + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final TransportHasPrivilegesAction transportHasPrivilegesAction = new TransportHasPrivilegesAction( - mock(TransportService.class), + transportService, new ActionFilters(Set.of()), mock(AuthorizationService.class), mock(NativePrivilegeStore.class), @@ -110,8 +113,10 @@ public void testRequiresSameUser() { ); return null; }).when(authorizationService).checkPrivileges(any(), any(), anyCollection(), anyActionListener()); + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final var action = new TransportHasPrivilegesAction( - mock(TransportService.class), + transportService, new ActionFilters(Set.of()), authorizationService, privilegeStore, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java index 600d4d1ba5fa4..2a15aa09ddccd 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.PageCacheRecycler; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.IOUtils; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.mocksocket.MockSocket; @@ -159,7 +160,7 @@ public TransportRequestHandler interceptHandler( DiscoveryNode remoteNode = remoteTransportService.getLocalDiscoNode(); remoteTransportService.registerRequestHandler( RemoteClusterNodesAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, RemoteClusterNodesAction.Request::new, (request, channel, task) -> channel.sendResponse(new RemoteClusterNodesAction.Response(List.of(remoteNode))) ); diff --git a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeActionTests.java b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeActionTests.java index 67ce8c255fd94..cf28bf9922b24 100644 --- a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeActionTests.java +++ b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeActionTests.java @@ -19,8 +19,8 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.shutdown.TransportDeleteShutdownNodeAction.DeleteShutdownNodeExecutor; import org.elasticsearch.xpack.shutdown.TransportDeleteShutdownNodeAction.DeleteShutdownNodeTask; import org.junit.Before; @@ -55,7 +55,7 @@ public void init() { MockitoAnnotations.openMocks(this); // TODO: it takes almost 2 seconds to create these mocks....WHY?!? var threadPool = mock(ThreadPool.class); - var transportService = mock(TransportService.class); + var transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); clusterService = mock(ClusterService.class); var actionFilters = mock(ActionFilters.class); var indexNameExpressionResolver = mock(IndexNameExpressionResolver.class); diff --git a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java index c264ad6ad1d03..cbd51ceebc729 100644 --- a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java +++ b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java @@ -19,8 +19,8 @@ import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.shutdown.TransportPutShutdownNodeAction.PutShutdownNodeExecutor; import org.elasticsearch.xpack.shutdown.TransportPutShutdownNodeAction.PutShutdownNodeTask; import org.junit.Before; @@ -61,7 +61,7 @@ public void init() { MockitoAnnotations.openMocks(this); // TODO: it takes almost 2 seconds to create these mocks....WHY?!? var threadPool = mock(ThreadPool.class); - var transportService = mock(TransportService.class); + var transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); clusterService = mock(ClusterService.class); var actionFilters = mock(ActionFilters.class); var indexNameExpressionResolver = mock(IndexNameExpressionResolver.class); diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java index 4c71683a972a1..954b5ba024418 100644 --- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java +++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.reservedstate.service.ReservedStateUpdateTask; import org.elasticsearch.reservedstate.service.ReservedStateUpdateTaskExecutor; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.XContentParser; @@ -363,10 +364,12 @@ public void testOperatorControllerFromJSONContent() throws IOException { } public void testDeleteSLMReservedStateHandler() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var deleteAction = new TransportDeleteSnapshotLifecycleAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); @@ -377,10 +380,12 @@ public void testDeleteSLMReservedStateHandler() { } public void testPutSLMReservedStateHandler() throws Exception { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var putAction = new TransportPutSnapshotLifecycleAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java index 0294c14f57494..5d0e002c49036 100644 --- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java +++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ilm.action.StopILMAction; @@ -33,11 +34,12 @@ public class TransportStopSLMActionTests extends ESTestCase { public void testStopILMClusterStatePriorityIsImmediate() { ClusterService clusterService = mock(ClusterService.class); - + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportStopSLMAction transportStopSLMAction = new TransportStopSLMAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java index 22f8ae8969ec7..66281cd21856b 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java @@ -194,7 +194,7 @@ public Map getProcessors(Processor.Parameters paramet @Override public List getGenericNamedWriteables() { - return List.of(new GenericNamedWriteableSpec(CartesianBoundingBox.class.getSimpleName(), CartesianBoundingBox::new)); + return List.of(new GenericNamedWriteableSpec("CartesianBoundingBox", CartesianBoundingBox::new)); } private static void registerGeoShapeBoundsAggregator(ValuesSourceRegistry.Builder builder) { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index 1cba21a526552..55929a1c1b83e 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -171,7 +171,7 @@ public GeoShapeWithDocValuesFieldMapper build(MapperBuilderContext context) { name, ft, multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, new GeoShapeIndexer(orientation.get().value(), ft.name()), parser, this diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java index bf63f2d84c189..378b78111ab19 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java @@ -119,7 +119,7 @@ public FieldMapper build(MapperBuilderContext context) { parser, meta.get() ); - return new PointFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), parser, this); + return new PointFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, parser, this); } } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java index 1b784595caa16..127a4fd1050cd 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java @@ -122,7 +122,7 @@ public ShapeFieldMapper build(MapperBuilderContext context) { parser, meta.get() ); - return new ShapeFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), parser, this); + return new ShapeFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, parser, this); } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportActionTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportActionTests.java index 83eb9d49a4e21..ede64decbca33 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportActionTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; @@ -56,13 +57,15 @@ public void init() { } public void testAvailable() throws Exception { - SpatialInfoTransportAction featureSet = new SpatialInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + SpatialInfoTransportAction featureSet = new SpatialInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); var usageAction = new SpatialUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, mockClient() @@ -80,14 +83,16 @@ public void testAvailable() throws Exception { } public void testEnabled() throws Exception { - SpatialInfoTransportAction featureSet = new SpatialInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + SpatialInfoTransportAction featureSet = new SpatialInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); assertTrue(featureSet.enabled()); SpatialUsageTransportAction usageAction = new SpatialUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, mockClient() diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportActionTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportActionTests.java index 4f42305984b57..1b91f686a7d8d 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportActionTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportActionTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -47,8 +48,8 @@ public class SpatialStatsTransportActionTests extends ESTestCase { @Before public void mockServices() { - transportService = mock(TransportService.class); threadPool = mock(ThreadPool.class); + transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); clusterService = mock(ClusterService.class); DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("nodeId"); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java index f95fd337fe5b7..01bad2c3dee60 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ObjectPath; @@ -57,11 +58,8 @@ public void init() throws Exception { } public void testAvailable() { - SqlInfoTransportAction featureSet = new SqlInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class), - licenseState - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + SqlInfoTransportAction featureSet = new SqlInfoTransportAction(transportService, mock(ActionFilters.class), licenseState); assertThat(featureSet.available(), is(true)); } @@ -95,10 +93,12 @@ public void testUsageStats() throws Exception { when(mockNode.getId()).thenReturn("mocknode"); when(clusterService.localNode()).thenReturn(mockNode); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var usageAction = new SqlUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, licenseState, diff --git a/x-pack/plugin/stack/build.gradle b/x-pack/plugin/stack/build.gradle index c6cb0e4ab91ff..6b26373c40544 100644 --- a/x-pack/plugin/stack/build.gradle +++ b/x-pack/plugin/stack/build.gradle @@ -1,4 +1,5 @@ apply plugin: 'elasticsearch.internal-es-plugin' +apply plugin: 'elasticsearch.internal-java-rest-test' esplugin { name 'x-pack-stack' @@ -8,13 +9,25 @@ esplugin { hasNativeController false requiresKeystore true } + base { archivesName = 'x-pack-stack' } dependencies { compileOnly project(path: xpackModule('core')) - testImplementation(testArtifact(project(xpackModule('core')))) + javaRestTestImplementation(testArtifact(project(xpackModule('core')))) + javaRestTestImplementation project(path: ':x-pack:plugin:stack') + clusterModules project(':modules:mapper-extras') + clusterModules project(xpackModule('wildcard')) +} + +// These tests are only invoked direclty as part of a dedicated build job +tasks.named('javaRestTest').configure {task -> + onlyIf("E2E test task must be invoked directly") { + gradle.startParameter.getTaskNames().contains(task.path) || + (gradle.startParameter.getTaskNames().contains(task.name) && gradle.startParameter.currentDir == project.projectDir) + } } addQaCheckDependencies(project) diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java new file mode 100644 index 0000000000000..9832c789e2a99 --- /dev/null +++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java @@ -0,0 +1,413 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.stack; + +import org.apache.http.HttpStatus; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.template.TemplateUtils; +import org.junit.BeforeClass; +import org.junit.ClassRule; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +@SuppressWarnings("unchecked") +public class EcsDynamicTemplatesIT extends ESRestTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("mapper-extras").module("wildcard").build(); + + // The dynamic templates we test against + public static final String ECS_DYNAMIC_TEMPLATES_FILE = "ecs-dynamic-mappings.json"; + + // The current ECS state (branch main) containing all fields in flattened form + private static final String ECS_FLAT_FILE_URL = "https://raw.githubusercontent.com/elastic/ecs/main/generated/ecs/ecs_flat.yml"; + + private static final Set OMIT_FIELD_TYPES = Set.of("object", "nested"); + + private static final Set OMIT_FIELDS = Set.of("data_stream.dataset", "data_stream.namespace", "data_stream.type"); + + private static Map ecsDynamicTemplates; + private static Map> ecsFlatFieldDefinitions; + private static Map ecsFlatMultiFieldDefinitions; + + @BeforeClass + public static void setupSuiteScopeCluster() throws Exception { + prepareEcsDynamicTemplates(); + prepareEcsDefinitions(); + } + + private static void prepareEcsDynamicTemplates() throws IOException { + String rawEcsComponentTemplate = TemplateUtils.loadTemplate( + "/" + ECS_DYNAMIC_TEMPLATES_FILE, + Integer.toString(1), + StackTemplateRegistry.TEMPLATE_VERSION_VARIABLE, + Collections.emptyMap() + ); + Map ecsDynamicTemplatesRaw; + try ( + XContentParser parser = XContentFactory.xContent(XContentType.JSON) + .createParser(XContentParserConfiguration.EMPTY, rawEcsComponentTemplate) + ) { + ecsDynamicTemplatesRaw = parser.map(); + } + + String errorMessage = String.format( + Locale.ENGLISH, + "ECS mappings component template '%s' structure has changed, this test needs to be adjusted", + ECS_DYNAMIC_TEMPLATES_FILE + ); + assertFalse(errorMessage, rawEcsComponentTemplate.isEmpty()); + Object mappings = ecsDynamicTemplatesRaw.get("template"); + assertNotNull(errorMessage, mappings); + assertThat(errorMessage, mappings, instanceOf(Map.class)); + Object dynamicTemplates = ((Map) mappings).get("mappings"); + assertNotNull(errorMessage, dynamicTemplates); + assertThat(errorMessage, dynamicTemplates, instanceOf(Map.class)); + assertEquals(errorMessage, 1, ((Map) dynamicTemplates).size()); + assertTrue(errorMessage, ((Map) dynamicTemplates).containsKey("dynamic_templates")); + ecsDynamicTemplates = (Map) dynamicTemplates; + } + + @SuppressForbidden(reason = "Opening socket connection to read ECS definitions from ECS GitHub repo") + private static void prepareEcsDefinitions() throws IOException { + Map ecsFlatFieldsRawMap; + URL ecsDefinitionsFlatFileUrl = new URL(ECS_FLAT_FILE_URL); + try (InputStream ecsDynamicTemplatesIS = ecsDefinitionsFlatFileUrl.openStream()) { + try ( + XContentParser parser = XContentFactory.xContent(XContentType.YAML) + .createParser(XContentParserConfiguration.EMPTY, ecsDynamicTemplatesIS) + ) { + ecsFlatFieldsRawMap = parser.map(); + } + } + String errorMessage = String.format( + Locale.ENGLISH, + "ECS flat mapping file at %s has changed, this test needs to be adjusted", + ECS_FLAT_FILE_URL + ); + assertFalse(errorMessage, ecsFlatFieldsRawMap.isEmpty()); + Map.Entry fieldEntry = ecsFlatFieldsRawMap.entrySet().iterator().next(); + assertThat(errorMessage, fieldEntry.getValue(), instanceOf(Map.class)); + Map fieldProperties = (Map) fieldEntry.getValue(); + assertFalse(errorMessage, fieldProperties.isEmpty()); + Map.Entry fieldProperty = fieldProperties.entrySet().iterator().next(); + assertThat(errorMessage, fieldProperty.getKey(), instanceOf(String.class)); + + OMIT_FIELDS.forEach(ecsFlatFieldsRawMap::remove); + + // noinspection + ecsFlatFieldDefinitions = (Map>) ecsFlatFieldsRawMap; + ecsFlatMultiFieldDefinitions = new HashMap<>(); + Iterator>> iterator = ecsFlatFieldDefinitions.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry> entry = iterator.next(); + Map definitions = entry.getValue(); + String type = (String) definitions.get("type"); + if (OMIT_FIELD_TYPES.contains(type)) { + iterator.remove(); + } + + List> multiFields = (List>) definitions.get("multi_fields"); + if (multiFields != null) { + multiFields.forEach(multiFieldsDefinitions -> { + String subfieldFlatName = Objects.requireNonNull(multiFieldsDefinitions.get("flat_name")); + String subfieldType = Objects.requireNonNull(multiFieldsDefinitions.get("type")); + ecsFlatMultiFieldDefinitions.put(subfieldFlatName, subfieldType); + }); + } + } + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + public void testFlattenedFields() throws IOException { + String indexName = "test-flattened-fields"; + createTestIndex(indexName); + Map flattenedFieldsMap = createTestDocument(true); + indexDocument(indexName, flattenedFieldsMap); + verifyEcsMappings(indexName); + } + + public void testFlattenedFieldsWithoutSubobjects() throws IOException { + String indexName = "test_flattened_fields_subobjects_false"; + createTestIndex(indexName, Map.of("subobjects", false)); + Map flattenedFieldsMap = createTestDocument(true); + indexDocument(indexName, flattenedFieldsMap); + verifyEcsMappings(indexName); + } + + public void testNestedFields() throws IOException { + String indexName = "test-nested-fields"; + createTestIndex(indexName); + Map nestedFieldsMap = createTestDocument(false); + indexDocument(indexName, nestedFieldsMap); + verifyEcsMappings(indexName); + } + + private static void indexDocument(String indexName, Map flattenedFieldsMap) throws IOException { + try (XContentBuilder bodyBuilder = JsonXContent.contentBuilder()) { + Request indexRequest = new Request("POST", "/" + indexName + "/_doc"); + indexRequest.setJsonEntity(Strings.toString(bodyBuilder.map(flattenedFieldsMap))); + // noinspection resource + Response response = ESRestTestCase.client().performRequest(indexRequest); + assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); + } + } + + private Map createTestDocument(boolean flattened) { + Map testFieldsMap = new HashMap<>(); + for (Map.Entry> fieldEntry : ecsFlatFieldDefinitions.entrySet()) { + String flattenedFieldName = fieldEntry.getKey(); + Map fieldDefinitions = fieldEntry.getValue(); + String type = (String) fieldDefinitions.get("type"); + assertNotNull( + String.format(Locale.ENGLISH, "Can't find type for field '%s' in %s file", flattenedFieldName, ECS_DYNAMIC_TEMPLATES_FILE), + type + ); + Object testValue = generateTestValue(type); + if (flattened) { + testFieldsMap.put(flattenedFieldName, testValue); + } else { + Map currentField = testFieldsMap; + Iterator fieldPathPartsIterator = Arrays.stream(flattenedFieldName.split("\\.")).iterator(); + String subfield = fieldPathPartsIterator.next(); + while (fieldPathPartsIterator.hasNext()) { + currentField = (Map) currentField.computeIfAbsent(subfield, ignore -> new HashMap<>()); + subfield = fieldPathPartsIterator.next(); + } + currentField.put(subfield, testValue); + } + } + return testFieldsMap; + } + + private static void createTestIndex(String indexName) throws IOException { + createTestIndex(indexName, null); + } + + private static void createTestIndex(String indexName, @Nullable Map customMappings) throws IOException { + final Map indexMappings; + if (customMappings != null) { + indexMappings = Stream.concat(ecsDynamicTemplates.entrySet().stream(), customMappings.entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + } else { + indexMappings = ecsDynamicTemplates; + } + try (XContentBuilder bodyBuilder = JsonXContent.contentBuilder()) { + bodyBuilder.startObject(); + bodyBuilder.startObject("settings"); + bodyBuilder.field("index.mapping.total_fields.limit", 10000); + bodyBuilder.endObject(); + bodyBuilder.field("mappings", indexMappings); + bodyBuilder.endObject(); + + Request createIndexRequest = new Request("PUT", "/" + indexName); + createIndexRequest.setJsonEntity(Strings.toString(bodyBuilder)); + // noinspection resource + Response response = ESRestTestCase.client().performRequest(createIndexRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + } + } + + private Object generateTestValue(String type) { + switch (type) { + case "geo_point" -> { + return new double[] { randomDouble(), randomDouble() }; + } + case "long" -> { + return randomLong(); + } + case "int" -> { + return randomInt(); + } + case "float", "scaled_float" -> { + return randomFloat(); + } + case "keyword", "wildcard", "text", "match_only_text" -> { + return randomAlphaOfLength(20); + } + case "constant_keyword" -> { + return "test"; + } + case "date" -> { + return DateFormatter.forPattern("strict_date_optional_time").formatMillis(System.currentTimeMillis()); + } + case "ip" -> { + return NetworkAddress.format(randomIp(true)); + } + case "boolean" -> { + return randomBoolean(); + } + case "flattened" -> { + // creating multiple subfields + return Map.of("subfield1", randomAlphaOfLength(20), "subfield2", randomAlphaOfLength(20)); + } + } + throw new IllegalArgumentException("Unknown field type: " + type); + } + + private Map getMappings(String indexName) throws IOException { + Request getMappingRequest = new Request("GET", "/" + indexName + "/_mapping"); + // noinspection resource + Response response = ESRestTestCase.client().performRequest(getMappingRequest); + assertEquals(response.getStatusLine().getStatusCode(), HttpStatus.SC_OK); + Map mappingResponse; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, response.getEntity().getContent())) { + mappingResponse = parser.map(); + } + assertThat(mappingResponse.size(), equalTo(1)); + Map indexMap = (Map) mappingResponse.get(indexName); + assertNotNull(indexMap); + Map mappings = (Map) indexMap.get("mappings"); + assertNotNull(mappings); + return (Map) mappings.get("properties"); + } + + private void processRawMappingsSubtree( + final Map fieldSubtrees, + final Map flatFieldMappings, + final Map flatMultiFieldsMappings, + final String subtreePrefix + ) { + fieldSubtrees.forEach((fieldName, fieldMappings) -> { + String fieldFullPath = subtreePrefix + fieldName; + Map fieldMappingsMap = ((Map) fieldMappings); + String type = (String) fieldMappingsMap.get("type"); + if (type != null) { + flatFieldMappings.put(fieldFullPath, type); + } + Map subfields = (Map) fieldMappingsMap.get("properties"); + if (subfields != null) { + processRawMappingsSubtree(subfields, flatFieldMappings, flatMultiFieldsMappings, fieldFullPath + "."); + } + + Map> fields = (Map>) fieldMappingsMap.get("fields"); + if (fields != null) { + fields.forEach((subFieldName, multiFieldMappings) -> { + String subFieldFullPath = fieldFullPath + "." + subFieldName; + String subFieldType = Objects.requireNonNull(multiFieldMappings.get("type")); + flatMultiFieldsMappings.put(subFieldFullPath, subFieldType); + }); + } + }); + } + + private void verifyEcsMappings(String indexName) throws IOException { + final Map rawMappings = getMappings(indexName); + final Map flatFieldMappings = new HashMap<>(); + final Map flatMultiFieldsMappings = new HashMap<>(); + processRawMappingsSubtree(rawMappings, flatFieldMappings, flatMultiFieldsMappings, ""); + + Map> shallowFieldMapCopy = new HashMap<>(ecsFlatFieldDefinitions); + logger.info("Testing mapping of {} ECS fields", shallowFieldMapCopy.size()); + List nonEcsFields = new ArrayList<>(); + Map fieldToWrongMappingType = new HashMap<>(); + flatFieldMappings.forEach((fieldName, actualMappingType) -> { + Map expectedMappings = shallowFieldMapCopy.remove(fieldName); + if (expectedMappings == null) { + nonEcsFields.add(fieldName); + } else { + String expectedType = (String) expectedMappings.get("type"); + if (actualMappingType.equals(expectedType) == false) { + fieldToWrongMappingType.put(fieldName, actualMappingType); + } + } + }); + + Map shallowMultiFieldMapCopy = new HashMap<>(ecsFlatMultiFieldDefinitions); + logger.info("Testing mapping of {} ECS multi-fields", shallowMultiFieldMapCopy.size()); + flatMultiFieldsMappings.forEach((fieldName, actualMappingType) -> { + String expectedType = shallowMultiFieldMapCopy.remove(fieldName); + if (expectedType != null) { + // not finding an entry in the expected multi-field mappings map is acceptable: our dynamic templates are required to + // ensure multi-field mapping for all fields with such ECS definitions. However, the patterns in these templates may lead + // to multi-field mapping for ECS fields for which such are not defined + if (actualMappingType.equals(expectedType) == false) { + fieldToWrongMappingType.put(fieldName, actualMappingType); + } + } + }); + + shallowFieldMapCopy.forEach( + (fieldName, expectedMappings) -> logger.error( + "ECS field '{}' is not covered by the current dynamic templates. Update {} so that this field is mapped to type '{}'.", + fieldName, + ECS_DYNAMIC_TEMPLATES_FILE, + expectedMappings.get("type") + ) + ); + shallowMultiFieldMapCopy.keySet().forEach(field -> { + int lastDotIndex = field.lastIndexOf('.'); + String parentField = field.substring(0, lastDotIndex); + String subfield = field.substring(lastDotIndex + 1); + logger.error( + "ECS field '{}' is expected to have a multi-field mapping with subfield '{}'. Fix {} accordingly.", + parentField, + subfield, + ECS_DYNAMIC_TEMPLATES_FILE + ); + }); + fieldToWrongMappingType.forEach((fieldName, actualMappingType) -> { + String ecsExpectedType = (String) ecsFlatFieldDefinitions.get(fieldName).get("type"); + logger.error( + "ECS field '{}' should be mapped to type '{}' but is mapped to type '{}'. Update {} accordingly.", + fieldName, + ecsExpectedType, + actualMappingType, + ECS_DYNAMIC_TEMPLATES_FILE + ); + }); + nonEcsFields.forEach(field -> logger.error("The test document contains '{}', which is not an ECS field", field)); + + assertTrue("ECS is not fully covered by the current ECS dynamic templates, see details above", shallowFieldMapCopy.isEmpty()); + assertTrue( + "ECS is not fully covered by the current ECS dynamic templates' multi-fields definitions, see details above", + shallowMultiFieldMapCopy.isEmpty() + ); + assertTrue( + "At least one field was mapped with a type that mismatches the ECS definitions, see details above", + fieldToWrongMappingType.isEmpty() + ); + assertTrue("The test document contains non-ECS fields, see details above", nonEcsFields.isEmpty()); + } +} diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/HistogramGroupByIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/HistogramGroupByIT.java index 797d592ef4571..ccf9409d84bd8 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/HistogramGroupByIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/HistogramGroupByIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.transform.integration.continuous; +import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.client.Response; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Strings; @@ -28,6 +29,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/97263") public class HistogramGroupByIT extends ContinuousTestCase { private static final String NAME = "continuous-histogram-pivot-test"; diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java index f4b717a108762..8ea0d5e62c6d3 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.transform.integration.continuous; +import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Strings; import org.elasticsearch.search.aggregations.AggregationBuilders; @@ -25,6 +26,7 @@ import static org.hamcrest.Matchers.equalTo; +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/97263") public class TermsGroupByIT extends ContinuousTestCase { private static final String NAME = "continuous-terms-pivot-test"; diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java index 5c6f875415384..d2c23f768cef6 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats; @@ -29,12 +30,14 @@ public class TransformInfoTransportActionTests extends ESTestCase { public void testAvailable() { - TransformInfoTransportAction featureSet = new TransformInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + TransformInfoTransportAction featureSet = new TransformInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testEnabledDefault() { - TransformInfoTransportAction featureSet = new TransformInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + TransformInfoTransportAction featureSet = new TransformInfoTransportAction(transportService, mock(ActionFilters.class)); assertTrue(featureSet.enabled()); } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java index 0b796ee3d3f7d..ac683dca5bf26 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ObjectPath; @@ -55,20 +56,23 @@ public class WatcherInfoTransportActionTests extends ESTestCase { private MockLicenseState licenseState; private Client client; + private ThreadPool threadPool; + private TransportService transportService; @Before public void init() throws Exception { licenseState = mock(MockLicenseState.class); client = mock(Client.class); - ThreadPool threadPool = mock(ThreadPool.class); + threadPool = mock(ThreadPool.class); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); when(threadPool.getThreadContext()).thenReturn(threadContext); when(client.threadPool()).thenReturn(threadPool); + transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); } public void testAvailable() { WatcherInfoTransportAction featureSet = new WatcherInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), Settings.EMPTY, licenseState @@ -89,7 +93,7 @@ public void testEnabled() { settings.put("xpack.watcher.enabled", enabled); } WatcherInfoTransportAction featureSet = new WatcherInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), settings.build(), licenseState @@ -130,9 +134,9 @@ public void testUsageStats() throws Exception { when(clusterService.localNode()).thenReturn(mockNode); var usageAction = new WatcherUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, Settings.EMPTY, diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchActionTests.java index 79c93cd5b913a..338f4e56f5663 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchActionTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.license.TestUtils; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.watcher.WatcherMetadata; @@ -56,8 +57,8 @@ public class TransportAckWatchActionTests extends ESTestCase { @Before public void setupAction() { - TransportService transportService = mock(TransportService.class); ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); when(threadPool.getThreadContext()).thenReturn(threadContext); WatchParser watchParser = mock(WatchParser.class); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchActionTests.java index 09cf9daaf3685..95272619fac5d 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchActionTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.license.TestUtils; import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ClientHelper; @@ -63,7 +64,7 @@ public void setupAction() throws Exception { ThreadPool threadPool = mock(ThreadPool.class); when(threadPool.getThreadContext()).thenReturn(threadContext); - TransportService transportService = mock(TransportService.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); WatchParser parser = mock(WatchParser.class); when(parser.parseWithSecrets(eq("_id"), eq(false), any(), any(), any(), anyBoolean(), anyLong(), anyLong())).thenReturn(watch); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsActionTests.java index 8d07d1eaf6043..6dfdf47a7a735 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsActionTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -44,8 +45,8 @@ public class TransportWatcherStatsActionTests extends ESTestCase { @Before public void setupTransportAction() { - TransportService transportService = mock(TransportService.class); ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); ClusterService clusterService = mock(ClusterService.class); DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("nodeId"); diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 469e140eb6b76..e38ee029bd123 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -242,7 +242,7 @@ public WildcardFieldMapper build(MapperBuilderContext context) { ignoreAbove.get(), context.isSourceSynthetic(), multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, nullValue.get(), indexVersionCreated ); @@ -879,9 +879,17 @@ protected String parseSourceValue(Object value) { } + private static final FieldType NGRAM_FIELD_TYPE; + + static { + FieldType ft = new FieldType(Defaults.FIELD_TYPE); + ft.setTokenized(true); + NGRAM_FIELD_TYPE = freezeAndDeduplicateFieldType(ft); + assert NGRAM_FIELD_TYPE.indexOptions() == IndexOptions.DOCS; + } + private final int ignoreAbove; private final String nullValue; - private final FieldType ngramFieldType; private final IndexVersion indexVersionCreated; private final boolean storeIgnored; @@ -900,10 +908,6 @@ private WildcardFieldMapper( this.ignoreAbove = ignoreAbove; this.storeIgnored = storeIgnored; this.indexVersionCreated = indexVersionCreated; - this.ngramFieldType = new FieldType(Defaults.FIELD_TYPE); - this.ngramFieldType.setTokenized(true); - this.ngramFieldType.freeze(); - assert ngramFieldType.indexOptions() == IndexOptions.DOCS; } @Override @@ -954,7 +958,7 @@ private String originalName() { void createFields(String value, LuceneDocument parseDoc, List fields) { String ngramValue = addLineEndChars(value); - Field ngramField = new Field(fieldType().name(), ngramValue, ngramFieldType); + Field ngramField = new Field(fieldType().name(), ngramValue, NGRAM_FIELD_TYPE); fields.add(ngramField); CustomBinaryDocValuesField dvField = (CustomBinaryDocValuesField) parseDoc.getByKey(fieldType().name()); diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransportVersionClusterStateUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransportVersionClusterStateUpgradeIT.java index 7c7a392a6cf83..470525a69ea0b 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransportVersionClusterStateUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransportVersionClusterStateUpgradeIT.java @@ -12,70 +12,157 @@ import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.common.util.Maps; +import org.elasticsearch.test.rest.ObjectPath; -import java.io.IOException; -import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.oneOf; public class TransportVersionClusterStateUpgradeIT extends AbstractUpgradeTestCase { - public void testReadsInferredTransportVersions() throws IOException { - assumeTrue("TransportVersion introduced in 8.8.0", UPGRADE_FROM_VERSION.before(Version.V_8_8_0)); - assumeTrue( - "This only has visible effects when upgrading beyond 8.8.0", - TransportVersion.current().after(TransportVersions.V_8_8_0) - ); - assumeTrue("Only runs on the mixed cluster", CLUSTER_TYPE == ClusterType.MIXED); - // if the master is not upgraded, and the secondary node is, then the cluster info from the secondary - // should have inferred transport versions in it - // rely on randomisation to hit this case at some point in testing - - Request request = new Request("GET", "/_cluster/state/nodes"); - Map masterResponse = entityAsMap(client().performRequest(request)); - assumeFalse("Master needs to not know about transport versions", masterResponse.containsKey("transport_versions")); - - request = new Request("GET", "/_cluster/state/nodes?local=true"); - Map localResponse = entityAsMap(client().performRequest(request)); - // should either be empty, or using inferred versions - assumeTrue("Local node needs to know about transport versions", masterResponse.containsKey("transport_versions")); - Map vs = Maps.transformValues( - ((Map) localResponse.get("nodes")), - v -> Version.fromString(((Map) v).get("version").toString()) - ); - Map tvs = ((List) localResponse.get("transport_versions")).stream() - .map(o -> (Map) o) - .collect(Collectors.toMap(m -> m.get("node_id"), m -> TransportVersion.fromString(m.get("transport_version").toString()))); - - for (var ver : vs.entrySet()) { - if (ver.getValue().after(Version.V_8_8_0)) { - assertThat( - "Node " + ver.getKey() + " should have an inferred transport version", - tvs.get(ver.getKey()), - equalTo(TransportVersions.V_8_8_0) - ); + private static final Version VERSION_INTRODUCING_TRANSPORT_VERSIONS = Version.V_8_8_0; + private static final Version VERSION_INTRODUCING_NODES_VERSIONS = Version.V_8_11_0; + private static final TransportVersion FIRST_TRANSPORT_VERSION = TransportVersions.V_8_8_0; + + public void testReadsInferredTransportVersions() throws Exception { + assertEquals(VERSION_INTRODUCING_TRANSPORT_VERSIONS.id(), FIRST_TRANSPORT_VERSION.id()); + + // waitUntil because the versions fixup on upgrade happens in the background so may need a retry + assertTrue(waitUntil(() -> { + try { + // check several responses in order to sample from a selection of nodes + for (int i = getClusterHosts().size(); i > 0; i--) { + if (runTransportVersionsTest() == false) { + return false; + } + } + return true; + } catch (Exception e) { + throw new AssertionError(e); } - } + })); } - public void testCompletesRealTransportVersions() throws IOException { - assumeTrue("TransportVersion introduced in 8.8.0", UPGRADE_FROM_VERSION.before(Version.V_8_8_0)); - assumeTrue( - "This only has visible effects when upgrading beyond 8.8.0", - TransportVersion.current().after(TransportVersions.V_8_8_0) + private boolean runTransportVersionsTest() throws Exception { + final var clusterState = ObjectPath.createFromResponse( + client().performRequest(new Request("GET", "/_cluster/state" + randomFrom("", "/nodes") + randomFrom("", "?local"))) ); - assumeTrue("Only runs on the upgraded cluster", CLUSTER_TYPE == ClusterType.UPGRADED); - // once everything is upgraded, the master should fill in the real transport versions + final var description = clusterState.toString(); + + final var nodeIds = clusterState.evaluateMapKeys("nodes"); + final Map versionsByNodeId = Maps.newHashMapWithExpectedSize(nodeIds.size()); + for (final var nodeId : nodeIds) { + versionsByNodeId.put(nodeId, Version.fromString(clusterState.evaluate("nodes." + nodeId + ".version"))); + } + + final var hasTransportVersions = clusterState.evaluate("transport_versions") != null; + final var hasNodesVersions = clusterState.evaluate("nodes_versions") != null; + assertFalse(description, hasNodesVersions && hasTransportVersions); + + switch (CLUSTER_TYPE) { + case OLD -> { + if (UPGRADE_FROM_VERSION.before(VERSION_INTRODUCING_TRANSPORT_VERSIONS)) { + // Before 8.8.0 there was only DiscoveryNode#version + assertFalse(description, hasTransportVersions); + assertFalse(description, hasNodesVersions); + } else if (UPGRADE_FROM_VERSION.before(VERSION_INTRODUCING_NODES_VERSIONS)) { + // In [8.8.0, 8.11.0) we exposed just transport_versions + assertTrue(description, hasTransportVersions); + assertFalse(description, hasNodesVersions); + } else { + // From 8.11.0 onwards we exposed nodes_versions + assertFalse(description, hasTransportVersions); + assertTrue(description, hasNodesVersions); + } + } + case MIXED -> { + if (UPGRADE_FROM_VERSION.before(VERSION_INTRODUCING_TRANSPORT_VERSIONS)) { + // Responding node might be <8.8.0 (so no extra versions) or >=8.11.0 (includes nodes_versions) + assertFalse(description, hasTransportVersions); + } else if (UPGRADE_FROM_VERSION.before(VERSION_INTRODUCING_NODES_VERSIONS)) { + // Responding node might be in [8.8.0, 8.11.0) (transport_versions) or >=8.11.0 (includes nodes_versions) but not both + assertTrue(description, hasNodesVersions || hasTransportVersions); + } else { + // Responding node is ≥8.11.0 so has nodes_versions for sure + assertFalse(description, hasTransportVersions); + assertTrue(description, hasNodesVersions); + } + } + case UPGRADED -> { + // All nodes are Version.CURRENT, ≥8.11.0, so we definitely have nodes_versions + assertFalse(description, hasTransportVersions); + assertTrue(description, hasNodesVersions); + assertThat(description, versionsByNodeId.values(), everyItem(equalTo(Version.CURRENT))); + } + } + + if (hasTransportVersions) { + // Upgrading from [8.8.0, 8.11.0) and the responding node is still on the old version + assertThat(description, UPGRADE_FROM_VERSION, lessThan(VERSION_INTRODUCING_NODES_VERSIONS)); + assertThat(description, UPGRADE_FROM_VERSION, greaterThanOrEqualTo(VERSION_INTRODUCING_TRANSPORT_VERSIONS)); + assertNotEquals(description, ClusterType.UPGRADED, CLUSTER_TYPE); - Request request = new Request("GET", "/_cluster/state/nodes"); - Map response = entityAsMap(client().performRequest(request)); - Map tvs = ((List) response.get("transport_versions")).stream() - .map(o -> (Map) o) - .collect(Collectors.toMap(m -> m.get("node_id"), m -> TransportVersion.fromString(m.get("transport_version").toString()))); + // transport_versions includes the correct version for all nodes, no inference is needed + assertEquals(description, nodeIds.size(), clusterState.evaluateArraySize("transport_versions")); + for (int i = 0; i < nodeIds.size(); i++) { + final var path = "transport_versions." + i; + final String nodeId = clusterState.evaluate(path + ".node_id"); + final var nodeDescription = nodeId + "/" + description; + final var transportVersion = TransportVersion.fromString(clusterState.evaluate(path + ".transport_version")); + final var nodeVersion = versionsByNodeId.get(nodeId); + assertNotNull(nodeDescription, nodeVersion); + if (nodeVersion.equals(Version.CURRENT)) { + assertEquals(nodeDescription, TransportVersion.current(), transportVersion); + } else if (nodeVersion.after(VERSION_INTRODUCING_TRANSPORT_VERSIONS)) { + assertThat(nodeDescription, transportVersion, greaterThan(FIRST_TRANSPORT_VERSION)); + } else { + assertEquals(nodeDescription, FIRST_TRANSPORT_VERSION, transportVersion); + } + } + } else if (hasNodesVersions) { + // Either upgrading from ≥8.11.0 (the responding node might be old or new), or from <8.8.0 (the responding node is new) + assertFalse(description, UPGRADE_FROM_VERSION.before(VERSION_INTRODUCING_NODES_VERSIONS) && CLUSTER_TYPE == ClusterType.OLD); + + // nodes_versions includes _a_ version for all nodes; it might be correct, or it might be inferred if we're upgrading from + // <8.8.0 and the master is still an old node or the TransportVersionsFixupListener hasn't run yet + assertEquals(description, nodeIds.size(), clusterState.evaluateArraySize("nodes_versions")); + for (int i = 0; i < nodeIds.size(); i++) { + final var path = "nodes_versions." + i; + final String nodeId = clusterState.evaluate(path + ".node_id"); + final var nodeDescription = nodeId + "/" + description; + final var transportVersion = TransportVersion.fromString(clusterState.evaluate(path + ".transport_version")); + final var nodeVersion = versionsByNodeId.get(nodeId); + assertNotNull(nodeDescription, nodeVersion); + if (nodeVersion.equals(Version.CURRENT)) { + // Either the responding node is upgraded or the upgrade is trivial; if the responding node is upgraded but the master + // is not then its transport version may be temporarily inferred as 8.8.0 until TransportVersionsFixupListener runs. + assertThat( + nodeDescription, + transportVersion, + UPGRADE_FROM_VERSION.onOrAfter(VERSION_INTRODUCING_TRANSPORT_VERSIONS) + ? equalTo(TransportVersion.current()) + : oneOf(TransportVersion.current(), FIRST_TRANSPORT_VERSION) + ); + if (CLUSTER_TYPE == ClusterType.UPGRADED && transportVersion.equals(FIRST_TRANSPORT_VERSION)) { + // TransportVersionsFixupListener should run soon, retry + logger.info("{} - not fixed up yet, retrying", nodeDescription); + return false; + } + } else if (nodeVersion.after(VERSION_INTRODUCING_TRANSPORT_VERSIONS)) { + // There's no relationship between node versions and transport versions any more, although we can be sure of this: + assertThat(nodeDescription, transportVersion, greaterThan(FIRST_TRANSPORT_VERSION)); + } else { + // Responding node is not upgraded, and no later than 8.8.0, so we infer its version correctly. + assertEquals(nodeDescription, TransportVersion.fromId(nodeVersion.id()), transportVersion); + } + } + } - assertThat(tvs + " should be updated", tvs.values(), everyItem(equalTo(TransportVersion.current()))); + return true; } }