diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+concurrent-search-tests-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+concurrent-search-tests-trigger.yml deleted file mode 100644 index 1816e7143355f..0000000000000 --- a/.ci/jobs.t/elastic+elasticsearch+periodic+concurrent-search-tests-trigger.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -jjbb-template: periodic-trigger-lgc.yml -vars: - - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+concurrent-search-tests - - lgc-job: elastic+elasticsearch+%BRANCH%+intake - - cron: "H H/12 * * *" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+concurrent-search-tests.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+concurrent-search-tests.yml index 99758cb9d088c..ad48635654459 100644 --- a/.ci/jobs.t/elastic+elasticsearch+periodic+concurrent-search-tests.yml +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+concurrent-search-tests.yml @@ -2,7 +2,8 @@ - job: name: elastic+elasticsearch+%BRANCH%+periodic+concurrent-search-tests display-name: "elastic / elasticsearch # %BRANCH% - concurrent search tests" - description: "Testing concurrent search enabled for the Elasticsearch %BRANCH% branch.\n" + description: "This job has been migrated to Buildkite.\n" + disabled: true node: "general-purpose && docker" builders: - inject: diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+ear-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+ear-trigger.yml deleted file mode 100644 index a50a1f96358ad..0000000000000 --- a/.ci/jobs.t/elastic+elasticsearch+periodic+ear-trigger.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -jjbb-template: periodic-trigger-lgc.yml -vars: - - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+ear - - lgc-job: elastic+elasticsearch+%BRANCH%+intake - - cron: "H H/12 * * *" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+ear.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+ear.yml index b1b5a39f92a5a..67462d3a2a809 100644 --- a/.ci/jobs.t/elastic+elasticsearch+periodic+ear.yml +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+ear.yml @@ -2,7 +2,8 @@ - job: name: elastic+elasticsearch+%BRANCH%+periodic+ear display-name: "elastic / elasticsearch # %BRANCH% - encryption at rest" - description: "The Elasticsearch %BRANCH% branch encryption at rest compatibility tests.\n\n" + description: "This job has been migrated to Buildkite.\n" + disabled: true node: packaging-large builders: - inject: diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+eql-correctness-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+eql-correctness-trigger.yml deleted file mode 100644 index 986c52a137de3..0000000000000 --- a/.ci/jobs.t/elastic+elasticsearch+periodic+eql-correctness-trigger.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -jjbb-template: periodic-trigger-lgc.yml -vars: - - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+eql-correctness - - lgc-job: elastic+elasticsearch+%BRANCH%+intake - - cron: "H H/8 * * *" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+eql-correctness.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+eql-correctness.yml index 2652732974661..a23bae19134fc 100644 --- a/.ci/jobs.t/elastic+elasticsearch+periodic+eql-correctness.yml +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+eql-correctness.yml @@ -3,7 +3,8 @@ name: elastic+elasticsearch+%BRANCH%+periodic+eql-correctness workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+eql-correctness display-name: "elastic / elasticsearch # %BRANCH% - eql correctness tests" - description: "Testing of Elasticsearch %BRANCH% EQL.\n" + description: "This job has been migrated to Buildkite.\n" + disabled: true builders: - inject: properties-file: '.ci/java-versions.properties' diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+example-plugins-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+example-plugins-trigger.yml deleted file mode 100644 index 909d175f11882..0000000000000 --- a/.ci/jobs.t/elastic+elasticsearch+periodic+example-plugins-trigger.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -jjbb-template: periodic-trigger-lgc.yml -vars: - - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+example-plugins - - lgc-job: elastic+elasticsearch+%BRANCH%+intake - - cron: "H H/12 * * *" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+example-plugins.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+example-plugins.yml index 2423a85b2a6bd..ee496690e82ce 100644 --- a/.ci/jobs.t/elastic+elasticsearch+periodic+example-plugins.yml +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+example-plugins.yml @@ -3,7 +3,8 @@ name: elastic+elasticsearch+%BRANCH%+periodic+example-plugins workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+example-plugins display-name: "elastic / elasticsearch # %BRANCH% - example plugin tests" - description: "Testing of Elasticsearch %BRANCH% example plugins.\n" + description: "This job has been migrated to Buildkite.\n" + disabled: true builders: - inject: properties-file: '.ci/java-versions.properties' diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests-trigger.yml deleted file mode 100644 index c624c929b3dd6..0000000000000 --- a/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests-trigger.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -jjbb-template: periodic-trigger-lgc.yml -vars: - - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+release-tests - - lgc-job: elastic+elasticsearch+%BRANCH%+intake - - cron: "H H/12 * * *" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests.yml index 5fc2d5cd1ca5d..abaf4242e1648 100644 --- a/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests.yml +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests.yml @@ -4,7 +4,8 @@ # Don't use ramdisk since this build generates lots of large artifacts and results in oomkiller issues # workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+release-tests display-name: "elastic / elasticsearch # %BRANCH% - release tests" - description: "Release version tests for the Elasticsearch %BRANCH% branch.\n" + description: "This job has been migrated to Buildkite.\n" + disabled: true node: "general-purpose && docker" builders: - inject: diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+single-processor-node-tests.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+single-processor-node-tests.yml index c67baa07da1ee..02240bf1bb339 100644 --- a/.ci/jobs.t/elastic+elasticsearch+periodic+single-processor-node-tests.yml +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+single-processor-node-tests.yml @@ -2,7 +2,8 @@ - job: name: elastic+elasticsearch+%BRANCH%+periodic+single-processor-node-tests display-name: "elastic / elasticsearch # %BRANCH% - single processor node tests" - description: "Testing with node.processors set to '1' for the Elasticsearch %BRANCH% branch.\n" + description: "This job has been migrated to Buildkite.\n" + disabled: true node: "general-purpose && docker" builders: - inject: diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+single-processor-tests-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+single-processor-tests-trigger.yml deleted file mode 100644 index 40ad9e9dd5446..0000000000000 --- a/.ci/jobs.t/elastic+elasticsearch+periodic+single-processor-tests-trigger.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -jjbb-template: periodic-trigger-lgc.yml -vars: - - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+single-processor-node-tests - - lgc-job: elastic+elasticsearch+%BRANCH%+intake - - cron: "H H/12 * * *" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+snyk-dependency-monitoring-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+snyk-dependency-monitoring-trigger.yml deleted file mode 100644 index fa0f06c3315af..0000000000000 --- a/.ci/jobs.t/elastic+elasticsearch+periodic+snyk-dependency-monitoring-trigger.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -jjbb-template: periodic-trigger-lgc.yml -vars: - - periodic-job: elastic+elasticsearch+%BRANCH%+snyk-dependency-monitoring - - lgc-job: elastic+elasticsearch+%BRANCH%+intake - - cron: "H H * * *" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+snyk-dependency-monitoring.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+snyk-dependency-monitoring.yml index 1a76003f3d13c..6190937cc6490 100644 --- a/.ci/jobs.t/elastic+elasticsearch+periodic+snyk-dependency-monitoring.yml +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+snyk-dependency-monitoring.yml @@ -3,7 +3,8 @@ name: elastic+elasticsearch+%BRANCH%+snyk-dependency-monitoring workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+snyk-dependency-monitoring display-name: "elastic / elasticsearch # %BRANCH% - snyk dependency monitoring" - description: "Publishing of the Elasticsearch %BRANCH% dependencies graph to snyk dependency monitoring" + description: "This job has been migrated to Buildkite.\n" + disabled: true builders: - inject: properties-file: '.ci/java-versions.properties' diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java index 1827babe6f091..9fa876a00c35c 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java @@ -19,6 +19,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.DocVector; @@ -132,6 +133,7 @@ private static BlockLoader numericBlockLoader(String name, NumberFieldMapper.Num @OperationsPerInvocation(INDEX_SIZE) public void benchmark() { ValuesSourceReaderOperator op = new ValuesSourceReaderOperator( + BlockFactory.getNonBreakingInstance(), List.of(BlockReaderFactories.loaderToFactory(reader, blockLoader(name))), 0, name diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 7b7040dfd7098..96e577d5635ab 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -407,7 +407,7 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) { if (base == DockerBase.IRON_BANK) { Map buildArgsMap = [ 'BASE_REGISTRY': 'docker.elastic.co', - 'BASE_IMAGE' : 'ubi8/ubi', + 'BASE_IMAGE' : 'ubi9/ubi', 'BASE_TAG' : 'latest' ] diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index a6ecfdaf417e1..b62fa983dd480 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -21,8 +21,8 @@ <% if (docker_base == 'iron_bank') { %> ARG BASE_REGISTRY=registry1.dso.mil -ARG BASE_IMAGE=ironbank/redhat/ubi/ubi8 -ARG BASE_TAG=8.6 +ARG BASE_IMAGE=redhat/ubi/ubi9 +ARG BASE_TAG=9.2 <% } %> ################################################################################ diff --git a/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml b/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml index 9fce16efad328..7152f6d18f1d2 100644 --- a/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml +++ b/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml @@ -13,8 +13,8 @@ tags: # Build args passed to Dockerfile ARGs args: - BASE_IMAGE: "redhat/ubi/ubi8" - BASE_TAG: "8.6" + BASE_IMAGE: "redhat/ubi/ubi9" + BASE_TAG: "9.2" # Docker image labels labels: diff --git a/docs/changelog/101147.yaml b/docs/changelog/101147.yaml new file mode 100644 index 0000000000000..cb556af35eead --- /dev/null +++ b/docs/changelog/101147.yaml @@ -0,0 +1,5 @@ +pr: 101147 +summary: Persist data counts on job close before results index refresh +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/101230.yaml b/docs/changelog/101230.yaml new file mode 100644 index 0000000000000..3ed7eacb3fce0 --- /dev/null +++ b/docs/changelog/101230.yaml @@ -0,0 +1,12 @@ +pr: 101230 +summary: Enable query phase parallelism within a single shard +area: Search +type: enhancement +issues: + - 80693 +highlight: + title: Enable query phase parallelism within a single shard + body: |- + Activate inter-segment search concurrency by default in the query phase, in order to + enable parallelizing search execution across segments that a single shard is made of. + notable: true diff --git a/docs/changelog/101346.yaml b/docs/changelog/101346.yaml new file mode 100644 index 0000000000000..b32b123c506d1 --- /dev/null +++ b/docs/changelog/101346.yaml @@ -0,0 +1,5 @@ +pr: 101346 +summary: Report full stack trace for non-state file settings transforms +area: Infra/Settings +type: bug +issues: [] diff --git a/docs/changelog/101358.yaml b/docs/changelog/101358.yaml new file mode 100644 index 0000000000000..3ae2a44e15e5e --- /dev/null +++ b/docs/changelog/101358.yaml @@ -0,0 +1,6 @@ +pr: 101358 +summary: Make DISSECT parameter `append_separator` case insensitive +area: ES|QL +type: bug +issues: + - 101138 diff --git a/docs/changelog/101383.yaml b/docs/changelog/101383.yaml new file mode 100644 index 0000000000000..4875403acfaeb --- /dev/null +++ b/docs/changelog/101383.yaml @@ -0,0 +1,5 @@ +pr: 101383 +summary: "ESQL: Track memory from values loaded from lucene" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/101396.yaml b/docs/changelog/101396.yaml new file mode 100644 index 0000000000000..a486b2bed9237 --- /dev/null +++ b/docs/changelog/101396.yaml @@ -0,0 +1,5 @@ +pr: 101396 +summary: "ESQL: Track blocks emitted from lucene" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/reference/data-streams/data-stream-apis.asciidoc b/docs/reference/data-streams/data-stream-apis.asciidoc index a25c5728be597..d3580ca4448a7 100644 --- a/docs/reference/data-streams/data-stream-apis.asciidoc +++ b/docs/reference/data-streams/data-stream-apis.asciidoc @@ -15,6 +15,8 @@ The following APIs are available for managing <>: [[data-stream-lifecycle-api]] The following APIs are available for managing the built-in lifecycle of data streams: +preview::[] + * <> preview:[] * <> diff --git a/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc index a3bdf20b85715..fd481d7ca4815 100644 --- a/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc @@ -4,7 +4,7 @@ Delete Data Stream Lifecycle ++++ -preview:[] +preview::[] Deletes the lifecycle from a set of data streams. diff --git a/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc index 1ccd36f5468e2..a2609dcb78ecf 100644 --- a/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc @@ -4,7 +4,7 @@ Explain Data Stream Lifecycle ++++ -preview:[] +preview::[] Retrieves the current data stream lifecycle status for one or more data stream backing indices. diff --git a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc index 66b30b7975c11..f20a3393c191c 100644 --- a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc @@ -4,7 +4,7 @@ Get Data Stream Lifecycle ++++ -preview:[] +preview::[] Gets the lifecycle of a set of data streams. diff --git a/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc index dd893d0abe757..89b8bbeb880c3 100644 --- a/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc @@ -4,7 +4,7 @@ Put Data Stream Lifecycle ++++ -preview:[] +preview::[] Configures the data stream lifecycle for the targeted data streams. diff --git a/docs/reference/data-streams/lifecycle/index.asciidoc b/docs/reference/data-streams/lifecycle/index.asciidoc index 5f7596087cb5b..6c0220ef0a80f 100644 --- a/docs/reference/data-streams/lifecycle/index.asciidoc +++ b/docs/reference/data-streams/lifecycle/index.asciidoc @@ -2,7 +2,7 @@ [[data-stream-lifecycle]] == Data stream lifecycle -preview:[] +preview::[] A data stream lifecycle is the built-in mechanism data streams use to manage their lifecycle. It enables you to easily automate the management of your data streams according to your retention requirements. For example, you could configure diff --git a/docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc index e7b4d99d25d80..5670faaade3ce 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc @@ -2,7 +2,7 @@ [[tutorial-manage-existing-data-stream]] === Tutorial: Update existing data stream -preview:[] +preview::[] To update the lifecycle of an existing data stream you do the following actions: diff --git a/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc index f22a0ae736d9e..6f1d81ab6ead2 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc @@ -2,7 +2,7 @@ [[tutorial-manage-new-data-stream]] === Tutorial: Create a data stream with a lifecycle -preview:[] +preview::[] To create a data stream with a built-in lifecycle, follow these steps: diff --git a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc index c96bfa75516a8..de11bbcfc2d4e 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc @@ -2,7 +2,7 @@ [[tutorial-migrate-data-stream-from-ilm-to-dsl]] === Tutorial: Migrate ILM managed data stream to Data stream lifecycle -preview:[] +preview::[] In this tutorial we'll look at migrating an existing data stream from {ilm-init} to Data stream lifecycle. The existing {ilm-init} managed backing indices will continue diff --git a/docs/reference/esql/esql-enrich-data.asciidoc b/docs/reference/esql/esql-enrich-data.asciidoc index f1ebe7bc218a8..69cc7817d2224 100644 --- a/docs/reference/esql/esql-enrich-data.asciidoc +++ b/docs/reference/esql/esql-enrich-data.asciidoc @@ -1,5 +1,5 @@ [[esql-enrich-data]] -== Data enrichment +=== Data enrichment ++++ Data enrichment @@ -17,7 +17,7 @@ For example, you can use `ENRICH` to: [discrete] [[esql-how-enrich-works]] -=== How the `ENRICH` command works +==== How the `ENRICH` command works The `ENRICH` command adds new columns to a table, with data from {es} indices. It requires a few special components: @@ -65,7 +65,7 @@ include::../ingest/enrich.asciidoc[tag=enrich-index] [discrete] [[esql-set-up-enrich-policy]] -=== Set up an enrich policy +==== Set up an enrich policy To start using `ENRICH`, follow these steps: @@ -89,25 +89,25 @@ your query. [discrete] [[esql-enrich-prereqs]] -=== Prerequisites +==== Prerequisites include::{es-repo-dir}/ingest/apis/enrich/put-enrich-policy.asciidoc[tag=enrich-policy-api-prereqs] [discrete] [[esql-create-enrich-source-index]] -=== Add enrich data +==== Add enrich data include::../ingest/enrich.asciidoc[tag=create-enrich-source-index] [discrete] [[esql-create-enrich-policy]] -=== Create an enrich policy +==== Create an enrich policy include::../ingest/enrich.asciidoc[tag=create-enrich-policy] [discrete] [[esql-execute-enrich-policy]] -=== Execute the enrich policy +==== Execute the enrich policy include::../ingest/enrich.asciidoc[tag=execute-enrich-policy1] @@ -117,7 +117,7 @@ include::../ingest/enrich.asciidoc[tag=execute-enrich-policy2] [discrete] [[esql-use-enrich]] -=== Use the enrich policy +==== Use the enrich policy After the policy has been executed, you can use the <> to enrich your data. @@ -128,12 +128,12 @@ include::processing-commands/enrich.asciidoc[tag=examples] [discrete] [[esql-update-enrich-data]] -=== Update an enrich index +==== Update an enrich index include::{es-repo-dir}/ingest/apis/enrich/execute-enrich-policy.asciidoc[tag=update-enrich-index] [discrete] [[esql-update-enrich-policies]] -=== Update an enrich policy +==== Update an enrich policy include::../ingest/enrich.asciidoc[tag=update-enrich-policy] diff --git a/docs/reference/esql/esql-examples.asciidoc b/docs/reference/esql/esql-examples.asciidoc index da13608175910..569dcf1172b38 100644 --- a/docs/reference/esql/esql-examples.asciidoc +++ b/docs/reference/esql/esql-examples.asciidoc @@ -1,5 +1,5 @@ [[esql-examples]] -== Examples +== {esql} examples ++++ Examples diff --git a/docs/reference/esql/esql-kibana.asciidoc b/docs/reference/esql/esql-kibana.asciidoc index 534cba22ed1a1..b8709364367b2 100644 --- a/docs/reference/esql/esql-kibana.asciidoc +++ b/docs/reference/esql/esql-kibana.asciidoc @@ -1,15 +1,256 @@ [[esql-kibana]] -== Using {esql} in {kib} +=== Using {esql} in {kib} ++++ -Kibana +Using {esql} in {kib} ++++ +You can use {esql} in {kib} to query and aggregate your data, create +visualizations, and set up alerts. -Use {esql} in Discover to explore a data set. From the data view dropdown, -select *Try {esql}* to get started. +This guide shows you how to use {esql} in Kibana. To follow along with the +queries, load the "Sample web logs" sample data set by clicking *Try sample +data* from the {kib} Home, selecting *Other sample data sets*, and clicking *Add +data* on the *Sample web logs* card. -NOTE: {esql} queries in Discover and Lens are subject to the time range selected -with the time filter. +[discrete] +[[esql-kibana-get-started]] +=== Get started with {esql} +To get started with {esql} in Discover, open the main menu and select +*Discover*. Next, from the Data views menu, select *Try ES|QL*. +image::images/esql/esql-data-view-menu.png[align="center",width=33%] + +The ability to select {esql} from the Data views menu can be enabled and +disabled using the `discover:enableESQL` setting from +{kibana-ref}/advanced-options.html[Advanced Settings]. + +[discrete] +[[esql-kibana-query-bar]] +=== The query bar + +After switching to {esql} mode, the query bar shows a sample query. For example: + +[source,esql] +---- +from kibana_sample_data_logs | limit 10 +---- + +Every query starts with a <>. In this query, the +source command is <>. `FROM` retrieves data from data streams, indices, or +aliases. In this example, the data is retrieved from `kibana_sample_data_logs`. + +A source command can be followed by one or more <>. In this query, the processing command is <>. `LIMIT` +limits the number of rows that are retrieved. + +TIP: Click the help icon (image:images/esql/esql-icon-help.svg[]) to open the +in-product reference documentation for all commands and functions. + +To make it easier to write queries, auto-complete offers suggestions with +possible commands and functions: + +image::images/esql/esql-kibana-auto-complete.png[align="center"] + +[NOTE] +==== +{esql} keywords are case-insensitive. The following query is identical to the +previous one: + +[source,esql] +---- +FROM kibana_sample_data_logs | LIMIT 10 +---- +==== + +[discrete] +==== Expand the query bar + +For readability, you can put each processing command on a new line. The +following query is identical to the previous one: + +[source,esql] +---- +FROM kibana_sample_data_logs +| LIMIT 10 +---- + +To make it easier to write multi-line queries, click the double-headed arrow +button (image:images/esql/esql-icon-expand-query-bar.svg[]) to expand the query +bar: + +image::images/esql/esql-expanded-query-bar.png[align="center"] + +To return to a compact query bar, click the minimize editor button +(image:images/esql/esql-icon-minimize-query-bar.svg[]). + +[discrete] +==== Warnings + +A query may result in warnings, for example when querying an unsupported field +type. When that happens, a warning symbol is shown in the query bar. To see the +detailed warning, expand the query bar, and click *warnings*. + +[discrete] +[[esql-kibana-results-table]] +=== The results table + +For the example query, the results table shows 10 rows. Omitting the `LIMIT` +command, the results table defaults to up to 500 rows. Using `LIMIT`, you can +increase the limit to up to 10,000 rows. + +NOTE: the 10,000 row limit only applies to the number of rows that are retrieved +by the query and displayed in Discover. Any query or aggregation runs on the +full data set. + +Each row shows two columns for the example query: a column with the `@timestamp` +field and a column with the full document. To display specific fields from the +documents, use the <> command: + +[source,esql] +---- +FROM kibana_sample_data_logs +| KEEP @timestamp, bytes, geo.dest +---- + +To display all fields as separate columns, use `KEEP *`: + +[source,esql] +---- +FROM kibana_sample_data_logs +| KEEP * +---- + +NOTE: The maximum number of columns in Discover is 50. If a query returns more +than 50 columns, Discover only shows the first 50. + +[discrete] +==== Sorting + +To sort on one of the columns, click the column name you want to sort on and +select the sort order. Note that this performs client-side sorting. It only +sorts the rows that were retrieved by the query, which may not be the full +dataset because of the (implicit) limit. To sort the full data set, use the +<> command: + +[source,esql] +---- +FROM kibana_sample_data_logs +| KEEP @timestamp, bytes, geo.dest +| SORT bytes DESC +---- + +[discrete] +[[esql-kibana-time-filter]] +=== Time filtering + +To display data within a specified time range, use the +{kibana-ref}/set-time-filter.html[time filter]. The time filter is only enabled +when the indices you're querying have a field called `@timestamp`. + +If your indices do not have a timestamp field called `@timestamp`, you can limit +the time range using the <> command and the <> function. +For example, if the timestamp field is called `timestamp`, to query the last 15 +minutes of data: +[source,esql] +---- +FROM kibana_sample_data_logs +| WHERE timestamp > NOW() - 15minutes +---- + +[discrete] +[[esql-kibana-visualizations]] +=== Analyze and visualize data + +Between the query bar and the results table, Discover shows a date histogram +visualization. If the indices you're querying do not contain an `@timestamp` +field, the histogram is not shown. + +The visualization adapts to the query. A query's nature determines the type of +visualization. For example, this query aggregates the total number of bytes per +destination country: + +[source,esql] +---- +FROM kibana_sample_data_logs +| STATS total_bytes = SUM(bytes) BY geo.dest +| SORT total_bytes DESC +| LIMIT 3 +---- + +The resulting visualization is a bar chart showing the top 3 countries: + +image::images/esql/esql-kibana-bar-chart.png[align="center"] + +To change the visualization into another type, click the visualization type +dropdown: + +image::images/esql/esql-kibana-visualization-type.png[align="center",width=33%] + +To make other changes to the visualization, like the axes and colors, click the +pencil button (image:images/esql/esql-icon-edit-visualization.svg[]). This opens +an in-line editor: + +image::images/esql/esql-kibana-in-line-editor.png[align="center"] + +You can save the visualization to a new or existing dashboard by clicking the +save button (image:images/esql/esql-icon-save-visualization.svg[]). Once saved +to a dashboard, you can continue to make changes to visualization. Click the +options button in the top-right (image:images/esql/esql-icon-options.svg[]) and +select *Edit ESQL visualization* to open the in-line editor: + +image::images/esql/esql-kibana-edit-on-dashboard.png[align="center"] + +[discrete] +[[esql-kibana-enrich]] +=== Create an enrich policy + +The {esql} <> command enables you to <> +your query dataset with fields from another dataset. Before you can use +`ENRICH`, you need to <>. If a policy exists, it will be suggested by auto-complete. If not, +click *Click to create* to create one. + +image::images/esql/esql-kibana-enrich-autocomplete.png[align="center"] + +Next, you can enter a policy name, the policy type, source indices, and +optionally a query: + +image::images/esql/esql-kibana-enrich-step-1.png[align="center",width="50%"] + +Click *Next* to select the match field and enrich fields: + +image::images/esql/esql-kibana-enrich-step-2.png[align="center",width="50%"] + +Finally, click *Create and execute*. + +Now, you can use the enrich policy in an {esql} query: + +image::images/esql/esql-kibana-enriched-data.png[align="center"] + +[discrete] +[[esql-kibana-alerting-rule]] +=== Create an alerting rule + +You can use {esql} queries to create alerts. From Discover, click *Alerts* and +select *Create search threshold rule*. This opens a panel that enables you to +create a rule using an {esql} query. Next, you can test the query, add a +connector, and save the rule. + +image::images/esql/esql-kibana-create-rule.png[align="center",width=50%] + +[discrete] +[[esql-kibana-limitations]] +=== Limitations + +* The user interface to filter data is not enabled when Discover is in {esql} +mode. To filter data, write a query that uses the <> command +instead. +* In {esql} mode, clicking a field in the field list in Discover does not show +quick statistics for that field. +* Discover shows no more than 10,000 rows. This limit only applies to the number +of rows that are retrieved by the query and displayed in Discover. Any query or +aggregation runs on the full data set. +* Discover shows no more than 50 columns. If a query returns +more than 50 columns, Discover only shows the first 50. \ No newline at end of file diff --git a/docs/reference/esql/esql-language.asciidoc b/docs/reference/esql/esql-language.asciidoc index 6f8e54ebd86cb..8ffc0af7cbeb2 100644 --- a/docs/reference/esql/esql-language.asciidoc +++ b/docs/reference/esql/esql-language.asciidoc @@ -10,11 +10,15 @@ Detailed information about the {esql} language: * <> * <> * <> +* <> * <> +* <> * <> include::esql-syntax.asciidoc[] include::esql-commands.asciidoc[] include::esql-functions-operators.asciidoc[] +include::metadata-fields.asciidoc[] include::multivalued-fields.asciidoc[] include::esql-process-data-with-dissect-grok.asciidoc[] +include::esql-enrich-data.asciidoc[] \ No newline at end of file diff --git a/docs/reference/esql/esql-rest.asciidoc b/docs/reference/esql/esql-rest.asciidoc index 55c9946ad08b4..2d47f6e46ff65 100644 --- a/docs/reference/esql/esql-rest.asciidoc +++ b/docs/reference/esql/esql-rest.asciidoc @@ -1,5 +1,5 @@ [[esql-rest]] -== {esql} REST API +=== {esql} REST API ++++ REST API @@ -38,7 +38,7 @@ James S.A. Corey |Leviathan Wakes |561 |2011-06-02T00:00:00.000Z [discrete] [[esql-kibana-console]] -=== Kibana Console +==== Kibana Console If you are using {kibana-ref}/console-kibana.html[Kibana Console] (which is highly recommended), take advantage of the triple quotes `"""` when creating the @@ -62,7 +62,7 @@ POST /_query?format=txt [discrete] [[esql-rest-format]] -=== Response formats +==== Response formats {esql} can return the data in the following human readable and binary formats. You can set the format by specifying the `format` parameter in the URL or by @@ -121,7 +121,7 @@ Use the `tsv` format instead. [discrete] [[esql-rest-filtering]] -=== Filtering using {es} Query DSL +==== Filtering using {es} Query DSL Specify a Query DSL query in the `filter` parameter to filter the set of documents that an {esql} query runs on. @@ -161,7 +161,7 @@ Douglas Adams |The Hitchhiker's Guide to the Galaxy|180 |1979-10-12T [discrete] [[esql-rest-columnar]] -=== Columnar results +==== Columnar results By default, {esql} returns results as rows. For example, `FROM` returns each individual document as one row. For the `json`, `yaml`, `cbor` and `smile` @@ -206,7 +206,7 @@ Which returns: [discrete] [[esql-rest-params]] -=== Passing parameters to a query +==== Passing parameters to a query Values, for example for a condition, can be passed to a query "inline", by integrating the value in the query string itself: diff --git a/docs/reference/esql/esql-using.asciidoc b/docs/reference/esql/esql-using.asciidoc new file mode 100644 index 0000000000000..f586f3a28de5c --- /dev/null +++ b/docs/reference/esql/esql-using.asciidoc @@ -0,0 +1,16 @@ +[[esql-using]] +== Using {esql} + +<>:: +Information about using the <>. + +<>:: +Using {esql} in {kib} to query and aggregate your data, create visualizations, +and set up alerts. + +<>:: +Using the <> to list and cancel {esql} queries. + +include::esql-rest.asciidoc[] +include::esql-kibana.asciidoc[] +include::task-management.asciidoc[] \ No newline at end of file diff --git a/docs/reference/esql/index.asciidoc b/docs/reference/esql/index.asciidoc index c86dc088ff6b8..2946f4e61d629 100644 --- a/docs/reference/esql/index.asciidoc +++ b/docs/reference/esql/index.asciidoc @@ -8,40 +8,67 @@ preview::[] -The {es} Query Language ({esql}) provides a powerful way to filter, transform, and analyze data stored in {es}, and in the future in other runtimes. -It is designed to be easy to learn and use, by end users, SRE teams, application developers, and administrators. +The {es} Query Language ({esql}) provides a powerful way to filter, transform, +and analyze data stored in {es}, and in the future in other runtimes. It is +designed to be easy to learn and use, by end users, SRE teams, application +developers, and administrators. -Users can author {esql} queries to find specific events, perform statistical analysis, and generate visualizations. -It supports a wide range of commands and functions that enable users to perform various data operations, -such as filtering, aggregation, time-series analysis, and more. +Users can author {esql} queries to find specific events, perform statistical +analysis, and generate visualizations. It supports a wide range of commands and +functions that enable users to perform various data operations, such as +filtering, aggregation, time-series analysis, and more. -The {es} Query Language ({esql}) makes use of "pipes" (|) to manipulate and transform data in a step-by-step fashion. -This approach allows users to compose a series of operations, where the output of one operation becomes the input for the next, -enabling complex data transformations and analysis. +The {es} Query Language ({esql}) makes use of "pipes" (|) to manipulate and +transform data in a step-by-step fashion. This approach allows users to compose +a series of operations, where the output of one operation becomes the input for +the next, enabling complex data transformations and analysis. [discrete] === The {esql} Compute Engine -{esql} is more than a language: it represents a significant investment in new compute capabilities within {es}. -To achieve both the functional and performance requirements for {esql}, it was necessary to build an entirely new -compute architecture. {esql} search, aggregation, and transformation functions are directly executed within Elasticsearch -itself. Query expressions are not transpiled to Query DSL for execution. This approach allows {esql} to be extremely performant and versatile. +{esql} is more than a language: it represents a significant investment in new +compute capabilities within {es}. To achieve both the functional and performance +requirements for {esql}, it was necessary to build an entirely new compute +architecture. {esql} search, aggregation, and transformation functions are +directly executed within Elasticsearch itself. Query expressions are not +transpiled to Query DSL for execution. This approach allows {esql} to be +extremely performant and versatile. -The new {esql} execution engine was designed with performance in mind — it operates on blocks at a time instead of per row, targets vectorization and cache locality, and embraces specialization and multi-threading. It is a separate component from the existing Elasticsearch aggregation framework with different performance characteristics. +The new {esql} execution engine was designed with performance in mind — it +operates on blocks at a time instead of per row, targets vectorization and cache +locality, and embraces specialization and multi-threading. It is a separate +component from the existing Elasticsearch aggregation framework with different +performance characteristics. -include::esql-get-started.asciidoc[] +The {esql} documentation is organized in these sections: -include::esql-language.asciidoc[] +<>:: +A tutorial to help you get started with {esql}. + +<>:: + +Reference documentation for the <>, +<>, and <>. Information about working with <> and <>. And guidance for +<> and <>. -include::esql-rest.asciidoc[] +<>:: +An overview of using the <>, <>, and +<>. -include::metadata-fields.asciidoc[] +<>:: +The current limitations of {esql}. -include::esql-kibana.asciidoc[] +<>:: +A few examples of what you can with {esql}. -include::task-management.asciidoc[] +include::esql-get-started.asciidoc[] + +include::esql-language.asciidoc[] -include::esql-enrich-data.asciidoc[] +include::esql-using.asciidoc[] include::esql-limitations.asciidoc[] diff --git a/docs/reference/esql/metadata-fields.asciidoc b/docs/reference/esql/metadata-fields.asciidoc index 69c9c0c04dd7b..c034d4d0dd2b3 100644 --- a/docs/reference/esql/metadata-fields.asciidoc +++ b/docs/reference/esql/metadata-fields.asciidoc @@ -1,5 +1,5 @@ [[esql-metadata-fields]] -== {esql} metadata fields +=== {esql} metadata fields ++++ Metadata fields diff --git a/docs/reference/esql/task-management.asciidoc b/docs/reference/esql/task-management.asciidoc index 96a624c89bf7d..dfaff96123035 100644 --- a/docs/reference/esql/task-management.asciidoc +++ b/docs/reference/esql/task-management.asciidoc @@ -1,5 +1,5 @@ [[esql-task-management]] -== {esql} task management +=== {esql} task management ++++ Task management diff --git a/docs/reference/images/esql/esql-data-view-menu.png b/docs/reference/images/esql/esql-data-view-menu.png new file mode 100644 index 0000000000000..fbbbdf44d315c Binary files /dev/null and b/docs/reference/images/esql/esql-data-view-menu.png differ diff --git a/docs/reference/images/esql/esql-expanded-query-bar.png b/docs/reference/images/esql/esql-expanded-query-bar.png new file mode 100644 index 0000000000000..1c26d72b86fb9 Binary files /dev/null and b/docs/reference/images/esql/esql-expanded-query-bar.png differ diff --git a/docs/reference/images/esql/esql-icon-edit-visualization.svg b/docs/reference/images/esql/esql-icon-edit-visualization.svg new file mode 100644 index 0000000000000..c559f0e589c72 --- /dev/null +++ b/docs/reference/images/esql/esql-icon-edit-visualization.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/reference/images/esql/esql-icon-expand-query-bar.svg b/docs/reference/images/esql/esql-icon-expand-query-bar.svg new file mode 100644 index 0000000000000..fc2641318acb2 --- /dev/null +++ b/docs/reference/images/esql/esql-icon-expand-query-bar.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/reference/images/esql/esql-icon-help.svg b/docs/reference/images/esql/esql-icon-help.svg new file mode 100644 index 0000000000000..84c9b8db397c9 --- /dev/null +++ b/docs/reference/images/esql/esql-icon-help.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/reference/images/esql/esql-icon-minimize-query-bar.svg b/docs/reference/images/esql/esql-icon-minimize-query-bar.svg new file mode 100644 index 0000000000000..54b9349a05d59 --- /dev/null +++ b/docs/reference/images/esql/esql-icon-minimize-query-bar.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/reference/images/esql/esql-icon-options.svg b/docs/reference/images/esql/esql-icon-options.svg new file mode 100644 index 0000000000000..6abb3ca9d0827 --- /dev/null +++ b/docs/reference/images/esql/esql-icon-options.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/reference/images/esql/esql-icon-save-visualization.svg b/docs/reference/images/esql/esql-icon-save-visualization.svg new file mode 100644 index 0000000000000..f80a5993acde1 --- /dev/null +++ b/docs/reference/images/esql/esql-icon-save-visualization.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/reference/images/esql/esql-kibana-auto-complete.png b/docs/reference/images/esql/esql-kibana-auto-complete.png new file mode 100644 index 0000000000000..5763e569c7668 Binary files /dev/null and b/docs/reference/images/esql/esql-kibana-auto-complete.png differ diff --git a/docs/reference/images/esql/esql-kibana-bar-chart.png b/docs/reference/images/esql/esql-kibana-bar-chart.png new file mode 100644 index 0000000000000..43190a34bf3c3 Binary files /dev/null and b/docs/reference/images/esql/esql-kibana-bar-chart.png differ diff --git a/docs/reference/images/esql/esql-kibana-create-rule.png b/docs/reference/images/esql/esql-kibana-create-rule.png new file mode 100644 index 0000000000000..c9fb14b0d2ee9 Binary files /dev/null and b/docs/reference/images/esql/esql-kibana-create-rule.png differ diff --git a/docs/reference/images/esql/esql-kibana-edit-on-dashboard.png b/docs/reference/images/esql/esql-kibana-edit-on-dashboard.png new file mode 100644 index 0000000000000..cea540e78f4a8 Binary files /dev/null and b/docs/reference/images/esql/esql-kibana-edit-on-dashboard.png differ diff --git a/docs/reference/images/esql/esql-kibana-enrich-autocomplete.png b/docs/reference/images/esql/esql-kibana-enrich-autocomplete.png new file mode 100644 index 0000000000000..15b95c650ea88 Binary files /dev/null and b/docs/reference/images/esql/esql-kibana-enrich-autocomplete.png differ diff --git a/docs/reference/images/esql/esql-kibana-enrich-step-1.png b/docs/reference/images/esql/esql-kibana-enrich-step-1.png new file mode 100644 index 0000000000000..22f71d987d3be Binary files /dev/null and b/docs/reference/images/esql/esql-kibana-enrich-step-1.png differ diff --git a/docs/reference/images/esql/esql-kibana-enrich-step-2.png b/docs/reference/images/esql/esql-kibana-enrich-step-2.png new file mode 100644 index 0000000000000..c8d7f5c6a05e4 Binary files /dev/null and b/docs/reference/images/esql/esql-kibana-enrich-step-2.png differ diff --git a/docs/reference/images/esql/esql-kibana-enriched-data.png b/docs/reference/images/esql/esql-kibana-enriched-data.png new file mode 100644 index 0000000000000..85df6a629eca9 Binary files /dev/null and b/docs/reference/images/esql/esql-kibana-enriched-data.png differ diff --git a/docs/reference/images/esql/esql-kibana-in-line-editor.png b/docs/reference/images/esql/esql-kibana-in-line-editor.png new file mode 100644 index 0000000000000..14caf02e60ea2 Binary files /dev/null and b/docs/reference/images/esql/esql-kibana-in-line-editor.png differ diff --git a/docs/reference/images/esql/esql-kibana-visualization-type.png b/docs/reference/images/esql/esql-kibana-visualization-type.png new file mode 100644 index 0000000000000..e36d70725b8f2 Binary files /dev/null and b/docs/reference/images/esql/esql-kibana-visualization-type.png differ diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java index 42bd1296f0b69..90a8d3379775f 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java @@ -99,17 +99,11 @@ import org.apache.lucene.analysis.tr.TurkishAnalyzer; import org.apache.lucene.analysis.util.ElisionFilter; import org.apache.lucene.util.SetOnce; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.AnalyzerProvider; @@ -120,21 +114,15 @@ import org.elasticsearch.index.analysis.PreConfiguredTokenizer; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.analysis.TokenizerFactory; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; import org.elasticsearch.indices.analysis.PreBuiltCacheFactory.CachingStrategy; import org.elasticsearch.lucene.analysis.miscellaneous.DisableGraphAttribute; import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; import org.elasticsearch.synonyms.SynonymsManagementAPIService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.tartarus.snowball.ext.DutchStemmer; import org.tartarus.snowball.ext.FrenchStemmer; @@ -144,7 +132,6 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; -import java.util.function.Supplier; import static org.elasticsearch.plugins.AnalysisPlugin.requiresAnalysisSettings; @@ -156,24 +143,9 @@ public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin, Scri private final SetOnce synonymsManagementServiceHolder = new SetOnce<>(); @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - this.scriptServiceHolder.set(scriptService); - this.synonymsManagementServiceHolder.set(new SynonymsManagementAPIService(client)); + public Collection createComponents(PluginServices services) { + this.scriptServiceHolder.set(services.scriptService()); + this.synonymsManagementServiceHolder.set(new SynonymsManagementAPIService(services.client())); return Collections.emptyList(); } diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java index 7b60c1a64abb6..3a519f594a57f 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java @@ -24,11 +24,11 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.scanners.StablePluginsRegistry; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.ESTokenStreamTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.threadpool.ThreadPool; @@ -36,6 +36,9 @@ import java.io.IOException; import java.util.Collections; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class PredicateTokenScriptFilterTests extends ESTokenStreamTestCase { public void testSimpleFilter() throws IOException { @@ -57,9 +60,9 @@ public boolean execute(Token token) { } }; - @SuppressWarnings("unchecked") ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap(), () -> 1L) { @Override + @SuppressWarnings("unchecked") public FactoryType compile(Script script, ScriptContext context) { assertEquals(context, AnalysisPredicateScript.CONTEXT); assertEquals(new Script("my_script"), script); @@ -67,23 +70,13 @@ public FactoryType compile(Script script, ScriptContext FactoryType compile(Script script, ScriptContext FactoryType compile(Script script, ScriptContext createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider unused, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { final APMTracer apmTracer = telemetryProvider.get().getTracer(); - apmTracer.setClusterName(clusterService.getClusterName().value()); - apmTracer.setNodeName(clusterService.getNodeName()); + apmTracer.setClusterName(services.clusterService().getClusterName().value()); + apmTracer.setNodeName(services.clusterService().getNodeName()); final APMAgentSettings apmAgentSettings = new APMAgentSettings(); apmAgentSettings.syncAgentSystemProperties(settings); final APMMeterService apmMeter = new APMMeterService(settings); - apmAgentSettings.addClusterSettingsListeners(clusterService, telemetryProvider.get(), apmMeter); + apmAgentSettings.addClusterSettingsListeners(services.clusterService(), telemetryProvider.get(), apmMeter); return List.of(apmTracer, apmMeter); } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java index de128c685ae98..a845b75450366 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java @@ -18,13 +18,9 @@ import org.elasticsearch.action.datastreams.MigrateToDataStreamAction; import org.elasticsearch.action.datastreams.ModifyDataStreamsAction; import org.elasticsearch.action.datastreams.PromoteDataStreamAction; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; @@ -60,20 +56,11 @@ import org.elasticsearch.datastreams.rest.RestMigrateToDataStreamAction; import org.elasticsearch.datastreams.rest.RestModifyDataStreamsAction; import org.elasticsearch.datastreams.rest.RestPromoteDataStreamAction; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexSettingProvider; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import java.io.IOException; import java.time.Clock; @@ -158,38 +145,27 @@ public List> getSettings() { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { Collection components = new ArrayList<>(); - var updateTimeSeriesRangeService = new UpdateTimeSeriesRangeService(environment.settings(), threadPool, clusterService); + var updateTimeSeriesRangeService = new UpdateTimeSeriesRangeService( + services.environment().settings(), + services.threadPool(), + services.clusterService() + ); this.updateTimeSeriesRangeService.set(updateTimeSeriesRangeService); components.add(this.updateTimeSeriesRangeService.get()); errorStoreInitialisationService.set(new DataStreamLifecycleErrorStore()); dataLifecycleInitialisationService.set( new DataStreamLifecycleService( settings, - new OriginSettingClient(client, DATA_STREAM_LIFECYCLE_ORIGIN), - clusterService, + new OriginSettingClient(services.client(), DATA_STREAM_LIFECYCLE_ORIGIN), + services.clusterService(), getClock(), - threadPool, - threadPool::absoluteTimeInMillis, + services.threadPool(), + services.threadPool()::absoluteTimeInMillis, errorStoreInitialisationService.get(), - allocationService + services.allocationService() ) ); dataLifecycleInitialisationService.get().init(); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java index afc6fa8a1c92a..26ddbaa7ba854 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java @@ -16,7 +16,6 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; @@ -25,9 +24,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.ingest.Processor; @@ -43,14 +39,10 @@ import org.elasticsearch.plugins.PersistentTaskPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.tasks.Task; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentBuilder; @@ -107,30 +99,20 @@ public Map getProcessors(Processor.Parameters paramet } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { try { - String nodeId = nodeEnvironment.nodeId(); - databaseRegistry.get().initialize(nodeId, resourceWatcherService, ingestService.get()); + String nodeId = services.nodeEnvironment().nodeId(); + databaseRegistry.get().initialize(nodeId, services.resourceWatcherService(), ingestService.get()); } catch (IOException e) { throw new UncheckedIOException(e); } - geoIpDownloaderTaskExecutor = new GeoIpDownloaderTaskExecutor(client, new HttpClient(), clusterService, threadPool); + geoIpDownloaderTaskExecutor = new GeoIpDownloaderTaskExecutor( + services.client(), + new HttpClient(), + services.clusterService(), + services.threadPool() + ); geoIpDownloaderTaskExecutor.init(); return List.of(databaseRegistry.get(), geoIpDownloaderTaskExecutor); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java index 52eccbe0dce90..62302331b38d8 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java @@ -11,20 +11,13 @@ import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.painless.action.PainlessContextAction; import org.elasticsearch.painless.action.PainlessExecuteAction; import org.elasticsearch.painless.spi.PainlessExtension; @@ -36,17 +29,11 @@ import org.elasticsearch.plugins.ExtensiblePlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.script.ScriptModule; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.ArrayList; import java.util.Arrays; @@ -135,22 +122,7 @@ public ScriptEngine getScriptEngine(Settings settings, Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { // this is a hack to bind the painless script engine in guice (all components are added to guice), so that // the painless context api. this is a temporary measure until transport actions do no require guice return Collections.singletonList(painlessScriptEngine.get()); diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java index f8f87011405fc..b07eb1b158087 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java @@ -12,35 +12,23 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.reindex.BulkByScrollTask; import org.elasticsearch.index.reindex.DeleteByQueryAction; import org.elasticsearch.index.reindex.ReindexAction; import org.elasticsearch.index.reindex.UpdateByQueryAction; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.tasks.Task; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.ArrayList; import java.util.Arrays; @@ -95,23 +83,10 @@ public List getRestHandlers( } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - return Collections.singletonList(new ReindexSslConfig(environment.settings(), environment, resourceWatcherService)); + public Collection createComponents(PluginServices services) { + return Collections.singletonList( + new ReindexSslConfig(services.environment().settings(), services.environment(), services.resourceWatcherService()) + ); } @Override diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java index 55b7b2cc902cc..5509e44b52a3e 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java @@ -19,38 +19,25 @@ import org.elasticsearch.action.support.ActionFilter; import org.elasticsearch.action.support.ActionFilterChain; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.http.HttpInfo; import org.elasticsearch.index.reindex.ReindexAction; import org.elasticsearch.index.reindex.ReindexRequestBuilder; import org.elasticsearch.index.reindex.RemoteInfo; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestHeaderDefinition; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.root.MainRestPlugin; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.tasks.Task; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.netty4.Netty4Plugin; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.junit.Before; import java.util.Arrays; @@ -58,7 +45,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.Supplier; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; @@ -164,23 +150,8 @@ public static class TestPlugin extends Plugin implements ActionPlugin { private final SetOnce testFilter = new SetOnce<>(); @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - testFilter.set(new ReindexFromRemoteWithAuthTests.TestFilter(threadPool)); + public Collection createComponents(PluginServices services) { + testFilter.set(new ReindexFromRemoteWithAuthTests.TestFilter(services.threadPool())); return Collections.emptyList(); } diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java index f88a96e765827..6ff9a40940e8c 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java @@ -11,30 +11,19 @@ import com.azure.core.util.serializer.JacksonAdapter; import org.apache.lucene.util.SetOnce; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ReloadablePlugin; import org.elasticsearch.plugins.RepositoryPlugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.ScalingExecutorBuilder; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.Arrays; @@ -42,7 +31,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.Supplier; /** * A plugin to add a repository type that writes to and from the Azure cloud storage service. @@ -84,23 +72,8 @@ public Map getRepositories( } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - AzureClientProvider azureClientProvider = AzureClientProvider.create(threadPool, settings); + public Collection createComponents(PluginServices services) { + AzureClientProvider azureClientProvider = AzureClientProvider.create(services.threadPool(), settings); azureStoreService.set(createAzureStorageService(settings, azureClientProvider)); return List.of(azureClientProvider); } diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java index a7a10a17668bc..6f43f253db4c8 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java @@ -14,7 +14,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.common.settings.SettingsModule; -import org.elasticsearch.telemetry.TelemetryProvider; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -38,6 +38,8 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class AzureStorageServiceTests extends ESTestCase { private ThreadPool threadPool; @@ -73,7 +75,9 @@ public void testReadSecuredSettings() { private AzureRepositoryPlugin pluginWithSettingsValidation(Settings settings) { final AzureRepositoryPlugin plugin = new AzureRepositoryPlugin(settings); new SettingsModule(settings, plugin.getSettings(), Collections.emptyList()); - plugin.createComponents(null, null, threadPool, null, null, null, null, null, null, null, null, TelemetryProvider.NOOP, null, null); + Plugin.PluginServices services = mock(Plugin.PluginServices.class); + when(services.threadPool()).thenReturn(threadPool); + plugin.createComponents(services); return plugin; } diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java index 296dd81b14ce0..c0b64c5c672f6 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java @@ -32,6 +32,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.support.RefCountingListener; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.Strings; import org.elasticsearch.common.blobstore.BlobContainer; @@ -65,7 +66,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import java.util.stream.Collectors; @@ -592,8 +592,14 @@ void run(BytesReference expected, BytesReference updated, ActionListener { - if (isComplete.compareAndSet(false, true)) { - safeAbortMultipartUpload(uploadId); - } - }; + SubscribableListener - try ( - var listeners = new RefCountingListener( - ActionListener.runAfter( - listener.delegateFailure( - (delegate1, ignored) -> getRegister( - purpose, - rawKey, - delegate1.delegateFailure((delegate2, currentValue) -> ActionListener.completeWith(delegate2, () -> { - if (currentValue.isPresent() && currentValue.bytesReference().equals(expected)) { - completeMultipartUpload(uploadId, partETag); - isComplete.set(true); - } - return currentValue; - })) - ) - ), - doCleanup - ) - ) - ) { - if (currentUploads.size() > 1) { - // This is a small optimization to improve the liveness properties of this algorithm. - // - // When there are multiple competing updates, we order them by upload id and the first one tries to cancel the competing - // updates in order to make progress. To avoid liveness issues when the winner fails, the rest wait based on their - // upload_id-based position and try to make progress. - - var delayListener = listeners.acquire(); - final Runnable cancelConcurrentUpdates = () -> { - try { - cancelOtherUploads(uploadId, currentUploads, listeners); - } finally { - delayListener.onResponse(null); - } - }; - - if (uploadIndex > 0) { - threadPool.scheduleUnlessShuttingDown( - TimeValue.timeValueMillis( - uploadIndex * blobStore.getCompareAndExchangeAntiContentionDelay().millis() + Randomness.get().nextInt(50) - ), - blobStore.getSnapshotExecutor(), - cancelConcurrentUpdates - ); + // Step 3: Ensure all other uploads in currentUploads are complete (either successfully, aborted by us or by another upload) + + .newForked(l -> ensureOtherUploadsComplete(uploadId, uploadIndex, currentUploads, l)) + + // Step 4: Read the current register value. + + .andThen((l, ignored) -> getRegister(purpose, rawKey, l)) + + // Step 5: Perform the compare-and-swap by completing our upload iff the witnessed value matches the expected value. + + .andThen((l, currentValue) -> ActionListener.completeWith(l, () -> { + if (currentValue.isPresent() && currentValue.bytesReference().equals(expected)) { + completeMultipartUpload(uploadId, partETag); } else { - cancelConcurrentUpdates.run(); + // Best-effort attempt to clean up after ourselves. + safeAbortMultipartUpload(uploadId); } - } - } + return currentValue; + })) + + // Step 6: Complete the listener. + + .addListener(listener.delegateResponse((l, e) -> { + // Best-effort attempt to clean up after ourselves. + safeAbortMultipartUpload(uploadId); + l.onFailure(e); + })); + + // No compare-and-exchange operations that started before ours can write to the register (in its step 5) after we have read the + // current value of the register (in our step 4) because we have ensured all earlier operations have completed (in our step 3). + // Conversely, if some other compare-and-exchange operation started after us then it will not read the register (in its step 4) + // until it has ensured we will not do a future write to the register (in our step 5) by cancelling all the racing uploads that + // it observed (in its step 3). Thus steps 4 and 5 can only complete successfully with no intervening writes to the register. } /** @@ -756,12 +741,48 @@ private int getUploadIndex(String targetUploadId, List multipar return found ? uploadIndex : -1; } - private void cancelOtherUploads(String uploadId, List currentUploads, RefCountingListener listeners) { - for (final var currentUpload : currentUploads) { - final var currentUploadId = currentUpload.getUploadId(); - if (uploadId.equals(currentUploadId) == false) { - blobStore.getSnapshotExecutor() - .execute(ActionRunnable.run(listeners.acquire(), () -> abortMultipartUploadIfExists(currentUploadId))); + private void ensureOtherUploadsComplete( + String uploadId, + int uploadIndex, + List currentUploads, + ActionListener listener + ) { + // This is a small optimization to improve the liveness properties of this algorithm. + // + // When there are updates racing to complete, we try and let them complete in order of their upload IDs. The one with the first + // upload ID immediately tries to cancel the competing updates in order to make progress, but the ones with greater upload IDs + // wait based on their position in the list before proceeding. + // + // Note that this does not guarantee that any of the uploads actually succeeds. Another operation could start and see a + // different collection of racing uploads and cancel all of them while they're sleeping. In theory this whole thing is provably + // impossible anyway [1] but in practice it'll eventually work with sufficient retries. + // + // [1] Michael J. Fischer, Nancy A. Lynch, and Michael S. Paterson. 1985. Impossibility of distributed consensus with one faulty + // process. J. ACM 32, 2 (April 1985), 374–382. + // + // TODO should we sort these by initiation time (and then upload ID as a tiebreaker)? + // TODO should we listMultipartUploads() while waiting, so we can fail quicker if we are concurrently cancelled? + if (uploadIndex > 0) { + threadPool.scheduleUnlessShuttingDown( + TimeValue.timeValueMillis( + uploadIndex * blobStore.getCompareAndExchangeAntiContentionDelay().millis() + Randomness.get().nextInt(50) + ), + blobStore.getSnapshotExecutor(), + ActionRunnable.wrap(listener, l -> cancelOtherUploads(uploadId, currentUploads, l)) + ); + } else { + cancelOtherUploads(uploadId, currentUploads, listener); + } + } + + private void cancelOtherUploads(String uploadId, List currentUploads, ActionListener listener) { + final var executor = blobStore.getSnapshotExecutor(); + try (var listeners = new RefCountingListener(listener)) { + for (final var currentUpload : currentUploads) { + final var currentUploadId = currentUpload.getUploadId(); + if (uploadId.equals(currentUploadId) == false) { + executor.execute(ActionRunnable.run(listeners.acquire(), () -> abortMultipartUploadIfExists(currentUploadId))); + } } } } diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java index c194fe67d106e..97c065e771ffd 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java @@ -12,29 +12,18 @@ import org.apache.lucene.util.SetOnce; import org.elasticsearch.SpecialPermission; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.RepositoryMetadata; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ReloadablePlugin; import org.elasticsearch.plugins.RepositoryPlugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.telemetry.metric.MeterRegistry; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.NamedXContentRegistry; import java.io.IOException; @@ -45,7 +34,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.Supplier; /** * A plugin to add a repository type that writes to and from the AWS S3. @@ -92,25 +80,10 @@ protected S3Repository createRepository( } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - service.set(s3Service(environment, clusterService.getSettings())); + public Collection createComponents(PluginServices services) { + service.set(s3Service(services.environment(), services.clusterService().getSettings())); this.service.get().refreshAndClearCache(S3ClientSettings.load(settings)); - meterRegistry.set(telemetryProvider.getMeterRegistry()); + meterRegistry.set(services.telemetryProvider().getMeterRegistry()); return List.of(service); } diff --git a/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java b/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java index 8057684375d69..fe33051df342e 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java @@ -9,28 +9,17 @@ package org.elasticsearch.plugin.repository.url; import org.apache.lucene.util.SetOnce; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.blobstore.url.http.URLHttpClient; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.IOUtils; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.RepositoryPlugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.url.URLRepository; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.NamedXContentRegistry; import java.io.IOException; @@ -39,7 +28,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.Supplier; public class URLRepositoryPlugin extends Plugin implements RepositoryPlugin { private final SetOnce httpClientFactory = new SetOnce<>(); @@ -76,22 +64,7 @@ public Map getRepositories( } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { final URLHttpClient.Factory apacheURLHttpClientFactory = new URLHttpClient.Factory(); diff --git a/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsCommonPlugin.java b/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsCommonPlugin.java index 6cad1e057ef74..b0adda3b2062b 100644 --- a/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsCommonPlugin.java +++ b/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsCommonPlugin.java @@ -8,28 +8,13 @@ package org.elasticsearch.runtimefields; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.Collection; import java.util.List; -import java.util.function.Supplier; /** * The plugin class for all the runtime fields common functionality that requires large dependencies. @@ -64,23 +49,8 @@ public List> getSettings() { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - grokHelper.finishInitializing(threadPool); + public Collection createComponents(PluginServices services) { + grokHelper.finishInitializing(services.threadPool()); return List.of(); } diff --git a/modules/systemd/src/main/java/org/elasticsearch/systemd/SystemdPlugin.java b/modules/systemd/src/main/java/org/elasticsearch/systemd/SystemdPlugin.java index f425de279129b..e3dca57472ade 100644 --- a/modules/systemd/src/main/java/org/elasticsearch/systemd/SystemdPlugin.java +++ b/modules/systemd/src/main/java/org/elasticsearch/systemd/SystemdPlugin.java @@ -12,29 +12,14 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.util.SetOnce; import org.elasticsearch.Build; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.ClusterPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.threadpool.Scheduler; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.Collection; import java.util.List; -import java.util.function.Supplier; public class SystemdPlugin extends Plugin implements ClusterPlugin { @@ -80,22 +65,7 @@ Scheduler.Cancellable extender() { } @Override - public Collection createComponents( - final Client client, - final ClusterService clusterService, - final ThreadPool threadPool, - final ResourceWatcherService resourceWatcherService, - final ScriptService scriptService, - final NamedXContentRegistry xContentRegistry, - final Environment environment, - final NodeEnvironment nodeEnvironment, - final NamedWriteableRegistry namedWriteableRegistry, - final IndexNameExpressionResolver expressionResolver, - final Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { if (enabled == false) { extender.set(null); return List.of(); @@ -107,7 +77,7 @@ public Collection createComponents( * Therefore, every fifteen seconds we send systemd a message via sd_notify to extend the timeout by thirty seconds. We will cancel * this scheduled task after we successfully notify systemd that we are ready. */ - extender.set(threadPool.scheduleWithFixedDelay(() -> { + extender.set(services.threadPool().scheduleWithFixedDelay(() -> { final int rc = sd_notify(0, "EXTEND_TIMEOUT_USEC=30000000"); if (rc < 0) { logger.warn("extending startup timeout via sd_notify failed with [{}]", rc); diff --git a/modules/systemd/src/test/java/org/elasticsearch/systemd/SystemdPluginTests.java b/modules/systemd/src/test/java/org/elasticsearch/systemd/SystemdPluginTests.java index b2bfbdb976eb9..ed5fdb79c4ba6 100644 --- a/modules/systemd/src/test/java/org/elasticsearch/systemd/SystemdPluginTests.java +++ b/modules/systemd/src/test/java/org/elasticsearch/systemd/SystemdPluginTests.java @@ -12,7 +12,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.telemetry.TelemetryProvider; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.hamcrest.OptionalMatchers; import org.elasticsearch.threadpool.Scheduler; @@ -58,30 +58,36 @@ public class SystemdPluginTests extends ESTestCase { ).thenReturn(extender); } + private void startPlugin(SystemdPlugin plugin) { + Plugin.PluginServices services = mock(Plugin.PluginServices.class); + when(services.threadPool()).thenReturn(threadPool); + plugin.createComponents(services); + } + public void testIsEnabled() { final SystemdPlugin plugin = new SystemdPlugin(false, randomPackageBuildType, Boolean.TRUE.toString()); - plugin.createComponents(null, null, threadPool, null, null, null, null, null, null, null, null, TelemetryProvider.NOOP, null, null); + startPlugin(plugin); assertTrue(plugin.isEnabled()); assertNotNull(plugin.extender()); } public void testIsNotPackageDistribution() { final SystemdPlugin plugin = new SystemdPlugin(false, randomNonPackageBuildType, Boolean.TRUE.toString()); - plugin.createComponents(null, null, threadPool, null, null, null, null, null, null, null, null, TelemetryProvider.NOOP, null, null); + startPlugin(plugin); assertFalse(plugin.isEnabled()); assertNull(plugin.extender()); } public void testIsImplicitlyNotEnabled() { final SystemdPlugin plugin = new SystemdPlugin(false, randomPackageBuildType, null); - plugin.createComponents(null, null, threadPool, null, null, null, null, null, null, null, null, TelemetryProvider.NOOP, null, null); + startPlugin(plugin); assertFalse(plugin.isEnabled()); assertNull(plugin.extender()); } public void testIsExplicitlyNotEnabled() { final SystemdPlugin plugin = new SystemdPlugin(false, randomPackageBuildType, Boolean.FALSE.toString()); - plugin.createComponents(null, null, threadPool, null, null, null, null, null, null, null, null, TelemetryProvider.NOOP, null, null); + startPlugin(plugin); assertFalse(plugin.isEnabled()); assertNull(plugin.extender()); } @@ -169,7 +175,7 @@ int sd_notify(final int unset_environment, final String state) { } }; - plugin.createComponents(null, null, threadPool, null, null, null, null, null, null, null, null, TelemetryProvider.NOOP, null, null); + startPlugin(plugin); if (Boolean.TRUE.toString().equals(esSDNotify)) { assertNotNull(plugin.extender()); } else { diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle index b21f6224c9fc2..5107bb9051bd1 100644 --- a/plugins/discovery-ec2/build.gradle +++ b/plugins/discovery-ec2/build.gradle @@ -95,9 +95,9 @@ tasks.named("test").configure { // this is needed to manipulate com.amazonaws.sdk.ec2MetadataServiceEndpointOverride system property // it is better rather disable security manager at all with `systemProperty 'tests.security.manager', 'false'` if (BuildParams.inFipsJvm){ - systemProperty 'java.security.policy', "=file://${buildDir}/tmp/java.policy" + nonInputProperties.systemProperty 'java.security.policy', "=file://${buildDir}/tmp/java.policy" } else { - systemProperty 'java.security.policy', "file://${buildDir}/tmp/java.policy" + nonInputProperties.systemProperty 'java.security.policy', "file://${buildDir}/tmp/java.policy" } } diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 78809a170fbbd..5db01ed636995 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -281,7 +281,7 @@ tasks.withType(RestIntegTestTask).configureEach { testTask -> if (disabledIntegTestTaskNames.contains(name) == false) { nonInputProperties.systemProperty "test.krb5.principal.es", "elasticsearch@${realm}" nonInputProperties.systemProperty "test.krb5.principal.hdfs", "hdfs/hdfs.build.elastic.co@${realm}" - jvmArgs "-Djava.security.krb5.conf=${project.configurations.krb5Config.getSingleFile().getPath()}" + nonInputProperties.systemProperty "java.security.krb5.conf", "${project.configurations.krb5Config.getSingleFile().getPath()}" nonInputProperties.systemProperty( "test.krb5.keytab.hdfs", new File(project.configurations.krb5Keytabs.singleFile, "hdfs_hdfs.build.elastic.co.keytab").getPath() @@ -291,7 +291,7 @@ tasks.withType(RestIntegTestTask).configureEach { testTask -> testClusters.matching { it.name == testTask.name }.configureEach { if (testTask.name.contains("Secure")) { - systemProperty "java.security.krb5.conf", configurations.krb5Config.singleFile.getPath() + systemProperty "java.security.krb5.conf", { configurations.krb5Config.singleFile.getPath() }, IGNORE_VALUE extraConfigFile( "repository-hdfs/krb5.keytab", new File(project.configurations.krb5Keytabs.singleFile, "elasticsearch.keytab"), diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/HotThreadsIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/HotThreadsIT.java index 9c931e15eeee3..04c59e1ce9214 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/HotThreadsIT.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/HotThreadsIT.java @@ -17,8 +17,10 @@ public class HotThreadsIT extends ESRestTestCase { + private static final String BWC_NODES_VERSION = System.getProperty("tests.bwc_nodes_version"); + public void testHotThreads() throws Exception { - final IndexingIT.Nodes nodes = IndexingIT.buildNodeAndVersions(client()); + final MixedClusterTestNodes nodes = MixedClusterTestNodes.buildNodes(client(), BWC_NODES_VERSION); assumeFalse("no new node found", nodes.getNewNodes().isEmpty()); assumeFalse("no bwc node found", nodes.getBWCNodes().isEmpty()); assumeTrue( diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java index 75a55e6e69af1..aac4b6a020d4b 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java @@ -28,7 +28,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -40,7 +39,7 @@ import static org.hamcrest.Matchers.oneOf; public class IndexingIT extends ESRestTestCase { - protected static final String BWC_NODES_VERSION = System.getProperty("tests.bwc_nodes_version"); + private static final String BWC_NODES_VERSION = System.getProperty("tests.bwc_nodes_version"); private int indexDocs(String index, final int idStart, final int numDocs) throws IOException { for (int i = 0; i < numDocs; i++) { @@ -78,10 +77,10 @@ private int indexDocWithConcurrentUpdates(String index, final int docId, int nUp } public void testIndexVersionPropagation() throws Exception { - Nodes nodes = buildNodeAndVersions(); + MixedClusterTestNodes nodes = buildNodeAndVersions(); assumeFalse("new nodes is empty", nodes.getNewNodes().isEmpty()); logger.info("cluster discovered: {}", nodes.toString()); - final List bwcNamesList = nodes.getBWCNodes().stream().map(Node::nodeName).collect(Collectors.toList()); + final List bwcNamesList = nodes.getBWCNodes().stream().map(MixedClusterTestNode::nodeName).collect(Collectors.toList()); final String bwcNames = bwcNamesList.stream().collect(Collectors.joining(",")); Settings.Builder settings = Settings.builder() .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) @@ -94,7 +93,7 @@ public void testIndexVersionPropagation() throws Exception { try ( RestClient newNodeClient = buildClient( restClientSettings(), - nodes.getNewNodes().stream().map(Node::publishAddress).toArray(HttpHost[]::new) + nodes.getNewNodes().stream().map(MixedClusterTestNode::publishAddress).toArray(HttpHost[]::new) ) ) { @@ -168,10 +167,10 @@ public void testIndexVersionPropagation() throws Exception { } public void testSeqNoCheckpoints() throws Exception { - Nodes nodes = buildNodeAndVersions(); + MixedClusterTestNodes nodes = buildNodeAndVersions(); assumeFalse("new nodes is empty", nodes.getNewNodes().isEmpty()); logger.info("cluster discovered: {}", nodes.toString()); - final List bwcNamesList = nodes.getBWCNodes().stream().map(Node::nodeName).collect(Collectors.toList()); + final List bwcNamesList = nodes.getBWCNodes().stream().map(MixedClusterTestNode::nodeName).collect(Collectors.toList()); final String bwcNames = bwcNamesList.stream().collect(Collectors.joining(",")); Settings.Builder settings = Settings.builder() .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) @@ -183,7 +182,7 @@ public void testSeqNoCheckpoints() throws Exception { try ( RestClient newNodeClient = buildClient( restClientSettings(), - nodes.getNewNodes().stream().map(Node::publishAddress).toArray(HttpHost[]::new) + nodes.getNewNodes().stream().map(MixedClusterTestNode::publishAddress).toArray(HttpHost[]::new) ) ) { int numDocs = 0; @@ -228,14 +227,14 @@ public void testSeqNoCheckpoints() throws Exception { assertOK(client().performRequest(new Request("POST", index + "/_refresh"))); for (Shard shard : buildShards(index, nodes, newNodeClient)) { - assertCount(index, "_only_nodes:" + shard.node.nodeName, numDocs); + assertCount(index, "_only_nodes:" + shard.node.nodeName(), numDocs); } assertSeqNoOnShards(index, nodes, numDocs, newNodeClient); } } public void testUpdateSnapshotStatus() throws Exception { - Nodes nodes = buildNodeAndVersions(); + MixedClusterTestNodes nodes = buildNodeAndVersions(); assumeFalse("new nodes is empty", nodes.getNewNodes().isEmpty()); logger.info("cluster discovered: {}", nodes.toString()); @@ -256,7 +255,7 @@ public void testUpdateSnapshotStatus() throws Exception { assertOK(client().performRequest(request)); - String bwcNames = nodes.getBWCNodes().stream().map(Node::nodeName).collect(Collectors.joining(",")); + String bwcNames = nodes.getBWCNodes().stream().map(MixedClusterTestNode::nodeName).collect(Collectors.joining(",")); // Allocating shards on the BWC nodes to makes sure that taking snapshot happens on those nodes. Settings.Builder settings = Settings.builder() @@ -310,7 +309,7 @@ private static boolean syncedFlushRemoved() { } public void testSyncedFlushTransition() throws Exception { - Nodes nodes = buildNodeAndVersions(); + MixedClusterTestNodes nodes = buildNodeAndVersions(); assumeTrue( "bwc version is on 7.x (synced flush deprecated but not removed yet)", syncedFlushDeprecated() && syncedFlushRemoved() == false @@ -318,7 +317,7 @@ public void testSyncedFlushTransition() throws Exception { assumeFalse("no new node found", nodes.getNewNodes().isEmpty()); assumeFalse("no bwc node found", nodes.getBWCNodes().isEmpty()); // Allocate shards to new nodes then verify synced flush requests processed by old nodes/new nodes - String newNodes = nodes.getNewNodes().stream().map(Node::nodeName).collect(Collectors.joining(",")); + String newNodes = nodes.getNewNodes().stream().map(MixedClusterTestNode::nodeName).collect(Collectors.joining(",")); int numShards = randomIntBetween(1, 10); int numOfReplicas = randomIntBetween(0, nodes.getNewNodes().size() - 1); int totalShards = numShards * (numOfReplicas + 1); @@ -336,7 +335,7 @@ public void testSyncedFlushTransition() throws Exception { try ( RestClient oldNodeClient = buildClient( restClientSettings(), - nodes.getBWCNodes().stream().map(Node::publishAddress).toArray(HttpHost[]::new) + nodes.getBWCNodes().stream().map(MixedClusterTestNode::publishAddress).toArray(HttpHost[]::new) ) ) { Request request = new Request("POST", index + "/_flush/synced"); @@ -364,7 +363,7 @@ public void testSyncedFlushTransition() throws Exception { try ( RestClient newNodeClient = buildClient( restClientSettings(), - nodes.getNewNodes().stream().map(Node::publishAddress).toArray(HttpHost[]::new) + nodes.getNewNodes().stream().map(MixedClusterTestNode::publishAddress).toArray(HttpHost[]::new) ) ) { Request request = new Request("POST", index + "/_flush/synced"); @@ -393,11 +392,11 @@ public void testSyncedFlushTransition() throws Exception { } public void testFlushTransition() throws Exception { - Nodes nodes = buildNodeAndVersions(); + MixedClusterTestNodes nodes = buildNodeAndVersions(); assumeFalse("no new node found", nodes.getNewNodes().isEmpty()); assumeFalse("no bwc node found", nodes.getBWCNodes().isEmpty()); // Allocate shards to new nodes then verify flush requests processed by old nodes/new nodes - String newNodes = nodes.getNewNodes().stream().map(Node::nodeName).collect(Collectors.joining(",")); + String newNodes = nodes.getNewNodes().stream().map(MixedClusterTestNode::nodeName).collect(Collectors.joining(",")); int numShards = randomIntBetween(1, 10); int numOfReplicas = randomIntBetween(0, nodes.getNewNodes().size() - 1); int totalShards = numShards * (numOfReplicas + 1); @@ -415,7 +414,7 @@ public void testFlushTransition() throws Exception { try ( RestClient oldNodeClient = buildClient( restClientSettings(), - nodes.getBWCNodes().stream().map(Node::publishAddress).toArray(HttpHost[]::new) + nodes.getBWCNodes().stream().map(MixedClusterTestNode::publishAddress).toArray(HttpHost[]::new) ) ) { Request request = new Request("POST", index + "/_flush"); @@ -432,7 +431,7 @@ public void testFlushTransition() throws Exception { try ( RestClient newNodeClient = buildClient( restClientSettings(), - nodes.getNewNodes().stream().map(Node::publishAddress).toArray(HttpHost[]::new) + nodes.getNewNodes().stream().map(MixedClusterTestNode::publishAddress).toArray(HttpHost[]::new) ) ) { Request request = new Request("POST", index + "/_flush"); @@ -466,7 +465,7 @@ private void assertVersion(final String index, final int docId, final String pre assertThat("version mismatch for doc [" + docId + "] preference [" + preference + "]", actualVersion, equalTo(expectedVersion)); } - private void assertSeqNoOnShards(String index, Nodes nodes, int numDocs, RestClient client) throws Exception { + private void assertSeqNoOnShards(String index, MixedClusterTestNodes nodes, int numDocs, RestClient client) throws Exception { assertBusy(() -> { try { List shards = buildShards(index, nodes, client); @@ -496,7 +495,7 @@ private void assertSeqNoOnShards(String index, Nodes nodes, int numDocs, RestCli }); } - private List buildShards(String index, Nodes nodes, RestClient client) throws IOException { + private List buildShards(String index, MixedClusterTestNodes nodes, RestClient client) throws IOException { Request request = new Request("GET", index + "/_stats"); request.addParameter("level", "shards"); Response response = client.performRequest(request); @@ -505,7 +504,7 @@ private List buildShards(String index, Nodes nodes, RestClient client) th for (Object shard : shardStats) { final String nodeId = ObjectPath.evaluate(shard, "routing.node"); final Boolean primary = ObjectPath.evaluate(shard, "routing.primary"); - final Node node = nodes.getSafe(nodeId); + final MixedClusterTestNode node = nodes.getSafe(nodeId); final SeqNoStats seqNoStats; Integer maxSeqNo = ObjectPath.evaluate(shard, "seq_no.max_seq_no"); Integer localCheckpoint = ObjectPath.evaluate(shard, "seq_no.local_checkpoint"); @@ -517,82 +516,9 @@ private List buildShards(String index, Nodes nodes, RestClient client) th return shards; } - private Nodes buildNodeAndVersions() throws IOException { - return buildNodeAndVersions(client()); + private MixedClusterTestNodes buildNodeAndVersions() throws IOException { + return MixedClusterTestNodes.buildNodes(client(), BWC_NODES_VERSION); } - static Nodes buildNodeAndVersions(RestClient client) throws IOException { - Response response = client.performRequest(new Request("GET", "_nodes")); - ObjectPath objectPath = ObjectPath.createFromResponse(response); - Map nodesAsMap = objectPath.evaluate("nodes"); - Nodes nodes = new Nodes(BWC_NODES_VERSION); - for (String id : nodesAsMap.keySet()) { - nodes.add( - new Node( - id, - objectPath.evaluate("nodes." + id + ".name"), - objectPath.evaluate("nodes." + id + ".version"), - HttpHost.create(objectPath.evaluate("nodes." + id + ".http.publish_address")) - ) - ); - } - response = client.performRequest(new Request("GET", "_cluster/state")); - nodes.setMasterNodeId(ObjectPath.createFromResponse(response).evaluate("master_node")); - return nodes; - } - - static final class Nodes extends HashMap { - - private final String bwcNodesVersion; - private String masterNodeId = null; - - Nodes(String bwcNodesVersion) { - this.bwcNodesVersion = bwcNodesVersion; - } - - public Node getMaster() { - return get(masterNodeId); - } - - public void setMasterNodeId(String id) { - if (get(id) == null) { - throw new IllegalArgumentException("node with id [" + id + "] not found. got:" + toString()); - } - masterNodeId = id; - } - - public void add(Node node) { - put(node.id(), node); - } - - public List getNewNodes() { - return values().stream().filter(n -> n.version().equals(bwcNodesVersion) == false).collect(Collectors.toList()); - } - - public List getBWCNodes() { - return values().stream().filter(n -> n.version().equals(bwcNodesVersion)).collect(Collectors.toList()); - } - - public Node getSafe(String id) { - Node node = get(id); - if (node == null) { - throw new IllegalArgumentException("node with id [" + id + "] not found"); - } - return node; - } - - @Override - public String toString() { - return "Nodes{" - + "masterNodeId='" - + masterNodeId - + "'\n" - + values().stream().map(Node::toString).collect(Collectors.joining("\n")) - + '}'; - } - } - - record Node(String id, String nodeName, String version, HttpHost publishAddress) {} - - record Shard(Node node, boolean primary, SeqNoStats seqNoStats) {} + private record Shard(MixedClusterTestNode node, boolean primary, SeqNoStats seqNoStats) {} } diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterTestNode.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterTestNode.java new file mode 100644 index 0000000000000..c9cdf9dd41735 --- /dev/null +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterTestNode.java @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.backwards; + +import org.apache.http.HttpHost; + +record MixedClusterTestNode(String id, String nodeName, String version, HttpHost publishAddress) {} diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterTestNodes.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterTestNodes.java new file mode 100644 index 0000000000000..d4bfea19fd1ff --- /dev/null +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterTestNodes.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.backwards; + +import org.apache.http.HttpHost; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.test.rest.ObjectPath; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +final class MixedClusterTestNodes { + private final Map nodesById; + private final String bwcNodesVersion; + + private MixedClusterTestNodes(String bwcNodesVersion, Map nodesById) { + this.bwcNodesVersion = bwcNodesVersion; + this.nodesById = nodesById; + } + + public List getNewNodes() { + return nodesById.values().stream().filter(n -> n.version().equals(bwcNodesVersion) == false).collect(Collectors.toList()); + } + + public List getBWCNodes() { + return nodesById.values().stream().filter(n -> n.version().equals(bwcNodesVersion)).collect(Collectors.toList()); + } + + public MixedClusterTestNode getSafe(String id) { + MixedClusterTestNode node = nodesById.get(id); + if (node == null) { + throw new IllegalArgumentException("node with id [" + id + "] not found"); + } + return node; + } + + static MixedClusterTestNodes buildNodes(RestClient client, String bwcNodesVersion) throws IOException { + Response response = client.performRequest(new Request("GET", "_nodes")); + ObjectPath objectPath = ObjectPath.createFromResponse(response); + Map nodesAsMap = objectPath.evaluate("nodes"); + + Map nodesById = new HashMap<>(); + for (var id : nodesAsMap.keySet()) { + nodesById.put( + id, + new MixedClusterTestNode( + id, + objectPath.evaluate("nodes." + id + ".name"), + objectPath.evaluate("nodes." + id + ".version"), + HttpHost.create(objectPath.evaluate("nodes." + id + ".http.publish_address")) + ) + ); + } + return new MixedClusterTestNodes(bwcNodesVersion, Collections.unmodifiableMap(nodesById)); + } + + public int size() { + return nodesById.size(); + } +} diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java index f28e014837dda..461b731e518fb 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java @@ -8,8 +8,6 @@ package org.elasticsearch.backwards; import org.apache.http.HttpHost; -import org.elasticsearch.backwards.IndexingIT.Node; -import org.elasticsearch.backwards.IndexingIT.Nodes; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; @@ -32,19 +30,19 @@ public class SearchWithMinCompatibleSearchNodeIT extends ESRestTestCase { - protected static final String BWC_NODES_VERSION = System.getProperty("tests.bwc_nodes_version"); - protected static final String NEW_NODES_VERSION = System.getProperty("tests.new_nodes_version"); + private static final String BWC_NODES_VERSION = System.getProperty("tests.bwc_nodes_version"); + private static final String NEW_NODES_VERSION = System.getProperty("tests.new_nodes_version"); private static String index = "test_min_version"; private static int numShards; private static int numReplicas = 1; private static int numDocs; - private static Nodes nodes; - private static List allNodes; + private static MixedClusterTestNodes nodes; + private static List allNodes; @Before public void prepareTestData() throws IOException { - nodes = IndexingIT.buildNodeAndVersions(client()); + nodes = MixedClusterTestNodes.buildNodes(client(), BWC_NODES_VERSION); numShards = nodes.size(); numDocs = randomIntBetween(numShards, 16); allNodes = new ArrayList<>(); @@ -69,7 +67,12 @@ public void prepareTestData() throws IOException { } public void testMinVersionAsNewVersion() throws Exception { - try (RestClient client = buildClient(restClientSettings(), allNodes.stream().map(Node::publishAddress).toArray(HttpHost[]::new))) { + try ( + RestClient client = buildClient( + restClientSettings(), + allNodes.stream().map(MixedClusterTestNode::publishAddress).toArray(HttpHost[]::new) + ) + ) { Request newVersionRequest = new Request( "POST", index + "/_search?min_compatible_shard_node=" + NEW_NODES_VERSION + "&ccs_minimize_roundtrips=false" @@ -90,7 +93,12 @@ public void testMinVersionAsNewVersion() throws Exception { } public void testMinVersionAsOldVersion() throws Exception { - try (RestClient client = buildClient(restClientSettings(), allNodes.stream().map(Node::publishAddress).toArray(HttpHost[]::new))) { + try ( + RestClient client = buildClient( + restClientSettings(), + allNodes.stream().map(MixedClusterTestNode::publishAddress).toArray(HttpHost[]::new) + ) + ) { Request oldVersionRequest = new Request( "POST", index + "/_search?min_compatible_shard_node=" + BWC_NODES_VERSION + "&ccs_minimize_roundtrips=false" @@ -112,7 +120,12 @@ public void testMinVersionAsOldVersion() throws Exception { } public void testCcsMinimizeRoundtripsIsFalse() throws Exception { - try (RestClient client = buildClient(restClientSettings(), allNodes.stream().map(Node::publishAddress).toArray(HttpHost[]::new))) { + try ( + RestClient client = buildClient( + restClientSettings(), + allNodes.stream().map(MixedClusterTestNode::publishAddress).toArray(HttpHost[]::new) + ) + ) { String version = randomBoolean() ? NEW_NODES_VERSION : BWC_NODES_VERSION; Request request = new Request( diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_dense_vector_dynamic_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_dense_vector_dynamic_mapping.yml index ffc7f749b8fd3..151698482368a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_dense_vector_dynamic_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_dense_vector_dynamic_mapping.yml @@ -3,34 +3,6 @@ setup: version: ' - 8.10.99' reason: 'Dynamic mapping of floats to dense_vector was added in 8.11' - # Additional logging for issue: https://github.com/elastic/elasticsearch/issues/100502 - - do: - cluster.put_settings: - body: > - { - "persistent": { - "logger.org.elasticsearch.index": "TRACE", - "logger.org.elasticsearch.action.admin.indices.mapping.get": "TRACE", - "logger.org.elasticsearch.cluster.service.ClusterApplierService": "TRACE" - } - } - ---- -teardown: - - skip: - version: ' - 8.10.99' - reason: 'Dynamic mapping of floats to dense_vector was added in 8.11' - - - do: - cluster.put_settings: - body: > - { - "persistent": { - "logger.org.elasticsearch.index": null, - "logger.org.elasticsearch.action.admin.indices.mapping.get": null, - "logger.org.elasticsearch.cluster.service.ClusterApplierService": null - } - } --- "Fields with float arrays below the threshold still map as float": diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java index fe24ed320d057..f4457a7db8b7c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java @@ -15,11 +15,9 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; @@ -28,20 +26,11 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.ClusterPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -55,7 +44,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Supplier; import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -402,22 +390,9 @@ public List getNamedWriteables() { private final AtomicBoolean installed = new AtomicBoolean(); @Override - public Collection createComponents( - final Client client, - final ClusterService clusterService, - final ThreadPool threadPool, - final ResourceWatcherService resourceWatcherService, - final ScriptService scriptService, - final NamedXContentRegistry xContentRegistry, - final Environment environment, - final NodeEnvironment nodeEnvironment, - final NamedWriteableRegistry namedWriteableRegistry, - final IndexNameExpressionResolver expressionResolver, - final Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { + ClusterService clusterService = services.clusterService(); + clusterService.addListener(event -> { final ClusterState state = event.state(); if (state.getBlocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK)) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java index d599b0649e829..8bbdaf7388699 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java @@ -10,30 +10,16 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; import java.util.function.UnaryOperator; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -65,41 +51,11 @@ public TestPlugin(Settings settings) { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - clusterService.getClusterSettings().addSettingsUpdateConsumer(UPDATE_TEMPLATE_DUMMY_SETTING, integer -> { + public Collection createComponents(PluginServices services) { + services.clusterService().getClusterSettings().addSettingsUpdateConsumer(UPDATE_TEMPLATE_DUMMY_SETTING, integer -> { logger.debug("the template dummy setting was updated to {}", integer); }); - return super.createComponents( - client, - clusterService, - threadPool, - resourceWatcherService, - scriptService, - xContentRegistry, - environment, - nodeEnvironment, - namedWriteableRegistry, - expressionResolver, - repositoriesServiceSupplier, - telemetryProvider, - allocationService, - indicesService - ); + return super.createComponents(services); } @Override diff --git a/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthActionIT.java index 11d32bb231a01..a31c3a08b8a4f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthActionIT.java @@ -11,27 +11,15 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.metrics.Counters; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.health.node.HealthInfo; import org.elasticsearch.health.stats.HealthApiStatsAction; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.HealthPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.ArrayList; import java.util.Collection; @@ -40,7 +28,6 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.concurrent.ExecutionException; -import java.util.function.Supplier; import java.util.stream.Stream; import static org.elasticsearch.common.util.CollectionUtils.appendToCopy; @@ -95,25 +82,10 @@ public List> getSettings() { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - healthIndicatorServices.add(new IlmHealthIndicatorService(clusterService)); - healthIndicatorServices.add(new SlmHealthIndicatorService(clusterService)); - healthIndicatorServices.add(new ClusterCoordinationHealthIndicatorService(clusterService)); + public Collection createComponents(PluginServices services) { + healthIndicatorServices.add(new IlmHealthIndicatorService(services.clusterService())); + healthIndicatorServices.add(new SlmHealthIndicatorService(services.clusterService())); + healthIndicatorServices.add(new ClusterCoordinationHealthIndicatorService(services.clusterService())); return new ArrayList<>(healthIndicatorServices); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/health/HealthServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/health/HealthServiceIT.java index 707644e28228a..2e741d6691d24 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/health/HealthServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/health/HealthServiceIT.java @@ -9,37 +9,22 @@ package org.elasticsearch.health; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.health.node.DiskHealthInfo; import org.elasticsearch.health.node.FetchHealthInfoCacheAction; import org.elasticsearch.health.node.HealthInfo; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.HealthPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Supplier; import static org.elasticsearch.common.util.CollectionUtils.appendToCopy; import static org.hamcrest.Matchers.equalTo; @@ -124,22 +109,7 @@ public static final class TestHealthPlugin extends Plugin implements HealthPlugi private final List healthIndicatorServices = new ArrayList<>(); @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { healthIndicatorServices.add(new TestHealthIndicatorService()); return new ArrayList<>(healthIndicatorServices); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java index ab4706c865be9..24372978834c6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java @@ -17,17 +17,9 @@ import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -35,14 +27,8 @@ import org.elasticsearch.ingest.Processor; import org.elasticsearch.plugins.IngestPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentType; import org.junit.After; @@ -52,7 +38,6 @@ import java.util.Map; import java.util.Objects; import java.util.function.BiConsumer; -import java.util.function.Supplier; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; @@ -392,26 +377,6 @@ public void testHighOrderFinalPipelinePreferred() throws IOException { public static class TestPlugin extends Plugin implements IngestPlugin { - @Override - public Collection createComponents( - final Client client, - final ClusterService clusterService, - final ThreadPool threadPool, - final ResourceWatcherService resourceWatcherService, - final ScriptService scriptService, - final NamedXContentRegistry xContentRegistry, - final Environment environment, - final NodeEnvironment nodeEnvironment, - final NamedWriteableRegistry namedWriteableRegistry, - final IndexNameExpressionResolver expressionResolver, - final Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - return List.of(); - } - @Override public Map getProcessors(Processor.Parameters parameters) { return Map.of( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/SettingsListenerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/SettingsListenerIT.java index fb0e3478c2cde..de783a28bce1d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/SettingsListenerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/SettingsListenerIT.java @@ -7,33 +7,18 @@ */ package org.elasticsearch.index; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.function.Supplier; import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -63,22 +48,7 @@ public void onIndexModule(IndexModule module) { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { return Collections.singletonList(service); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java index f8d2b0e464b72..fa4d4c0fbb669 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java @@ -13,32 +13,18 @@ import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.ingest.PutPipelineRequest; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.IngestPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentType; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.function.BiConsumer; -import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; @@ -85,23 +71,8 @@ public static class TestPlugin extends Plugin implements IngestPlugin { private ThreadPool threadPool; @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - this.threadPool = threadPool; + public Collection createComponents(PluginServices services) { + this.threadPool = services.threadPool(); return List.of(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java index 049a779c97503..90a295e5a25f2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java @@ -26,7 +26,6 @@ import org.elasticsearch.index.mapper.BlockLoader.DoubleBuilder; import org.elasticsearch.index.mapper.BlockLoader.IntBuilder; import org.elasticsearch.index.mapper.BlockLoader.LongBuilder; -import org.elasticsearch.index.mapper.BlockLoader.SingletonOrdinalsBuilder; import java.io.IOException; @@ -61,7 +60,7 @@ public BlockDocValuesReader() { /** * Reads the values of the given documents specified in the input block */ - public abstract Builder readValues(BuilderFactory factory, Docs docs) throws IOException; + public abstract BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException; /** * Reads the values of the given document into the builder @@ -189,27 +188,28 @@ private static class SingletonLongs extends BlockDocValuesReader { } @Override - public LongBuilder builder(BuilderFactory factory, int expectedCount) { + public BlockLoader.LongBuilder builder(BuilderFactory factory, int expectedCount) { return factory.longsFromDocValues(expectedCount); } @Override - public LongBuilder readValues(BuilderFactory factory, Docs docs) throws IOException { - var blockBuilder = builder(factory, docs.count()); - int lastDoc = -1; - for (int i = 0; i < docs.count(); i++) { - int doc = docs.get(i); - if (doc < lastDoc) { - throw new IllegalStateException("docs within same block must be in order"); - } - if (numericDocValues.advanceExact(doc)) { - blockBuilder.appendLong(numericDocValues.longValue()); - } else { - blockBuilder.appendNull(); + public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { + try (BlockLoader.LongBuilder builder = builder(factory, docs.count())) { + int lastDoc = -1; + for (int i = 0; i < docs.count(); i++) { + int doc = docs.get(i); + if (doc < lastDoc) { + throw new IllegalStateException("docs within same block must be in order"); + } + if (numericDocValues.advanceExact(doc)) { + builder.appendLong(numericDocValues.longValue()); + } else { + builder.appendNull(); + } + lastDoc = doc; } - lastDoc = doc; + return builder.build(); } - return blockBuilder; } @Override @@ -247,16 +247,17 @@ public BlockLoader.LongBuilder builder(BuilderFactory factory, int expectedCount } @Override - public BlockLoader.LongBuilder readValues(BuilderFactory factory, Docs docs) throws IOException { - var blockBuilder = builder(factory, docs.count()); - for (int i = 0; i < docs.count(); i++) { - int doc = docs.get(i); - if (doc < this.docID) { - throw new IllegalStateException("docs within same block must be in order"); + public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { + try (BlockLoader.LongBuilder builder = builder(factory, docs.count())) { + for (int i = 0; i < docs.count(); i++) { + int doc = docs.get(i); + if (doc < this.docID) { + throw new IllegalStateException("docs within same block must be in order"); + } + read(doc, builder); } - read(doc, blockBuilder); + return builder.build(); } - return blockBuilder; } @Override @@ -307,22 +308,23 @@ public IntBuilder builder(BuilderFactory factory, int expectedCount) { } @Override - public IntBuilder readValues(BuilderFactory factory, Docs docs) throws IOException { - var blockBuilder = builder(factory, docs.count()); - int lastDoc = -1; - for (int i = 0; i < docs.count(); i++) { - int doc = docs.get(i); - if (doc < lastDoc) { - throw new IllegalStateException("docs within same block must be in order"); - } - if (numericDocValues.advanceExact(doc)) { - blockBuilder.appendInt(Math.toIntExact(numericDocValues.longValue())); - } else { - blockBuilder.appendNull(); + public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { + try (BlockLoader.IntBuilder builder = builder(factory, docs.count())) { + int lastDoc = -1; + for (int i = 0; i < docs.count(); i++) { + int doc = docs.get(i); + if (doc < lastDoc) { + throw new IllegalStateException("docs within same block must be in order"); + } + if (numericDocValues.advanceExact(doc)) { + builder.appendInt(Math.toIntExact(numericDocValues.longValue())); + } else { + builder.appendNull(); + } + lastDoc = doc; } - lastDoc = doc; + return builder.build(); } - return blockBuilder; } @Override @@ -360,16 +362,17 @@ public IntBuilder builder(BuilderFactory factory, int expectedCount) { } @Override - public IntBuilder readValues(BuilderFactory factory, Docs docs) throws IOException { - var blockBuilder = builder(factory, docs.count()); - for (int i = 0; i < docs.count(); i++) { - int doc = docs.get(i); - if (doc < this.docID) { - throw new IllegalStateException("docs within same block must be in order"); + public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { + try (BlockLoader.IntBuilder builder = builder(factory, docs.count())) { + for (int i = 0; i < docs.count(); i++) { + int doc = docs.get(i); + if (doc < this.docID) { + throw new IllegalStateException("docs within same block must be in order"); + } + read(doc, builder); } - read(doc, blockBuilder); + return builder.build(); } - return blockBuilder; } @Override @@ -423,23 +426,24 @@ public DoubleBuilder builder(BuilderFactory factory, int expectedCount) { } @Override - public DoubleBuilder readValues(BuilderFactory factory, Docs docs) throws IOException { - var blockBuilder = builder(factory, docs.count()); - int lastDoc = -1; - for (int i = 0; i < docs.count(); i++) { - int doc = docs.get(i); - if (doc < lastDoc) { - throw new IllegalStateException("docs within same block must be in order"); + public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { + try (BlockLoader.DoubleBuilder builder = builder(factory, docs.count())) { + int lastDoc = -1; + for (int i = 0; i < docs.count(); i++) { + int doc = docs.get(i); + if (doc < lastDoc) { + throw new IllegalStateException("docs within same block must be in order"); + } + if (docValues.advanceExact(doc)) { + builder.appendDouble(toDouble.convert(docValues.longValue())); + } else { + builder.appendNull(); + } + lastDoc = doc; + this.docID = doc; } - if (docValues.advanceExact(doc)) { - blockBuilder.appendDouble(toDouble.convert(docValues.longValue())); - } else { - blockBuilder.appendNull(); - } - lastDoc = doc; - this.docID = doc; + return builder.build(); } - return blockBuilder; } @Override @@ -480,16 +484,17 @@ public DoubleBuilder builder(BuilderFactory factory, int expectedCount) { } @Override - public DoubleBuilder readValues(BuilderFactory factory, Docs docs) throws IOException { - var blockBuilder = builder(factory, docs.count()); - for (int i = 0; i < docs.count(); i++) { - int doc = docs.get(i); - if (doc < this.docID) { - throw new IllegalStateException("docs within same block must be in order"); + public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { + try (BlockLoader.DoubleBuilder builder = builder(factory, docs.count())) { + for (int i = 0; i < docs.count(); i++) { + int doc = docs.get(i); + if (doc < this.docID) { + throw new IllegalStateException("docs within same block must be in order"); + } + read(doc, builder); } - read(doc, blockBuilder); + return builder.build(); } - return blockBuilder; } @Override @@ -539,21 +544,21 @@ public BytesRefBuilder builder(BuilderFactory factory, int expectedCount) { } @Override - public SingletonOrdinalsBuilder readValues(BuilderFactory factory, Docs docs) throws IOException { - SingletonOrdinalsBuilder builder = factory.singletonOrdinalsBuilder(ordinals, docs.count()); - - for (int i = 0; i < docs.count(); i++) { - int doc = docs.get(i); - if (doc < ordinals.docID()) { - throw new IllegalStateException("docs within same block must be in order"); - } - if (ordinals.advanceExact(doc)) { - builder.appendOrd(ordinals.ordValue()); - } else { - builder.appendNull(); + public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { + try (BlockLoader.SingletonOrdinalsBuilder builder = factory.singletonOrdinalsBuilder(ordinals, docs.count())) { + for (int i = 0; i < docs.count(); i++) { + int doc = docs.get(i); + if (doc < ordinals.docID()) { + throw new IllegalStateException("docs within same block must be in order"); + } + if (ordinals.advanceExact(doc)) { + builder.appendOrd(ordinals.ordValue()); + } else { + builder.appendNull(); + } } + return builder.build(); } - return builder; } @Override @@ -589,17 +594,17 @@ public BytesRefBuilder builder(BuilderFactory factory, int expectedCount) { } @Override - public BytesRefBuilder readValues(BuilderFactory factory, Docs docs) throws IOException { - BytesRefBuilder builder = builder(factory, docs.count()); - - for (int i = 0; i < docs.count(); i++) { - int doc = docs.get(i); - if (doc < ordinals.docID()) { - throw new IllegalStateException("docs within same block must be in order"); + public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { + try (BytesRefBuilder builder = builder(factory, docs.count())) { + for (int i = 0; i < docs.count(); i++) { + int doc = docs.get(i); + if (doc < ordinals.docID()) { + throw new IllegalStateException("docs within same block must be in order"); + } + read(doc, builder); } - read(doc, builder); + return builder.build(); } - return builder; } @Override @@ -649,16 +654,17 @@ public BytesRefBuilder builder(BuilderFactory factory, int expectedCount) { } @Override - public BytesRefBuilder readValues(BuilderFactory factory, Docs docs) throws IOException { - var blockBuilder = builder(factory, docs.count()); - for (int i = 0; i < docs.count(); i++) { - int doc = docs.get(i); - if (doc < docID) { - throw new IllegalStateException("docs within same block must be in order"); + public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { + try (BlockLoader.BytesRefBuilder builder = builder(factory, docs.count())) { + for (int i = 0; i < docs.count(); i++) { + int doc = docs.get(i); + if (doc < docID) { + throw new IllegalStateException("docs within same block must be in order"); + } + read(doc, builder); } - read(doc, blockBuilder); + return builder.build(); } - return blockBuilder; } @Override @@ -709,22 +715,23 @@ public BooleanBuilder builder(BuilderFactory factory, int expectedCount) { } @Override - public BooleanBuilder readValues(BuilderFactory factory, Docs docs) throws IOException { - var blockBuilder = builder(factory, docs.count()); - int lastDoc = -1; - for (int i = 0; i < docs.count(); i++) { - int doc = docs.get(i); - if (doc < lastDoc) { - throw new IllegalStateException("docs within same block must be in order"); - } - if (numericDocValues.advanceExact(doc)) { - blockBuilder.appendBoolean(numericDocValues.longValue() != 0); - } else { - blockBuilder.appendNull(); + public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { + try (BlockLoader.BooleanBuilder builder = builder(factory, docs.count())) { + int lastDoc = -1; + for (int i = 0; i < docs.count(); i++) { + int doc = docs.get(i); + if (doc < lastDoc) { + throw new IllegalStateException("docs within same block must be in order"); + } + if (numericDocValues.advanceExact(doc)) { + builder.appendBoolean(numericDocValues.longValue() != 0); + } else { + builder.appendNull(); + } + lastDoc = doc; } - lastDoc = doc; + return builder.build(); } - return blockBuilder; } @Override @@ -762,16 +769,17 @@ public BooleanBuilder builder(BuilderFactory factory, int expectedCount) { } @Override - public BooleanBuilder readValues(BuilderFactory factory, Docs docs) throws IOException { - var blockBuilder = builder(factory, docs.count()); - for (int i = 0; i < docs.count(); i++) { - int doc = docs.get(i); - if (doc < this.docID) { - throw new IllegalStateException("docs within same block must be in order"); + public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { + try (BlockLoader.BooleanBuilder builder = builder(factory, docs.count())) { + for (int i = 0; i < docs.count(); i++) { + int doc = docs.get(i); + if (doc < this.docID) { + throw new IllegalStateException("docs within same block must be in order"); + } + read(doc, builder); } - read(doc, blockBuilder); + return builder.build(); } - return blockBuilder; } @Override @@ -813,17 +821,18 @@ private static class Nulls extends BlockDocValuesReader { private int docID = -1; @Override - public Builder builder(BuilderFactory factory, int expectedCount) { + public BlockLoader.Builder builder(BuilderFactory factory, int expectedCount) { return factory.nulls(expectedCount); } @Override - public Builder readValues(BuilderFactory factory, Docs docs) throws IOException { - Builder builder = builder(factory, docs.count()); - for (int i = 0; i < docs.count(); i++) { - builder.appendNull(); + public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { + try (BlockLoader.Builder builder = builder(factory, docs.count())) { + for (int i = 0; i < docs.count(); i++) { + builder.appendNull(); + } + return builder.build(); } - return builder; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java index 7e973f9c32033..af53ab42d35d9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.core.Releasable; import java.io.IOException; @@ -124,13 +125,24 @@ interface BuilderFactory { // TODO support non-singleton ords } + /** + * Marker interface for block results. The compute engine has a fleshed + * out implementation. + */ + interface Block {} + /** * A builder for typed values. For each document you may either call * {@link #appendNull}, {@code append}, or * {@link #beginPositionEntry} followed by two or more {@code append} * calls, and then {@link #endPositionEntry}. */ - interface Builder { + interface Builder extends Releasable { + /** + * Build the actual block. + */ + Block build(); + /** * Insert a null value. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java index 2b9daadda31d6..1261a3612d3cb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java @@ -156,16 +156,17 @@ public String toString() { } @Override - public BlockLoader.Builder readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) throws IOException { - BlockLoader.Builder blockBuilder = builder(factory, docs.count()); - for (int i = 0; i < docs.count(); i++) { - int doc = docs.get(i); - if (doc < this.docID) { - throw new IllegalStateException("docs within same block must be in order"); - } - readValuesFromSingleDoc(doc, blockBuilder); + public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) throws IOException { + try (BlockLoader.Builder builder = builder(factory, docs.count())) { + for (int i = 0; i < docs.count(); i++) { + int doc = docs.get(i); + if (doc < this.docID) { + throw new IllegalStateException("docs within same block must be in order"); + } + readValuesFromSingleDoc(doc, builder); + } + return builder.build(); } - return blockBuilder; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java index d38d30a03b275..5984482fd9441 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java @@ -63,12 +63,13 @@ protected BlockStoredFieldsReader(LeafStoredFieldLoader loader) { } @Override - public final BlockLoader.Builder readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) throws IOException { - var builder = builder(factory, docs.count()); - for (int i = 0; i < docs.count(); i++) { - readValuesFromSingleDoc(docs.get(i), builder); + public final BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) throws IOException { + try (BlockLoader.Builder builder = builder(factory, docs.count())) { + for (int i = 0; i < docs.count(); i++) { + readValuesFromSingleDoc(docs.get(i), builder); + } + return builder.build(); } - return builder; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptBlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptBlockDocValuesReader.java index 0d29bc43700e8..b59df56791fbe 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptBlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptBlockDocValuesReader.java @@ -37,12 +37,13 @@ public BlockLoader.BooleanBuilder builder(BlockLoader.BuilderFactory factory, in } @Override - public BlockLoader.Builder readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - BlockLoader.BooleanBuilder builder = builder(factory, docs.count()); - for (int i = 0; i < docs.count(); i++) { - read(docs.get(i), builder); + public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { + try (BlockLoader.BooleanBuilder builder = builder(factory, docs.count())) { + for (int i = 0; i < docs.count(); i++) { + read(docs.get(i), builder); + } + return builder.build(); } - return builder; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptBlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptBlockDocValuesReader.java index 6e6cdd3d1f057..ad630a71870a4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptBlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptBlockDocValuesReader.java @@ -36,12 +36,13 @@ public BlockLoader.LongBuilder builder(BlockLoader.BuilderFactory factory, int e } @Override - public BlockLoader.Builder readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - BlockLoader.LongBuilder builder = builder(factory, docs.count()); - for (int i = 0; i < docs.count(); i++) { - read(docs.get(i), builder); + public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { + try (BlockLoader.LongBuilder builder = builder(factory, docs.count())) { + for (int i = 0; i < docs.count(); i++) { + read(docs.get(i), builder); + } + return builder.build(); } - return builder; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptBlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptBlockDocValuesReader.java index 856321f53244d..4e317a3ed11cb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptBlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptBlockDocValuesReader.java @@ -36,12 +36,13 @@ public BlockLoader.DoubleBuilder builder(BlockLoader.BuilderFactory factory, int } @Override - public BlockLoader.Builder readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - BlockLoader.DoubleBuilder builder = builder(factory, docs.count()); - for (int i = 0; i < docs.count(); i++) { - read(docs.get(i), builder); + public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { + try (BlockLoader.DoubleBuilder builder = builder(factory, docs.count())) { + for (int i = 0; i < docs.count(); i++) { + read(docs.get(i), builder); + } + return builder.build(); } - return builder; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index cb69fa4b7c50a..5f987fd96ca66 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -96,12 +96,13 @@ public BlockLoader.BytesRefBuilder builder(BlockLoader.BuilderFactory factory, i } @Override - public BlockLoader.Builder readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - BlockLoader.BytesRefBuilder builder = builder(factory, docs.count()); - for (int i = 0; i < docs.count(); i++) { - builder.appendBytesRef(bytes); + public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { + try (BlockLoader.BytesRefBuilder builder = builder(factory, docs.count())) { + for (int i = 0; i < docs.count(); i++) { + builder.appendBytesRef(bytes); + } + return builder.build(); } - return builder; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptBlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptBlockDocValuesReader.java index f05b9aff890af..23229a6533cdb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptBlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptBlockDocValuesReader.java @@ -36,12 +36,13 @@ public BlockLoader.BytesRefBuilder builder(BlockLoader.BuilderFactory factory, i } @Override - public BlockLoader.Builder readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - BlockLoader.BytesRefBuilder builder = builder(factory, docs.count()); - for (int i = 0; i < docs.count(); i++) { - read(docs.get(i), builder); + public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { + try (BlockLoader.BytesRefBuilder builder = builder(factory, docs.count())) { + for (int i = 0; i < docs.count(); i++) { + read(docs.get(i), builder); + } + return builder.build(); } - return builder; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptBlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptBlockDocValuesReader.java index 51058b3b60bf4..6afbcae50d31f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptBlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptBlockDocValuesReader.java @@ -38,12 +38,13 @@ public BlockLoader.BytesRefBuilder builder(BlockLoader.BuilderFactory factory, i } @Override - public BlockLoader.Builder readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - BlockLoader.BytesRefBuilder builder = builder(factory, docs.count()); - for (int i = 0; i < docs.count(); i++) { - read(docs.get(i), builder); + public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { + try (BlockLoader.BytesRefBuilder builder = builder(factory, docs.count())) { + for (int i = 0; i < docs.count(); i++) { + read(docs.get(i), builder); + } + return builder.build(); } - return builder; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LongScriptBlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/LongScriptBlockDocValuesReader.java index 4896f7d858144..91c099cd2813b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LongScriptBlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LongScriptBlockDocValuesReader.java @@ -36,12 +36,13 @@ public BlockLoader.LongBuilder builder(BlockLoader.BuilderFactory factory, int e } @Override - public BlockLoader.Builder readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - BlockLoader.LongBuilder builder = builder(factory, docs.count()); - for (int i = 0; i < docs.count(); i++) { - read(docs.get(i), builder); + public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { + try (BlockLoader.LongBuilder builder = builder(factory, docs.count())) { + for (int i = 0; i < docs.count(); i++) { + read(docs.get(i), builder); + } + return builder.build(); } - return builder; } @Override diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java index 2cdd383114497..a570c88ddaba7 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java @@ -336,8 +336,18 @@ public void onResponse(ActionResponse.Empty ignored) { @Override public void onFailure(Exception e) { + final var cause = ExceptionsHelper.unwrapCause(e); + final var sendShardFailure = + // these indicate the source shard has already failed, which will independently notify the master and fail + // the target shard + false == (cause instanceof ShardNotFoundException || cause instanceof IndexNotFoundException); + // TODO retries? See RecoveryResponseHandler#handleException - onGoingRecoveries.failRecovery(recoveryId, new RecoveryFailedException(recoveryState, null, e), true); + onGoingRecoveries.failRecovery( + recoveryId, + new RecoveryFailedException(recoveryState, null, e), + sendShardFailure + ); } } ); diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index 246a41c99c922..adcb9d29861c0 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -40,6 +40,7 @@ import org.elasticsearch.cluster.coordination.StableMasterHealthIndicatorService; import org.elasticsearch.cluster.desirednodes.DesiredNodesSettingsValidator; import org.elasticsearch.cluster.metadata.IndexMetadataVerifier; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; @@ -52,6 +53,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.routing.BatchedRerouteService; import org.elasticsearch.cluster.routing.RerouteService; +import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.DiskThresholdMonitor; import org.elasticsearch.cluster.routing.allocation.ShardsAvailabilityHealthIndicatorService; import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; @@ -739,24 +741,40 @@ private void construct(Environment initialEnvironment, NodeServiceProvider servi threadPool ); - Collection pluginComponents = pluginsService.flatMap( - p -> p.createComponents( - client, - clusterService, - threadPool, - resourceWatcherService, - scriptService, - xContentRegistry, - environment, - nodeEnvironment, - namedWriteableRegistry, - clusterModule.getIndexNameExpressionResolver(), - repositoriesServiceReference::get, - telemetryProvider, - clusterModule.getAllocationService(), - indicesService - ) - ).toList(); + record PluginServiceInstances( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier, + TelemetryProvider telemetryProvider, + AllocationService allocationService, + IndicesService indicesService + ) implements Plugin.PluginServices {} + PluginServiceInstances pluginServices = new PluginServiceInstances( + client, + clusterService, + threadPool, + resourceWatcherService, + scriptService, + xContentRegistry, + environment, + nodeEnvironment, + namedWriteableRegistry, + clusterModule.getIndexNameExpressionResolver(), + repositoriesServiceReference::get, + telemetryProvider, + clusterModule.getAllocationService(), + indicesService + ); + + Collection pluginComponents = pluginsService.flatMap(p -> p.createComponents(pluginServices)).toList(); List> reservedStateHandlers = new ArrayList<>(); diff --git a/server/src/main/java/org/elasticsearch/plugins/Plugin.java b/server/src/main/java/org/elasticsearch/plugins/Plugin.java index bd5d8e9220517..b0e1946a4e5dd 100644 --- a/server/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -62,6 +62,83 @@ */ public abstract class Plugin implements Closeable { + /** + * Provides access to various Elasticsearch services. + */ + public interface PluginServices { + /** + * A client to make requests to the system + */ + Client client(); + + /** + * A service to allow watching and updating cluster state + */ + ClusterService clusterService(); + + /** + * A service to allow retrieving an executor to run an async action + */ + ThreadPool threadPool(); + + /** + * A service to watch for changes to node local files + */ + ResourceWatcherService resourceWatcherService(); + + /** + * A service to allow running scripts on the local node + */ + ScriptService scriptService(); + + /** + * The registry for extensible xContent parsing + */ + NamedXContentRegistry xContentRegistry(); + + /** + * The environment for path and setting configurations + */ + Environment environment(); + + /** + * The node environment used coordinate access to the data paths + */ + NodeEnvironment nodeEnvironment(); + + /** + * The registry for {@link NamedWriteable} object parsing + */ + NamedWriteableRegistry namedWriteableRegistry(); + + /** + * A service that resolves expression to index and alias names + */ + IndexNameExpressionResolver indexNameExpressionResolver(); + + /** + * A supplier for the service that manages snapshot repositories. + * This will return null when {@link #createComponents(PluginServices)} is called, + * but will return the repositories service once the node is initialized. + */ + Supplier repositoriesServiceSupplier(); + + /** + * An interface for distributed tracing + */ + TelemetryProvider telemetryProvider(); + + /** + * A service to manage shard allocation in the cluster + */ + AllocationService allocationService(); + + /** + * A service to manage indices in the cluster + */ + IndicesService indicesService(); + } + /** * Returns components added by this plugin. *

@@ -69,22 +146,54 @@ public abstract class Plugin implements Closeable { * Note: To aid in the migration away from guice, all objects returned as components will be bound in guice * to themselves. * + * @param services Provides access to various Elasticsearch services + */ + public Collection createComponents(PluginServices services) { + return createComponents( + services.client(), + services.clusterService(), + services.threadPool(), + services.resourceWatcherService(), + services.scriptService(), + services.xContentRegistry(), + services.environment(), + services.nodeEnvironment(), + services.namedWriteableRegistry(), + services.indexNameExpressionResolver(), + services.repositoriesServiceSupplier(), + services.telemetryProvider(), + services.allocationService(), + services.indicesService() + ); + } + + /** + * Returns components added by this plugin. Either this method or {@link #createComponents(PluginServices)} + * should be implemented. + *

+ * Any components returned that implement {@link LifecycleComponent} will have their lifecycle managed. + * Note: To aid in the migration away from guice, all objects returned as components will be bound in guice + * to themselves. + * * @param client A client to make requests to the system * @param clusterService A service to allow watching and updating cluster state * @param threadPool A service to allow retrieving an executor to run an async action * @param resourceWatcherService A service to watch for changes to node local files * @param scriptService A service to allow running scripts on the local node - * @param xContentRegistry the registry for extensible xContent parsing - * @param environment the environment for path and setting configurations - * @param nodeEnvironment the node environment used coordinate access to the data paths - * @param namedWriteableRegistry the registry for {@link NamedWriteable} object parsing + * @param xContentRegistry The registry for extensible xContent parsing + * @param environment The environment for path and setting configurations + * @param nodeEnvironment The node environment used coordinate access to the data paths + * @param namedWriteableRegistry The registry for {@link NamedWriteable} object parsing * @param indexNameExpressionResolver A service that resolves expression to index and alias names * @param repositoriesServiceSupplier A supplier for the service that manages snapshot repositories; will return null when this method * is called, but will return the repositories service once the node is initialized. * @param telemetryProvider An interface for distributed tracing * @param allocationService A service to manage shard allocation in the cluster * @param indicesService A service to manage indices in the cluster + * + * @deprecated New services will only be added to {@link PluginServices}; this method is maintained for compatibility. */ + @Deprecated public Collection createComponents( Client client, ClusterService clusterService, diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java index 3adf32454cc20..f6d5ab3ead6af 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java @@ -239,7 +239,7 @@ public void onFailure(Exception e) { @Override public void onFailure(Exception e) { // If we encounter an error while runnin the non-state transforms, we avoid saving any cluster state. - errorListener.accept(checkAndReportError(namespace, List.of(e.getMessage()), reservedStateVersion)); + errorListener.accept(checkAndReportError(namespace, List.of(stackTrace(e)), reservedStateVersion)); } }); } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index dafcf45454aaf..6919cfdbc00b4 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -224,7 +224,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv public static final Setting QUERY_PHASE_PARALLEL_COLLECTION_ENABLED = Setting.boolSetting( "search.query_phase_parallel_collection_enabled", - false, + true, Property.NodeScope, Property.Dynamic ); diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 1733029af9024..3c69db98c7588 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -87,7 +87,7 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable { private QueryProfiler profiler; private final MutableQueryTimeout cancellable; - private final LeafSlice[] leafSlices; + private final int maximumNumberOfSlices; // don't create slices with less than this number of docs private final int minimumDocsPerSlice; @@ -150,13 +150,15 @@ public ContextIndexSearcher( setQueryCachingPolicy(queryCachingPolicy); this.cancellable = cancellable; this.minimumDocsPerSlice = minimumDocsPerSlice; - if (executor == null) { - this.leafSlices = null; - } else { - // we offload to the executor unconditionally, including requests that don't support concurrency - this.leafSlices = computeSlices(getLeafContexts(), maximumNumberOfSlices, minimumDocsPerSlice); - assert this.leafSlices.length <= maximumNumberOfSlices : "more slices created than the maximum allowed"; - } + this.maximumNumberOfSlices = maximumNumberOfSlices; + } + + @Override + protected LeafSlice[] slices(List leaves) { + // we offload to the executor unconditionally, including requests that don't support concurrency + LeafSlice[] leafSlices = computeSlices(getLeafContexts(), maximumNumberOfSlices, minimumDocsPerSlice); + assert leafSlices.length <= maximumNumberOfSlices : "more slices created than the maximum allowed"; + return leafSlices; } // package private for testing @@ -238,15 +240,6 @@ public Weight createWeight(Query query, ScoreMode scoreMode, float boost) throws } } - /** - * Returns the slices created by this {@link ContextIndexSearcher}, different from those created by the base class and - * returned by {@link IndexSearcher#getSlices()}. The former are used for parallelizing the collection, while the latter are used - * for now to parallelize rewrite (e.g. knn query rewrite) - */ - final LeafSlice[] getSlicesForCollection() { - return leafSlices; - } - /** * Each computed slice contains at least 10% of the total data in the leaves with a * minimum given by the minDocsPerSlice parameter and the final number @@ -346,7 +339,9 @@ private T search(Weight weight, CollectorManager if (getExecutor() == null) { search(leafContexts, weight, firstCollector); return collectorManager.reduce(Collections.singletonList(firstCollector)); - } else if (leafSlices.length == 0) { + } + LeafSlice[] leafSlices = getSlices(); + if (leafSlices.length == 0) { assert leafContexts.isEmpty(); doAggregationPostCollection(firstCollector); return collectorManager.reduce(Collections.singletonList(firstCollector)); diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/MessagesTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/MessagesTests.java index f779d5ea56dfa..f4eb05b573463 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/MessagesTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/MessagesTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; -import org.elasticsearch.test.EqualsHashCodeTestUtils.CopyFunction; import org.elasticsearch.test.TransportVersionUtils; import java.util.Map; @@ -36,59 +35,56 @@ public void testJoinEqualsHashCodeSerialization() { randomNonNegativeLong(), randomNonNegativeLong() ); - // Note: the explicit cast of the CopyFunction is needed for some IDE (specifically Eclipse 4.8.0) to infer the right type EqualsHashCodeTestUtils.checkEqualsAndHashCode( initialJoin, - (CopyFunction) join -> copyWriteable(join, writableRegistry(), Join::new), - join -> { - return switch (randomInt(4)) { - case 0 -> - // change sourceNode - new Join( - createNode(randomAlphaOfLength(20)), - join.getTargetNode(), - join.getTerm(), - join.getLastAcceptedTerm(), - join.getLastAcceptedVersion() - ); - case 1 -> - // change targetNode - new Join( - join.getSourceNode(), - createNode(randomAlphaOfLength(20)), - join.getTerm(), - join.getLastAcceptedTerm(), - join.getLastAcceptedVersion() - ); - case 2 -> - // change term - new Join( - join.getSourceNode(), - join.getTargetNode(), - randomValueOtherThan(join.getTerm(), ESTestCase::randomNonNegativeLong), - join.getLastAcceptedTerm(), - join.getLastAcceptedVersion() - ); - case 3 -> - // change last accepted term - new Join( - join.getSourceNode(), - join.getTargetNode(), - join.getTerm(), - randomValueOtherThan(join.getLastAcceptedTerm(), ESTestCase::randomNonNegativeLong), - join.getLastAcceptedVersion() - ); - case 4 -> - // change version - new Join( - join.getSourceNode(), - join.getTargetNode(), - join.getTerm(), - join.getLastAcceptedTerm(), - randomValueOtherThan(join.getLastAcceptedVersion(), ESTestCase::randomNonNegativeLong) - ); - default -> throw new AssertionError(); - }; + join -> copyWriteable(join, writableRegistry(), Join::new), + join -> switch (randomInt(4)) { + case 0 -> + // change sourceNode + new Join( + createNode(randomAlphaOfLength(20)), + join.getTargetNode(), + join.getTerm(), + join.getLastAcceptedTerm(), + join.getLastAcceptedVersion() + ); + case 1 -> + // change targetNode + new Join( + join.getSourceNode(), + createNode(randomAlphaOfLength(20)), + join.getTerm(), + join.getLastAcceptedTerm(), + join.getLastAcceptedVersion() + ); + case 2 -> + // change term + new Join( + join.getSourceNode(), + join.getTargetNode(), + randomValueOtherThan(join.getTerm(), ESTestCase::randomNonNegativeLong), + join.getLastAcceptedTerm(), + join.getLastAcceptedVersion() + ); + case 3 -> + // change last accepted term + new Join( + join.getSourceNode(), + join.getTargetNode(), + join.getTerm(), + randomValueOtherThan(join.getLastAcceptedTerm(), ESTestCase::randomNonNegativeLong), + join.getLastAcceptedVersion() + ); + case 4 -> + // change version + new Join( + join.getSourceNode(), + join.getTargetNode(), + join.getTerm(), + join.getLastAcceptedTerm(), + randomValueOtherThan(join.getLastAcceptedVersion(), ESTestCase::randomNonNegativeLong) + ); + default -> throw new AssertionError(); } ); } @@ -104,26 +100,23 @@ public void testPublishRequestEqualsHashCode() { public void testPublishResponseEqualsHashCodeSerialization() { PublishResponse initialPublishResponse = new PublishResponse(randomNonNegativeLong(), randomNonNegativeLong()); - // Note: the explicit cast of the CopyFunction is needed for some IDE (specifically Eclipse 4.8.0) to infer the right type EqualsHashCodeTestUtils.checkEqualsAndHashCode( initialPublishResponse, - (CopyFunction) publishResponse -> copyWriteable(publishResponse, writableRegistry(), PublishResponse::new), - publishResponse -> { - return switch (randomInt(1)) { - case 0 -> - // change term - new PublishResponse( - randomValueOtherThan(publishResponse.getTerm(), ESTestCase::randomNonNegativeLong), - publishResponse.getVersion() - ); - case 1 -> - // change version - new PublishResponse( - publishResponse.getTerm(), - randomValueOtherThan(publishResponse.getVersion(), ESTestCase::randomNonNegativeLong) - ); - default -> throw new AssertionError(); - }; + publishResponse -> copyWriteable(publishResponse, writableRegistry(), PublishResponse::new), + publishResponse -> switch (randomInt(1)) { + case 0 -> + // change term + new PublishResponse( + randomValueOtherThan(publishResponse.getTerm(), ESTestCase::randomNonNegativeLong), + publishResponse.getVersion() + ); + case 1 -> + // change version + new PublishResponse( + publishResponse.getTerm(), + randomValueOtherThan(publishResponse.getVersion(), ESTestCase::randomNonNegativeLong) + ); + default -> throw new AssertionError(); } ); } @@ -141,61 +134,51 @@ public void testPublishWithJoinResponseEqualsHashCodeSerialization() { initialPublishResponse, randomBoolean() ? Optional.empty() : Optional.of(initialJoin) ); - // Note: the explicit cast of the CopyFunction is needed for some IDE (specifically Eclipse 4.8.0) to infer the right type EqualsHashCodeTestUtils.checkEqualsAndHashCode( initialPublishWithJoinResponse, - (CopyFunction) publishWithJoinResponse -> copyWriteable( - publishWithJoinResponse, - writableRegistry(), - PublishWithJoinResponse::new - ), - publishWithJoinResponse -> { - switch (randomInt(1)) { - case 0: - // change publish response - return new PublishWithJoinResponse( - new PublishResponse(randomNonNegativeLong(), randomNonNegativeLong()), - publishWithJoinResponse.getJoin() - ); - case 1: - // change optional join - Join newJoin = new Join( - createNode(randomAlphaOfLength(10)), - createNode(randomAlphaOfLength(10)), - randomNonNegativeLong(), - randomNonNegativeLong(), - randomNonNegativeLong() - ); - return new PublishWithJoinResponse( - publishWithJoinResponse.getPublishResponse(), - publishWithJoinResponse.getJoin().isPresent() && randomBoolean() ? Optional.empty() : Optional.of(newJoin) - ); - default: - throw new AssertionError(); + publishWithJoinResponse -> copyWriteable(publishWithJoinResponse, writableRegistry(), PublishWithJoinResponse::new), + publishWithJoinResponse -> switch (randomInt(1)) { + case 0 -> + // change publish response + new PublishWithJoinResponse( + new PublishResponse(randomNonNegativeLong(), randomNonNegativeLong()), + publishWithJoinResponse.getJoin() + ); + case 1 -> { + // change optional join + Join newJoin = new Join( + createNode(randomAlphaOfLength(10)), + createNode(randomAlphaOfLength(10)), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong() + ); + yield new PublishWithJoinResponse( + publishWithJoinResponse.getPublishResponse(), + publishWithJoinResponse.getJoin().isPresent() && randomBoolean() ? Optional.empty() : Optional.of(newJoin) + ); } + default -> throw new AssertionError(); } ); } public void testStartJoinRequestEqualsHashCodeSerialization() { StartJoinRequest initialStartJoinRequest = new StartJoinRequest(createNode(randomAlphaOfLength(10)), randomNonNegativeLong()); - // Note: the explicit cast of the CopyFunction is needed for some IDE (specifically Eclipse 4.8.0) to infer the right type EqualsHashCodeTestUtils.checkEqualsAndHashCode( initialStartJoinRequest, - (CopyFunction) startJoinRequest -> copyWriteable(startJoinRequest, writableRegistry(), StartJoinRequest::new), - startJoinRequest -> { - return switch (randomInt(1)) { - case 0 -> - // change sourceNode - new StartJoinRequest(createNode(randomAlphaOfLength(20)), startJoinRequest.getTerm()); - case 1 -> - // change term - new StartJoinRequest( - startJoinRequest.getSourceNode(), - randomValueOtherThan(startJoinRequest.getTerm(), ESTestCase::randomNonNegativeLong) - ); - default -> throw new AssertionError(); - }; + startJoinRequest -> copyWriteable(startJoinRequest, writableRegistry(), StartJoinRequest::new), + startJoinRequest -> switch (randomInt(1)) { + case 0 -> + // change sourceNode + new StartJoinRequest(createNode(randomAlphaOfLength(20)), startJoinRequest.getTerm()); + case 1 -> + // change term + new StartJoinRequest( + startJoinRequest.getSourceNode(), + randomValueOtherThan(startJoinRequest.getTerm(), ESTestCase::randomNonNegativeLong) + ); + default -> throw new AssertionError(); } ); } @@ -206,31 +189,28 @@ public void testApplyCommitEqualsHashCodeSerialization() { randomNonNegativeLong(), randomNonNegativeLong() ); - // Note: the explicit cast of the CopyFunction is needed for some IDE (specifically Eclipse 4.8.0) to infer the right type EqualsHashCodeTestUtils.checkEqualsAndHashCode( initialApplyCommit, - (CopyFunction) applyCommit -> copyWriteable(applyCommit, writableRegistry(), ApplyCommitRequest::new), - applyCommit -> { - return switch (randomInt(2)) { - case 0 -> - // change sourceNode - new ApplyCommitRequest(createNode(randomAlphaOfLength(20)), applyCommit.getTerm(), applyCommit.getVersion()); - case 1 -> - // change term - new ApplyCommitRequest( - applyCommit.getSourceNode(), - randomValueOtherThan(applyCommit.getTerm(), ESTestCase::randomNonNegativeLong), - applyCommit.getVersion() - ); - case 2 -> - // change version - new ApplyCommitRequest( - applyCommit.getSourceNode(), - applyCommit.getTerm(), - randomValueOtherThan(applyCommit.getVersion(), ESTestCase::randomNonNegativeLong) - ); - default -> throw new AssertionError(); - }; + applyCommit -> copyWriteable(applyCommit, writableRegistry(), ApplyCommitRequest::new), + applyCommit -> switch (randomInt(2)) { + case 0 -> + // change sourceNode + new ApplyCommitRequest(createNode(randomAlphaOfLength(20)), applyCommit.getTerm(), applyCommit.getVersion()); + case 1 -> + // change term + new ApplyCommitRequest( + applyCommit.getSourceNode(), + randomValueOtherThan(applyCommit.getTerm(), ESTestCase::randomNonNegativeLong), + applyCommit.getVersion() + ); + case 2 -> + // change version + new ApplyCommitRequest( + applyCommit.getSourceNode(), + applyCommit.getTerm(), + randomValueOtherThan(applyCommit.getVersion(), ESTestCase::randomNonNegativeLong) + ); + default -> throw new AssertionError(); } ); } @@ -249,58 +229,51 @@ public void testJoinRequestEqualsHashCodeSerialization() { randomNonNegativeLong(), randomBoolean() ? Optional.empty() : Optional.of(initialJoin) ); - // Note: the explicit cast of the CopyFunction is needed for some IDE (specifically Eclipse 4.8.0) to infer the right type EqualsHashCodeTestUtils.checkEqualsAndHashCode( initialJoinRequest, - (CopyFunction) joinRequest -> copyWriteable(joinRequest, writableRegistry(), JoinRequest::new), - joinRequest -> { - if (randomBoolean() && joinRequest.getOptionalJoin().isPresent() == false) { - return new JoinRequest( + joinRequest -> copyWriteable(joinRequest, writableRegistry(), JoinRequest::new), + joinRequest -> switch (randomInt(3)) { + case 0 -> { + assumeTrue("Optional join needs to be empty", joinRequest.getOptionalJoin().isEmpty()); + yield new JoinRequest( createNode(randomAlphaOfLength(10)), joinRequest.getCompatibilityVersions(), joinRequest.getMinimumTerm(), joinRequest.getOptionalJoin() ); - } else if (randomBoolean()) { - return new JoinRequest( - joinRequest.getSourceNode(), - new CompatibilityVersions( - TransportVersionUtils.randomVersion(Set.of(joinRequest.getCompatibilityVersions().transportVersion())), - Map.of() - ), - joinRequest.getMinimumTerm(), - joinRequest.getOptionalJoin() - ); - } else if (randomBoolean()) { - return new JoinRequest( + } + case 1 -> new JoinRequest( + joinRequest.getSourceNode(), + new CompatibilityVersions( + TransportVersionUtils.randomVersion(Set.of(joinRequest.getCompatibilityVersions().transportVersion())), + Map.of() + ), + joinRequest.getMinimumTerm(), + joinRequest.getOptionalJoin() + ); + case 2 -> new JoinRequest( + joinRequest.getSourceNode(), + joinRequest.getCompatibilityVersions(), + randomValueOtherThan(joinRequest.getMinimumTerm(), ESTestCase::randomNonNegativeLong), + joinRequest.getOptionalJoin() + ); + case 3 -> { + // change OptionalJoin + Join newJoin = new Join( joinRequest.getSourceNode(), - joinRequest.getCompatibilityVersions(), - randomValueOtherThan(joinRequest.getMinimumTerm(), ESTestCase::randomNonNegativeLong), - joinRequest.getOptionalJoin() + createNode(randomAlphaOfLength(10)), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong() ); - } else { - // change OptionalJoin - final Optional newOptionalJoin; - if (joinRequest.getOptionalJoin().isPresent() && randomBoolean()) { - newOptionalJoin = Optional.empty(); - } else { - newOptionalJoin = Optional.of( - new Join( - joinRequest.getSourceNode(), - createNode(randomAlphaOfLength(10)), - randomNonNegativeLong(), - randomNonNegativeLong(), - randomNonNegativeLong() - ) - ); - } - return new JoinRequest( + yield new JoinRequest( joinRequest.getSourceNode(), joinRequest.getCompatibilityVersions(), joinRequest.getMinimumTerm(), - newOptionalJoin + joinRequest.getOptionalJoin().isPresent() && randomBoolean() ? Optional.empty() : Optional.of(newJoin) ); } + default -> throw new AssertionError(); } ); } @@ -318,16 +291,13 @@ public ClusterState randomClusterState() { public void testPreVoteRequestEqualsHashCodeSerialization() { PreVoteRequest initialPreVoteRequest = new PreVoteRequest(createNode(randomAlphaOfLength(10)), randomNonNegativeLong()); - // Note: the explicit cast of the CopyFunction is needed for some IDE (specifically Eclipse 4.8.0) to infer the right type EqualsHashCodeTestUtils.checkEqualsAndHashCode( initialPreVoteRequest, - (CopyFunction) preVoteRequest -> copyWriteable(preVoteRequest, writableRegistry(), PreVoteRequest::new), - preVoteRequest -> { - if (randomBoolean()) { - return new PreVoteRequest(createNode(randomAlphaOfLength(10)), preVoteRequest.getCurrentTerm()); - } else { - return new PreVoteRequest(preVoteRequest.getSourceNode(), randomNonNegativeLong()); - } + preVoteRequest -> copyWriteable(preVoteRequest, writableRegistry(), PreVoteRequest::new), + preVoteRequest -> switch (randomInt(1)) { + case 0 -> new PreVoteRequest(createNode(randomAlphaOfLength(10)), preVoteRequest.getCurrentTerm()); + case 1 -> new PreVoteRequest(preVoteRequest.getSourceNode(), randomNonNegativeLong()); + default -> throw new AssertionError(); } ); } @@ -339,41 +309,38 @@ public void testPreVoteResponseEqualsHashCodeSerialization() { randomLongBetween(1, currentTerm), randomNonNegativeLong() ); - // Note: the explicit cast of the CopyFunction is needed for some IDE (specifically Eclipse 4.8.0) to infer the right type EqualsHashCodeTestUtils.checkEqualsAndHashCode( initialPreVoteResponse, - (CopyFunction) preVoteResponse -> copyWriteable(preVoteResponse, writableRegistry(), PreVoteResponse::new), - preVoteResponse -> { - switch (randomInt(2)) { - case 0: - assumeTrue("last-accepted term is Long.MAX_VALUE", preVoteResponse.getLastAcceptedTerm() < Long.MAX_VALUE); - return new PreVoteResponse( - randomValueOtherThan( - preVoteResponse.getCurrentTerm(), - () -> randomLongBetween(preVoteResponse.getLastAcceptedTerm(), Long.MAX_VALUE) - ), - preVoteResponse.getLastAcceptedTerm(), - preVoteResponse.getLastAcceptedVersion() - ); - case 1: - assumeTrue("current term is 1", 1 < preVoteResponse.getCurrentTerm()); - return new PreVoteResponse( - preVoteResponse.getCurrentTerm(), - randomValueOtherThan( - preVoteResponse.getLastAcceptedTerm(), - () -> randomLongBetween(1, preVoteResponse.getCurrentTerm()) - ), - preVoteResponse.getLastAcceptedVersion() - ); - case 2: - return new PreVoteResponse( + preVoteResponse -> copyWriteable(preVoteResponse, writableRegistry(), PreVoteResponse::new), + preVoteResponse -> switch (randomInt(2)) { + case 0 -> { + assumeTrue("last-accepted term is Long.MAX_VALUE", preVoteResponse.getLastAcceptedTerm() < Long.MAX_VALUE); + yield new PreVoteResponse( + randomValueOtherThan( preVoteResponse.getCurrentTerm(), + () -> randomLongBetween(preVoteResponse.getLastAcceptedTerm(), Long.MAX_VALUE) + ), + preVoteResponse.getLastAcceptedTerm(), + preVoteResponse.getLastAcceptedVersion() + ); + } + case 1 -> { + assumeTrue("current term is 1", 1 < preVoteResponse.getCurrentTerm()); + yield new PreVoteResponse( + preVoteResponse.getCurrentTerm(), + randomValueOtherThan( preVoteResponse.getLastAcceptedTerm(), - randomValueOtherThan(preVoteResponse.getLastAcceptedVersion(), ESTestCase::randomNonNegativeLong) - ); - default: - throw new AssertionError(); + () -> randomLongBetween(1, preVoteResponse.getCurrentTerm()) + ), + preVoteResponse.getLastAcceptedVersion() + ); } + case 2 -> new PreVoteResponse( + preVoteResponse.getCurrentTerm(), + preVoteResponse.getLastAcceptedTerm(), + randomValueOtherThan(preVoteResponse.getLastAcceptedVersion(), ESTestCase::randomNonNegativeLong) + ); + default -> throw new AssertionError(); } ); } diff --git a/server/src/test/java/org/elasticsearch/node/NodeTests.java b/server/src/test/java/org/elasticsearch/node/NodeTests.java index 9a9ffeea84610..218073a1eb3f1 100644 --- a/server/src/test/java/org/elasticsearch/node/NodeTests.java +++ b/server/src/test/java/org/elasticsearch/node/NodeTests.java @@ -11,23 +11,17 @@ import org.apache.lucene.util.SetOnce; import org.elasticsearch.bootstrap.BootstrapCheck; import org.elasticsearch.bootstrap.BootstrapContext; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.component.LifecycleComponent; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.gateway.PersistedClusterStateService; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexService; @@ -43,18 +37,14 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsServiceTests; import org.elasticsearch.plugins.RecoveryPlannerPlugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.tasks.Task; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.MockHttpTransport; import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.ContextParser; import org.elasticsearch.xcontent.MediaType; import org.elasticsearch.xcontent.NamedObjectNotFoundException; @@ -77,7 +67,6 @@ import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Supplier; import static org.elasticsearch.test.NodeRoles.dataNode; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -442,22 +431,7 @@ protected void doClose() throws IOException { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { List components = new ArrayList<>(); components.add(new PluginComponentBinding<>(MyInterface.class, getRandomBool() ? new Foo() : new Bar())); return components; diff --git a/server/src/test/java/org/elasticsearch/plugins/PluginIntrospectorTests.java b/server/src/test/java/org/elasticsearch/plugins/PluginIntrospectorTests.java index c5e974c4abcfd..5e80b6d217a55 100644 --- a/server/src/test/java/org/elasticsearch/plugins/PluginIntrospectorTests.java +++ b/server/src/test/java/org/elasticsearch/plugins/PluginIntrospectorTests.java @@ -8,38 +8,24 @@ package org.elasticsearch.plugins; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.health.HealthIndicatorService; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.engine.EngineFactory; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.BreakerSettings; import org.elasticsearch.indices.recovery.plan.RecoveryPlannerService; import org.elasticsearch.indices.recovery.plan.ShardSnapshotsService; import org.elasticsearch.ingest.Processor; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.PrivilegedOperations; import org.elasticsearch.test.compiler.InMemoryJavaCompiler; import org.elasticsearch.test.jar.JarUtils; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import java.net.URL; import java.net.URLClassLoader; @@ -259,22 +245,7 @@ public final class FooPlugin extends q.AbstractFooPlugin { } public void testOverriddenMethodsBasic() { class FooPlugin extends Plugin { @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { return null; } } diff --git a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java index 3f7d67d292761..9e6b6330d2f23 100644 --- a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java @@ -70,6 +70,7 @@ import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.lucene.util.CombinedBitSet; +import org.elasticsearch.lucene.util.MatchAllBitSet; import org.elasticsearch.search.aggregations.BucketCollector; import org.elasticsearch.search.aggregations.LeafBucketCollector; import org.elasticsearch.test.ESTestCase; @@ -96,6 +97,7 @@ import static org.elasticsearch.search.internal.ExitableDirectoryReader.ExitableLeafReader; import static org.elasticsearch.search.internal.ExitableDirectoryReader.ExitablePointValues; import static org.elasticsearch.search.internal.ExitableDirectoryReader.ExitableTerms; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.instanceOf; @@ -279,7 +281,6 @@ public void testContextIndexSearcherSparseWithDeletions() throws IOException { doTestContextIndexSearcher(true, true); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/94615") public void testContextIndexSearcherDenseWithDeletions() throws IOException { doTestContextIndexSearcher(false, true); } @@ -332,7 +333,7 @@ public void onRemoval(ShardId shardId, Accountable accountable) { if (sparse) { assertThat(bitSet, instanceOf(SparseFixedBitSet.class)); } else { - assertThat(bitSet, instanceOf(FixedBitSet.class)); + assertThat(bitSet, anyOf(instanceOf(FixedBitSet.class), instanceOf(MatchAllBitSet.class))); } DocumentSubsetDirectoryReader filteredReader = new DocumentSubsetDirectoryReader(reader, cache, roleQuery); @@ -564,7 +565,7 @@ public void testCancelSliceTasksOnException() throws Exception { 1 ) ) { - leafSlices = contextIndexSearcher.getSlicesForCollection(); + leafSlices = contextIndexSearcher.getSlices(); int numThrowingLeafSlices = randomIntBetween(1, 3); for (int i = 0; i < numThrowingLeafSlices; i++) { LeafSlice throwingLeafSlice = leafSlices[randomIntBetween(0, Math.min(leafSlices.length, numAvailableThreads) - 1)]; @@ -700,7 +701,7 @@ public void testCancelSliceTasksOnTimeout() throws Exception { 1 ) ) { - leafSlices = contextIndexSearcher.getSlicesForCollection(); + leafSlices = contextIndexSearcher.getSlices(); int numThrowingLeafSlices = randomIntBetween(1, 3); for (int i = 0; i < numThrowingLeafSlices; i++) { LeafSlice throwingLeafSlice = leafSlices[randomIntBetween(0, Math.min(leafSlices.length, numAvailableThreads) - 1)]; diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java index 47d75a78d65d4..3e05743741f73 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java @@ -52,7 +52,6 @@ public void tearDown() throws Exception { ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/97080") public void testConnectAndExecuteRequest() throws Exception { Settings remoteSettings = Settings.builder() .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), "foo_bar_cluster") diff --git a/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekTrackerPlugin.java b/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekTrackerPlugin.java index de2ac43f7fb51..aa9ff52b00824 100644 --- a/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekTrackerPlugin.java +++ b/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekTrackerPlugin.java @@ -11,31 +11,18 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexModule; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.Collection; import java.util.Collections; @@ -66,22 +53,7 @@ public List> getSettings() { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { return Collections.singletonList(seekStatsService); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java index 5a42d5c6890b5..298acb9519532 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java @@ -28,7 +28,8 @@ public class TestBlock BlockLoader.DoubleBuilder, BlockLoader.IntBuilder, BlockLoader.LongBuilder, - BlockLoader.SingletonOrdinalsBuilder { + BlockLoader.SingletonOrdinalsBuilder, + BlockLoader.Block { public static BlockLoader.BuilderFactory FACTORY = new BlockLoader.BuilderFactory() { @Override public BlockLoader.BooleanBuilder booleansFromDocValues(int expectedCount) { @@ -192,8 +193,18 @@ public TestBlock appendOrd(int value) { } } + @Override + public TestBlock build() { + return this; + } + private TestBlock add(Object value) { (currentPosition == null ? values : currentPosition).add(value); return this; } + + @Override + public void close() { + // TODO assert that we close the test blocks + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 78ea21f117114..832902f52deb2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -2075,8 +2075,9 @@ private NodeConfigurationSource getNodeConfigSource() { } boolean enableConcurrentSearch = enableConcurrentSearch(); if (enableConcurrentSearch) { - initialNodeSettings.put(SearchService.QUERY_PHASE_PARALLEL_COLLECTION_ENABLED.getKey(), true); initialNodeSettings.put(SearchService.MINIMUM_DOCS_PER_SLICE.getKey(), 1); + } else { + initialNodeSettings.put(SearchService.QUERY_PHASE_PARALLEL_COLLECTION_ENABLED.getKey(), false); } return new NodeConfigurationSource() { @Override diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index ca7cb72417fac..b5ac94b53d3ca 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -254,8 +254,9 @@ private Node newNode() { boolean enableConcurrentSearch = enableConcurrentSearch(); if (enableConcurrentSearch) { - settingBuilder.put(SearchService.QUERY_PHASE_PARALLEL_COLLECTION_ENABLED.getKey(), true) - .put(SearchService.MINIMUM_DOCS_PER_SLICE.getKey(), 1); + settingBuilder.put(SearchService.MINIMUM_DOCS_PER_SLICE.getKey(), 1); + } else { + settingBuilder.put(SearchService.QUERY_PHASE_PARALLEL_COLLECTION_ENABLED.getKey(), false); } Settings settings = settingBuilder.build(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java b/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java index 9b9f464d8dff3..d504bebaa33e7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java +++ b/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java @@ -7,18 +7,11 @@ */ package org.elasticsearch.test; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexService; @@ -27,21 +20,13 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardState; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.function.Supplier; /** * This is a testing plugin that registers a generic @@ -76,22 +61,7 @@ public void onIndexModule(IndexModule module) { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { return Collections.singletonList(listener); } } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java index 9360f97990c82..28d9d87f147d3 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java @@ -8,27 +8,14 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.analytics.action.AnalyticsInfoTransportAction; import org.elasticsearch.xpack.analytics.action.AnalyticsUsageTransportAction; import org.elasticsearch.xpack.analytics.action.TransportAnalyticsStatsAction; @@ -64,7 +51,6 @@ import java.util.List; import java.util.Map; import java.util.function.Consumer; -import java.util.function.Supplier; public class AnalyticsPlugin extends Plugin implements SearchPlugin, ActionPlugin, MapperPlugin { private final AnalyticsUsage usage = new AnalyticsUsage(); @@ -175,22 +161,7 @@ public List> getAggregationExtentions() { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { return List.of(usage); } diff --git a/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java b/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java index a0e6d0daac322..3dbe59dc825ff 100644 --- a/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java +++ b/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java @@ -7,33 +7,18 @@ package org.elasticsearch.xpack.async; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.async.AsyncTaskIndexService; import org.elasticsearch.xpack.core.async.AsyncTaskMaintenanceService; import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.function.Supplier; import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; @@ -61,31 +46,16 @@ public String getFeatureDescription() { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { List components = new ArrayList<>(); - if (DiscoveryNode.canContainData(environment.settings())) { + if (DiscoveryNode.canContainData(services.environment().settings())) { // only data nodes should be eligible to run the maintenance service. AsyncTaskMaintenanceService maintenanceService = new AsyncTaskMaintenanceService( - clusterService, - nodeEnvironment.nodeId(), + services.clusterService(), + services.nodeEnvironment().nodeId(), settings, - threadPool, - new OriginSettingClient(client, ASYNC_SEARCH_ORIGIN) + services.threadPool(), + new OriginSettingClient(services.client(), ASYNC_SEARCH_ORIGIN) ); components.add(maintenanceService); } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java index c832a3c7eb461..e9d54826436c2 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java @@ -10,7 +10,6 @@ import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; @@ -23,22 +22,14 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.license.License; import org.elasticsearch.license.LicensedFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.ExtensiblePlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.autoscaling.action.DeleteAutoscalingPolicyAction; @@ -109,27 +100,16 @@ public Autoscaling() { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - this.clusterServiceHolder.set(clusterService); - this.allocationServiceHolder.set(allocationService); + public Collection createComponents(PluginServices services) { + this.clusterServiceHolder.set(services.clusterService()); + this.allocationServiceHolder.set(services.allocationService()); var capacityServiceHolder = new AutoscalingCalculateCapacityService.Holder(this); this.reservedAutoscalingPolicyAction.set(new ReservedAutoscalingPolicyAction(capacityServiceHolder)); - return List.of(capacityServiceHolder, autoscalingLicenseChecker, new AutoscalingNodeInfoService(clusterService, client)); + return List.of( + capacityServiceHolder, + autoscalingLicenseChecker, + new AutoscalingNodeInfoService(services.clusterService(), services.client()) + ); } @Override diff --git a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle b/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle index 3eaf6f267a545..da39d221f92f1 100644 --- a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle +++ b/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle @@ -1,5 +1,6 @@ import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.test.RestIntegTestTask +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.standalone-rest-test' @@ -23,8 +24,7 @@ def followCluster = testClusters.register("follow-cluster") { setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' user username: 'admin', password: 'admin-password', role: 'superuser' - setting 'cluster.remote.leader_cluster.seeds', { "\"${leaderCluster.get().getAllTransportPortURI().join(",")}\"" - } + setting 'cluster.remote.leader_cluster.seeds', { "\"${leaderCluster.get().getAllTransportPortURI().join(",")}\"" }, IGNORE_VALUE } tasks.register("leader-cluster", RestIntegTestTask) { @@ -51,8 +51,8 @@ tasks.register("writeJavaPolicy") { tasks.register("follow-cluster", RestIntegTestTask) { dependsOn 'writeJavaPolicy', "leader-cluster" useCluster leaderCluster - systemProperty 'java.security.policy', "file://${policyFile}" systemProperty 'tests.target_cluster', 'follow' + nonInputProperties.systemProperty 'java.security.policy', "file://${policyFile}" nonInputProperties.systemProperty 'tests.leader_host', leaderCluster.map(c -> c.allHttpSocketURI.get(0)) nonInputProperties.systemProperty 'log', followCluster.map(c -> c.getFirstNode().getServerLog()) } diff --git a/x-pack/plugin/ccr/qa/multi-cluster/build.gradle b/x-pack/plugin/ccr/qa/multi-cluster/build.gradle index 31cdf04f6dc94..2475a56aa87aa 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/ccr/qa/multi-cluster/build.gradle @@ -1,6 +1,7 @@ import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.test.RestIntegTestTask +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.standalone-rest-test' @@ -25,7 +26,7 @@ def middleCluster = testClusters.register('middle-cluster') { setting 'xpack.security.enabled', 'true' user username: 'admin', password: 'admin-password', role: 'superuser' setting 'cluster.remote.leader_cluster.seeds', - { "\"${leaderCluster.get().getAllTransportPortURI().join(",")}\"" } + { "\"${leaderCluster.get().getAllTransportPortURI().join(",")}\"" }, IGNORE_VALUE } tasks.register("leader-cluster", RestIntegTestTask) { @@ -60,9 +61,9 @@ testClusters.matching {it.name == "follow-cluster" }.configureEach { setting 'xpack.security.enabled', 'true' user username: 'admin', password: 'admin-password', role: 'superuser' setting 'cluster.remote.leader_cluster.seeds', - { "\"${leaderCluster.get().getAllTransportPortURI().join(",")}\"" } + { "\"${leaderCluster.get().getAllTransportPortURI().join(",")}\"" }, IGNORE_VALUE setting 'cluster.remote.middle_cluster.seeds', - { "\"${middleCluster.get().getAllTransportPortURI().join(",")}\"" } + { "\"${middleCluster.get().getAllTransportPortURI().join(",")}\"" }, IGNORE_VALUE } diff --git a/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle b/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle index 67465bc782ad9..7661ea08b057d 100644 --- a/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle +++ b/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle @@ -1,4 +1,5 @@ import org.elasticsearch.gradle.internal.test.RestIntegTestTask +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.standalone-rest-test' @@ -21,7 +22,7 @@ def followerCluster = testClusters.register('follow-cluster') { setting 'xpack.security.enabled', 'true' user username: 'admin', password: 'admin-password', role: 'superuser' setting 'cluster.remote.leader_cluster.seeds', - { "\"${leaderCluster.get().getAllTransportPortURI().join(",")}\"" } + { "\"${leaderCluster.get().getAllTransportPortURI().join(",")}\"" }, IGNORE_VALUE } tasks.register('leader-cluster', RestIntegTestTask) { @@ -36,4 +37,4 @@ tasks.register('follow-cluster', RestIntegTestTask) { nonInputProperties.systemProperty 'tests.leader_host', followerCluster.map(c -> c.allHttpSocketURI.get(0)) } -tasks.named("check").configure { dependsOn "follow-cluster" } \ No newline at end of file +tasks.named("check").configure { dependsOn "follow-cluster" } diff --git a/x-pack/plugin/ccr/qa/restart/build.gradle b/x-pack/plugin/ccr/qa/restart/build.gradle index d354cd911f2f8..47d37801e2dcf 100644 --- a/x-pack/plugin/ccr/qa/restart/build.gradle +++ b/x-pack/plugin/ccr/qa/restart/build.gradle @@ -1,5 +1,6 @@ import org.elasticsearch.gradle.internal.test.RestIntegTestTask import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.standalone-rest-test' @@ -22,7 +23,7 @@ def followCluster = testClusters.register('follow-cluster') { setting 'xpack.security.enabled', 'true' user username: 'admin', password: 'admin-password', role: 'superuser' setting 'cluster.remote.leader_cluster.seeds', - { "\"${leaderCluster.get().getAllTransportPortURI().get(0)}\"" } + { "\"${leaderCluster.get().getAllTransportPortURI().get(0)}\"" }, IGNORE_VALUE nameCustomization = { 'follow' } } diff --git a/x-pack/plugin/ccr/qa/security/build.gradle b/x-pack/plugin/ccr/qa/security/build.gradle index ff4a4f857fe32..5515aefeaa091 100644 --- a/x-pack/plugin/ccr/qa/security/build.gradle +++ b/x-pack/plugin/ccr/qa/security/build.gradle @@ -1,4 +1,5 @@ import org.elasticsearch.gradle.internal.test.RestIntegTestTask +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.standalone-rest-test' @@ -22,7 +23,7 @@ testClusters.register('follow-cluster') { testDistribution = 'DEFAULT' setting 'cluster.remote.leader_cluster.seeds', { "\"${leadCluster.get().getAllTransportPortURI().join(",")}\"" - } + }, IGNORE_VALUE setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' setting 'xpack.monitoring.collection.enabled', 'false' // will be enabled by tests diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java index a139b9a55f1be..7234b7babffdc 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java @@ -18,7 +18,6 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -30,11 +29,9 @@ import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.persistent.PersistentTasksExecutor; @@ -44,17 +41,13 @@ import org.elasticsearch.plugins.PersistentTaskPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.RepositoryPlugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.tasks.Task; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.ccr.action.AutoFollowCoordinator; @@ -177,44 +170,29 @@ public Ccr(final Settings settings) { @Override @SuppressWarnings("HiddenField") - public Collection createComponents( - final Client client, - final ClusterService clusterService, - final ThreadPool threadPool, - final ResourceWatcherService resourceWatcherService, - final ScriptService scriptService, - final NamedXContentRegistry xContentRegistry, - final Environment environment, - final NodeEnvironment nodeEnvironment, - final NamedWriteableRegistry namedWriteableRegistry, - final IndexNameExpressionResolver expressionResolver, - final Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - this.client = client; + public Collection createComponents(PluginServices services) { + this.client = services.client(); if (enabled == false) { return emptyList(); } - CcrSettings ccrSettings = new CcrSettings(settings, clusterService.getClusterSettings()); + CcrSettings ccrSettings = new CcrSettings(settings, services.clusterService().getClusterSettings()); this.ccrSettings.set(ccrSettings); - CcrRestoreSourceService restoreSourceService = new CcrRestoreSourceService(threadPool, ccrSettings); + CcrRestoreSourceService restoreSourceService = new CcrRestoreSourceService(services.threadPool(), ccrSettings); this.restoreSourceService.set(restoreSourceService); - return Arrays.asList( + return List.of( ccrLicenseChecker, restoreSourceService, - new CcrRepositoryManager(settings, clusterService, client), - new ShardFollowTaskCleaner(clusterService, threadPool, client), + new CcrRepositoryManager(settings, services.clusterService(), client), + new ShardFollowTaskCleaner(services.clusterService(), services.threadPool(), client), new AutoFollowCoordinator( settings, client, - clusterService, + services.clusterService(), ccrLicenseChecker, - threadPool::relativeTimeInMillis, - threadPool::absoluteTimeInMillis, - threadPool.executor(Ccr.CCR_THREAD_POOL_NAME) + services.threadPool()::relativeTimeInMillis, + services.threadPool()::absoluteTimeInMillis, + services.threadPool().executor(Ccr.CCR_THREAD_POOL_NAME) ) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index 1c8881637d4c6..d02e3f43d80cb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -13,17 +13,14 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.support.TransportAction; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.ClusterSettings; @@ -35,12 +32,10 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.Booleans; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.mapper.MetadataFieldMapper; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.license.ClusterStateLicenseService; import org.elasticsearch.license.DeleteLicenseAction; @@ -83,14 +78,10 @@ import org.elasticsearch.protocol.xpack.XPackInfoRequest; import org.elasticsearch.protocol.xpack.XPackInfoResponse; import org.elasticsearch.protocol.xpack.XPackUsageRequest; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.snapshots.sourceonly.SourceOnlySnapshotRepository; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider; @@ -321,33 +312,24 @@ public Settings additionalSettings() { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { List components = new ArrayList<>(); - final SSLService sslService = createSSLService(environment, resourceWatcherService); + final SSLService sslService = createSSLService(services.environment(), services.resourceWatcherService()); LicenseService licenseService = getLicenseService(); if (licenseService == null) { - licenseService = new ClusterStateLicenseService(settings, threadPool, clusterService, getClock(), getLicenseState()); + licenseService = new ClusterStateLicenseService( + settings, + services.threadPool(), + services.clusterService(), + getClock(), + getLicenseState() + ); setLicenseService(licenseService); } - setEpochMillisSupplier(threadPool::absoluteTimeInMillis); + setEpochMillisSupplier(services.threadPool()::absoluteTimeInMillis); // It is useful to override these as they are what guice is injecting into actions components.add(sslService); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java index afc64140004c7..26436e497a644 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.cluster.service.ClusterService; @@ -44,7 +43,6 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.IOUtils; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.http.HttpPreRequest; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.index.IndexModule; @@ -54,7 +52,6 @@ import org.elasticsearch.index.analysis.TokenizerFactory; import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.mapper.MetadataFieldMapper; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.CircuitBreakerService; @@ -81,22 +78,18 @@ import org.elasticsearch.plugins.ShutdownAwarePlugin; import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.plugins.interceptor.RestServerActionPlugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.rest.RestHeaderDefinition; import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.snapshots.Snapshot; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.telemetry.tracing.Tracer; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportInterceptor; -import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.ssl.SSLService; @@ -193,61 +186,10 @@ protected void setEpochMillisSupplier(LongSupplier epochMillisSupplier) { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - List components = new ArrayList<>( - super.createComponents( - client, - clusterService, - threadPool, - resourceWatcherService, - scriptService, - xContentRegistry, - environment, - nodeEnvironment, - namedWriteableRegistry, - expressionResolver, - repositoriesServiceSupplier, - telemetryProvider, - allocationService, - indicesService - ) - ); + public Collection createComponents(PluginServices services) { + List components = new ArrayList<>(super.createComponents(services)); - filterPlugins(Plugin.class).forEach( - p -> components.addAll( - p.createComponents( - client, - clusterService, - threadPool, - resourceWatcherService, - scriptService, - xContentRegistry, - environment, - nodeEnvironment, - namedWriteableRegistry, - expressionResolver, - repositoriesServiceSupplier, - telemetryProvider, - allocationService, - indicesService - ) - ) - ); + filterPlugins(Plugin.class).forEach(p -> components.addAll(p.createComponents(services))); return components; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackPluginTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackPluginTests.java index 4f15d719d4193..d4c8dfa5fd0a7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackPluginTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackPluginTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.license.PutLicenseRequest; import org.elasticsearch.license.internal.MutableLicenseService; import org.elasticsearch.plugins.ExtensiblePlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.protocol.xpack.license.PutLicenseResponse; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; @@ -143,22 +144,11 @@ public List loadExtensions(Class extensionPointType) { when(mockEnvironment.settings()).thenReturn(Settings.builder().build()); when(mockEnvironment.configFile()).thenReturn(PathUtils.get("")); // ensure createComponents does not influence the results - xpackPlugin.createComponents( - null, - mock(ClusterService.class), - mock(ThreadPool.class), - null, - null, - null, - mockEnvironment, - null, - null, - null, - null, - null, - null, - null - ); + Plugin.PluginServices services = mock(Plugin.PluginServices.class); + when(services.clusterService()).thenReturn(mock(ClusterService.class)); + when(services.threadPool()).thenReturn(mock(ThreadPool.class)); + when(services.environment()).thenReturn(mockEnvironment); + xpackPlugin.createComponents(services); assertEquals(license, XPackPlugin.getSharedLicenseService().getLicense()); assertEquals(License.OperationMode.resolve(licenseType), XPackPlugin.getSharedLicenseState().getOperationMode()); } @@ -197,22 +187,11 @@ public List loadExtensions(Class extensionPointType) { Environment mockEnvironment = mock(Environment.class); when(mockEnvironment.settings()).thenReturn(Settings.builder().build()); when(mockEnvironment.configFile()).thenReturn(PathUtils.get("")); - xpackPlugin.createComponents( - null, - mock(ClusterService.class), - mock(ThreadPool.class), - null, - null, - null, - mockEnvironment, - null, - null, - null, - null, - null, - null, - null - ); + Plugin.PluginServices services = mock(Plugin.PluginServices.class); + when(services.clusterService()).thenReturn(mock(ClusterService.class)); + when(services.threadPool()).thenReturn(mock(ThreadPool.class)); + when(services.environment()).thenReturn(mockEnvironment); + xpackPlugin.createComponents(services); assertThat(XPackPlugin.getSharedLicenseService(), instanceOf(ClusterStateLicenseService.class)); assertEquals(License.OperationMode.TRIAL, XPackPlugin.getSharedLicenseState().getOperationMode()); } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java index 349ff1042d3a9..dd060653a4f34 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java @@ -8,31 +8,18 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.logging.RateLimitingFilter; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.deprecation.logging.DeprecationCacheResetAction; import org.elasticsearch.xpack.deprecation.logging.DeprecationIndexingComponent; import org.elasticsearch.xpack.deprecation.logging.DeprecationIndexingTemplateRegistry; @@ -88,43 +75,29 @@ public List getRestHandlers( } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { final DeprecationIndexingTemplateRegistry templateRegistry = new DeprecationIndexingTemplateRegistry( - environment.settings(), - clusterService, - threadPool, - client, - xContentRegistry + services.environment().settings(), + services.clusterService(), + services.threadPool(), + services.client(), + services.xContentRegistry() ); templateRegistry.initialize(); final RateLimitingFilter rateLimitingFilterForIndexing = new RateLimitingFilter(); // enable on start. - rateLimitingFilterForIndexing.setUseXOpaqueId(USE_X_OPAQUE_ID_IN_FILTERING.get(environment.settings())); - clusterService.getClusterSettings() + rateLimitingFilterForIndexing.setUseXOpaqueId(USE_X_OPAQUE_ID_IN_FILTERING.get(services.environment().settings())); + services.clusterService() + .getClusterSettings() .addSettingsUpdateConsumer(USE_X_OPAQUE_ID_IN_FILTERING, rateLimitingFilterForIndexing::setUseXOpaqueId); final DeprecationIndexingComponent component = DeprecationIndexingComponent.createDeprecationIndexingComponent( - client, - environment.settings(), + services.client(), + services.environment().settings(), rateLimitingFilterForIndexing, - WRITE_DEPRECATION_LOGS_TO_INDEX.get(environment.settings()), // pass the default on startup - clusterService + WRITE_DEPRECATION_LOGS_TO_INDEX.get(services.environment().settings()), // pass the default on startup + services.clusterService() ); return List.of(component, rateLimitingFilterForIndexing); diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java index 5fb20a883560f..8e0c96c6ee245 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java @@ -8,13 +8,10 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; @@ -22,22 +19,14 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.ingest.Processor; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.IngestPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.core.XPackPlugin; @@ -189,44 +178,29 @@ public List getRestHandlers( } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { EnrichPolicyLocks enrichPolicyLocks = new EnrichPolicyLocks(); EnrichPolicyExecutor enrichPolicyExecutor = new EnrichPolicyExecutor( settings, - clusterService, - indicesService, - client, - threadPool, - expressionResolver, + services.clusterService(), + services.indicesService(), + services.client(), + services.threadPool(), + services.indexNameExpressionResolver(), enrichPolicyLocks, System::currentTimeMillis ); EnrichPolicyMaintenanceService enrichPolicyMaintenanceService = new EnrichPolicyMaintenanceService( settings, - client, - clusterService, - threadPool, + services.client(), + services.clusterService(), + services.threadPool(), enrichPolicyLocks ); enrichPolicyMaintenanceService.initialize(); return List.of( enrichPolicyLocks, - new EnrichCoordinatorProxyAction.Coordinator(client, settings), + new EnrichCoordinatorProxyAction.Coordinator(services.client(), settings), enrichPolicyMaintenanceService, enrichPolicyExecutor, enrichCache diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java index ad2c033b0ee0c..cfe6e65b20263 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java @@ -9,20 +9,13 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.logging.LogManager; @@ -31,14 +24,8 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.plugins.SystemIndexPlugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.application.analytics.AnalyticsTemplateRegistry; import org.elasticsearch.xpack.application.analytics.action.DeleteAnalyticsCollectionAction; import org.elasticsearch.xpack.application.analytics.action.GetAnalyticsCollectionAction; @@ -197,41 +184,26 @@ public List getRestHandlers( } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { if (enabled == false) { return Collections.emptyList(); } // Behavioral analytics components final AnalyticsTemplateRegistry analyticsTemplateRegistry = new AnalyticsTemplateRegistry( - clusterService, - threadPool, - client, - xContentRegistry + services.clusterService(), + services.threadPool(), + services.client(), + services.xContentRegistry() ); analyticsTemplateRegistry.initialize(); // Connector components final ConnectorTemplateRegistry connectorTemplateRegistry = new ConnectorTemplateRegistry( - clusterService, - threadPool, - client, - xContentRegistry + services.clusterService(), + services.threadPool(), + services.client(), + services.xContentRegistry() ); connectorTemplateRegistry.initialize(); diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/LocalStateEnterpriseSearch.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/LocalStateEnterpriseSearch.java index f10480e9f64a7..2e181fda1ef88 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/LocalStateEnterpriseSearch.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/LocalStateEnterpriseSearch.java @@ -9,31 +9,18 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.mockito.Mockito; @@ -102,38 +89,8 @@ public List getRestHandlers( } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - return entSearchPlugin.createComponents( - client, - clusterService, - threadPool, - resourceWatcherService, - scriptService, - xContentRegistry, - environment, - nodeEnvironment, - namedWriteableRegistry, - indexNameExpressionResolver, - repositoriesServiceSupplier, - telemetryProvider, - allocationService, - indicesService - ); + public Collection createComponents(PluginServices services) { + return entSearchPlugin.createComponents(services); } @Override diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle b/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle index 5fc247693c453..47a405517a309 100644 --- a/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle +++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle @@ -1,4 +1,5 @@ import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'elasticsearch.legacy-java-rest-test' @@ -24,7 +25,7 @@ def integTestClusterReg = testClusters.register('javaRestTest') { setting 'xpack.watcher.enabled', 'false' setting 'cluster.remote.my_remote_cluster.seeds', { remoteClusterReg.get().getAllTransportPortURI().collect { "\"$it\"" }.toString() - } + }, IGNORE_VALUE setting 'cluster.remote.connections_per_cluster', "1" setting 'xpack.security.enabled', 'true' setting 'xpack.security.autoconfiguration.enabled', 'false' diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java index d65a81db11266..881cb083a48f2 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java @@ -12,32 +12,21 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.breaker.BreakerSettings; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.monitor.jvm.JvmInfo; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.CircuitBreakerPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; @@ -71,23 +60,8 @@ public class EqlPlugin extends Plugin implements ActionPlugin, CircuitBreakerPlu public EqlPlugin() {} @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - return createComponents(client, environment.settings(), clusterService); + public Collection createComponents(PluginServices services) { + return createComponents(services.client(), services.environment().settings(), services.clusterService()); } private Collection createComponents(Client client, Settings settings, ClusterService clusterService) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractBlockBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractBlockBuilder.java index bd13a9045a28a..d6d6584e1b534 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractBlockBuilder.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractBlockBuilder.java @@ -140,8 +140,9 @@ protected final void ensureCapacity() { return; } int newSize = calculateNewArraySize(valuesLength); - adjustBreaker((long) (newSize - valuesLength) * elementSize()); + adjustBreaker(newSize * elementSize()); growValuesArray(newSize); + adjustBreaker(-valuesLength * elementSize()); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java index d7c3a5cb9bfab..b9506153aac5a 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java @@ -25,16 +25,9 @@ * or dense data. A Block can represent either single or multi valued data. A Block that represents * dense single-valued data can be viewed as a {@link Vector}. * - * TODO: update comment - *

All Blocks share the same set of data retrieval methods, but actual concrete implementations - * effectively support a subset of these, throwing {@code UnsupportedOperationException} where a - * particular data retrieval method is not supported. For example, a Block of primitive longs may - * not support retrieval as an integer, {code getInt}. This greatly simplifies Block usage and - * avoids cumbersome use-site casting. - * *

Block are immutable and can be passed between threads. */ -public interface Block extends Accountable, NamedWriteable, Releasable { +public interface Block extends Accountable, BlockLoader.Block, NamedWriteable, Releasable { /** * {@return an efficient dense single-value view of this block}. diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java index 56a4dc249388f..89b40d6e46a14 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java @@ -161,6 +161,7 @@ public static Block[] fromList(BlockFactory blockFactory, List> lis public static Block deepCopyOf(Block block, BlockFactory blockFactory) { try (Block.Builder builder = block.elementType().newBlockBuilder(block.getPositionCount(), blockFactory)) { builder.copyFrom(block, 0, block.getPositionCount()); + builder.mvOrdering(block.mvOrdering()); return builder.build(); } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java index dba0ced86e60e..00b93d08a36ff 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java @@ -182,7 +182,12 @@ public Block.Builder appendAllValuesToCurrentPosition(Block block) { @Override public Block.Builder mvOrdering(MvOrdering mvOrdering) { - throw new UnsupportedOperationException(); + /* + * This is called when copying but otherwise doesn't do + * anything because there aren't multivalue fields in a + * block containing only nulls. + */ + return this; } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java index ccd740bc91ba9..ed7e317bfc4c7 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java @@ -145,7 +145,13 @@ public Block.Builder appendAllValuesToCurrentPosition(Block block) { @Override public Block.Builder mvOrdering(MvOrdering mvOrdering) { - throw new UnsupportedOperationException("doc blocks only contain one value per position"); + /* + * This is called when copying but otherwise doesn't do + * anything because there aren't multivalue fields in a + * block containing doc references. Every position can + * only reference one doc. + */ + return this; } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/IdValueSource.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/IdValueSource.java deleted file mode 100644 index 906d6a0932806..0000000000000 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/IdValueSource.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.compute.lucene; - -import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.index.fielddata.SortedBinaryDocValues; -import org.elasticsearch.search.aggregations.support.ValuesSource; - -public class IdValueSource extends ValuesSource.Bytes { - - private final IdFieldIndexFieldData indexFieldData; - - public IdValueSource(IdFieldIndexFieldData indexFieldData) { - this.indexFieldData = indexFieldData; - } - - @Override - public SortedBinaryDocValues bytesValues(LeafReaderContext leafReaderContext) { - return indexFieldData.load(leafReaderContext).getBytesValues(); - } -} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneCountOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneCountOperator.java index 7e0b5297b629b..75bd230638928 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneCountOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneCountOperator.java @@ -13,11 +13,13 @@ import org.apache.lucene.search.Scorable; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Weight; +import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Releasables; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; @@ -62,7 +64,7 @@ public Factory( @Override public SourceOperator get(DriverContext driverContext) { - return new LuceneCountOperator(sliceQueue, limit); + return new LuceneCountOperator(driverContext.blockFactory(), sliceQueue, limit); } @Override @@ -80,8 +82,8 @@ public String describe() { } } - public LuceneCountOperator(LuceneSliceQueue sliceQueue, int limit) { - super(PAGE_SIZE, sliceQueue); + public LuceneCountOperator(BlockFactory blockFactory, LuceneSliceQueue sliceQueue, int limit) { + super(blockFactory, PAGE_SIZE, sliceQueue); this.remainingDocs = limit; this.leafCollector = new LeafCollector() { @Override @@ -155,11 +157,17 @@ public Page getOutput() { // emit only one page if (remainingDocs <= 0 && pagesEmitted == 0) { pagesEmitted++; - page = new Page( - PAGE_SIZE, - LongBlock.newConstantBlockWith(totalHits, PAGE_SIZE), - BooleanBlock.newConstantBlockWith(true, PAGE_SIZE) - ); + LongBlock count = null; + BooleanBlock seen = null; + try { + count = LongBlock.newConstantBlockWith(totalHits, PAGE_SIZE, blockFactory); + seen = BooleanBlock.newConstantBlockWith(true, PAGE_SIZE, blockFactory); + page = new Page(PAGE_SIZE, count, seen); + } finally { + if (page == null) { + Releasables.closeExpectNoException(count, seen); + } + } } return page; } catch (IOException e) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java index abb96446bb831..6536b08cd2419 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.operator.Operator; import org.elasticsearch.compute.operator.SourceOperator; import org.elasticsearch.logging.LogManager; @@ -37,6 +38,8 @@ public abstract class LuceneOperator extends SourceOperator { public static final int NO_LIMIT = Integer.MAX_VALUE; + protected final BlockFactory blockFactory; + private int processSlices; final int maxPageSize; private final LuceneSliceQueue sliceQueue; @@ -49,7 +52,8 @@ public abstract class LuceneOperator extends SourceOperator { int pagesEmitted; boolean doneCollecting; - public LuceneOperator(int maxPageSize, LuceneSliceQueue sliceQueue) { + public LuceneOperator(BlockFactory blockFactory, int maxPageSize, LuceneSliceQueue sliceQueue) { + this.blockFactory = blockFactory; this.maxPageSize = maxPageSize; this.sliceQueue = sliceQueue; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSourceOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSourceOperator.java index 0bbb6571dc4fd..7b2b276a619c6 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSourceOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSourceOperator.java @@ -11,12 +11,14 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorable; import org.apache.lucene.search.ScoreMode; +import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.DocVector; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Releasables; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; @@ -61,7 +63,7 @@ public Factory( @Override public SourceOperator get(DriverContext driverContext) { - return new LuceneSourceOperator(maxPageSize, sliceQueue, limit); + return new LuceneSourceOperator(driverContext.blockFactory(), maxPageSize, sliceQueue, limit); } @Override @@ -89,11 +91,11 @@ public String describe() { } } - public LuceneSourceOperator(int maxPageSize, LuceneSliceQueue sliceQueue, int limit) { - super(maxPageSize, sliceQueue); + public LuceneSourceOperator(BlockFactory blockFactory, int maxPageSize, LuceneSliceQueue sliceQueue, int limit) { + super(blockFactory, maxPageSize, sliceQueue); this.minPageSize = Math.max(1, maxPageSize / 2); this.remainingDocs = limit; - this.docsBuilder = IntVector.newVectorBuilder(Math.min(limit, maxPageSize)); + this.docsBuilder = IntVector.newVectorBuilder(Math.min(limit, maxPageSize), blockFactory); this.leafCollector = new LeafCollector() { @Override public void setScorer(Scorable scorer) { @@ -143,16 +145,20 @@ public Page getOutput() { Page page = null; if (currentPagePos >= minPageSize || remainingDocs <= 0 || scorer.isDone()) { pagesEmitted++; - page = new Page( - currentPagePos, - new DocVector( - IntBlock.newConstantBlockWith(scorer.shardIndex(), currentPagePos).asVector(), - IntBlock.newConstantBlockWith(scorer.leafReaderContext().ord, currentPagePos).asVector(), - docsBuilder.build(), - true - ).asBlock() - ); - docsBuilder = IntVector.newVectorBuilder(Math.min(remainingDocs, maxPageSize)); + IntBlock shard = null; + IntBlock leaf = null; + IntVector docs = null; + try { + shard = IntBlock.newConstantBlockWith(scorer.shardIndex(), currentPagePos, blockFactory); + leaf = IntBlock.newConstantBlockWith(scorer.leafReaderContext().ord, currentPagePos, blockFactory); + docs = docsBuilder.build(); + docsBuilder = IntVector.newVectorBuilder(Math.min(remainingDocs, maxPageSize), blockFactory); + page = new Page(currentPagePos, new DocVector(shard.asVector(), leaf.asVector(), docs, true).asBlock()); + } finally { + if (page == null) { + Releasables.closeExpectNoException(shard, leaf, docs); + } + } currentPagePos = 0; } return page; @@ -161,6 +167,11 @@ public Page getOutput() { } } + @Override + public void close() { + docsBuilder.close(); + } + @Override protected void describe(StringBuilder sb) { sb.append(", remainingDocs=").append(remainingDocs); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperator.java index 4c6bb50ce9f7f..4ce0af3bd0ffe 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperator.java @@ -16,12 +16,14 @@ import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.TopFieldCollector; import org.elasticsearch.common.Strings; +import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.DocVector; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Releasables; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; @@ -38,25 +40,6 @@ * Source operator that builds Pages out of the output of a TopFieldCollector (aka TopN) */ public final class LuceneTopNSourceOperator extends LuceneOperator { - /** - * Collected docs. {@code null} until we're {@link #emit(boolean)}. - */ - private ScoreDoc[] scoreDocs; - /** - * The offset in {@link #scoreDocs} of the next page. - */ - private int offset = 0; - - private PerShardCollector perShardCollector; - private final List> sorts; - private final int limit; - - public LuceneTopNSourceOperator(int maxPageSize, List> sorts, int limit, LuceneSliceQueue sliceQueue) { - super(maxPageSize, sliceQueue); - this.sorts = sorts; - this.limit = limit; - } - public static final class Factory implements LuceneOperator.Factory { private final int taskConcurrency; private final int maxPageSize; @@ -85,7 +68,7 @@ public Factory( @Override public SourceOperator get(DriverContext driverContext) { - return new LuceneTopNSourceOperator(maxPageSize, sorts, limit, sliceQueue); + return new LuceneTopNSourceOperator(driverContext.blockFactory(), maxPageSize, sorts, limit, sliceQueue); } @Override @@ -116,6 +99,31 @@ public String describe() { } } + /** + * Collected docs. {@code null} until we're {@link #emit(boolean)}. + */ + private ScoreDoc[] scoreDocs; + /** + * The offset in {@link #scoreDocs} of the next page. + */ + private int offset = 0; + + private PerShardCollector perShardCollector; + private final List> sorts; + private final int limit; + + public LuceneTopNSourceOperator( + BlockFactory blockFactory, + int maxPageSize, + List> sorts, + int limit, + LuceneSliceQueue sliceQueue + ) { + super(blockFactory, maxPageSize, sliceQueue); + this.sorts = sorts; + this.limit = limit; + } + @Override public boolean isFinished() { return doneCollecting && isEmitting() == false; @@ -187,29 +195,35 @@ private Page emit(boolean startEmitting) { return null; } int size = Math.min(maxPageSize, scoreDocs.length - offset); - IntVector.Builder currentSegmentBuilder = IntVector.newVectorBuilder(size); - IntVector.Builder currentDocsBuilder = IntVector.newVectorBuilder(size); + IntBlock shard = null; + IntVector segments = null; + IntVector docs = null; + Page page = null; + try ( + IntVector.Builder currentSegmentBuilder = IntVector.newVectorBuilder(size, blockFactory); + IntVector.Builder currentDocsBuilder = IntVector.newVectorBuilder(size, blockFactory) + ) { + int start = offset; + offset += size; + List leafContexts = perShardCollector.searchContext.searcher().getLeafContexts(); + for (int i = start; i < offset; i++) { + int doc = scoreDocs[i].doc; + int segment = ReaderUtil.subIndex(doc, leafContexts); + currentSegmentBuilder.appendInt(segment); + currentDocsBuilder.appendInt(doc - leafContexts.get(segment).docBase); // the offset inside the segment + } - int start = offset; - offset += size; - List leafContexts = perShardCollector.searchContext.searcher().getLeafContexts(); - for (int i = start; i < offset; i++) { - int doc = scoreDocs[i].doc; - int segment = ReaderUtil.subIndex(doc, leafContexts); - currentSegmentBuilder.appendInt(segment); - currentDocsBuilder.appendInt(doc - leafContexts.get(segment).docBase); // the offset inside the segment + shard = IntBlock.newConstantBlockWith(perShardCollector.shardIndex, size, blockFactory); + segments = currentSegmentBuilder.build(); + docs = currentDocsBuilder.build(); + page = new Page(size, new DocVector(shard.asVector(), segments, docs, null).asBlock()); + } finally { + if (page == null) { + Releasables.close(shard, segments, docs); + } } - pagesEmitted++; - return new Page( - size, - new DocVector( - IntBlock.newConstantBlockWith(perShardCollector.shardIndex, size).asVector(), - currentSegmentBuilder.build(), - currentDocsBuilder.build(), - null - ).asBlock() - ); + return page; } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java index 8b1c4f78825ad..61c1bd9730e02 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java @@ -53,7 +53,7 @@ public record ValuesSourceReaderOperatorFactory(List factories, int docChannel, String field) { + public ValuesSourceReaderOperator( + BlockFactory blockFactory, + List factories, + int docChannel, + String field + ) { this.factories = factories; this.docChannel = docChannel; this.field = field; - this.blockFactory = new ComputeBlockLoaderFactory(BlockFactory.getNonBreakingInstance()); // TODO breaking! + this.blockFactory = new ComputeBlockLoaderFactory(blockFactory); } @Override @@ -106,7 +111,7 @@ protected Page process(Page page) { private Block loadFromSingleLeaf(DocVector docVector) throws IOException { setupReader(docVector.shards().getInt(0), docVector.segments().getInt(0), docVector.docs().getInt(0)); - return ((Block.Builder) lastReader.readValues(blockFactory, new BlockLoader.Docs() { + return ((Block) lastReader.readValues(blockFactory, new BlockLoader.Docs() { private final IntVector docs = docVector.docs(); @Override @@ -118,26 +123,28 @@ public int count() { public int get(int i) { return docs.getInt(i); } - })).build(); + })); } private Block loadFromManyLeaves(DocVector docVector) throws IOException { int[] forwards = docVector.shardSegmentDocMapForwards(); int doc = docVector.docs().getInt(forwards[0]); setupReader(docVector.shards().getInt(forwards[0]), docVector.segments().getInt(forwards[0]), doc); - BlockLoader.Builder builder = lastReader.builder(blockFactory, forwards.length); - lastReader.readValuesFromSingleDoc(doc, builder); - for (int i = 1; i < forwards.length; i++) { - int shard = docVector.shards().getInt(forwards[i]); - int segment = docVector.segments().getInt(forwards[i]); - doc = docVector.docs().getInt(forwards[i]); - if (segment != lastSegment || shard != lastShard) { - setupReader(shard, segment, doc); - } + try (BlockLoader.Builder builder = lastReader.builder(blockFactory, forwards.length)) { lastReader.readValuesFromSingleDoc(doc, builder); + for (int i = 1; i < forwards.length; i++) { + int shard = docVector.shards().getInt(forwards[i]); + int segment = docVector.segments().getInt(forwards[i]); + doc = docVector.docs().getInt(forwards[i]); + if (segment != lastSegment || shard != lastShard) { + setupReader(shard, segment, doc); + } + lastReader.readValuesFromSingleDoc(doc, builder); + } + try (Block orig = ((Block.Builder) builder).build()) { + return orig.filter(docVector.shardSegmentDocMapBackwards()); + } } - // TODO maybe it's better for downstream consumers if we perform a copy here. - return ((Block.Builder) builder).build().filter(docVector.shardSegmentDocMapBackwards()); } private void setupReader(int shard, int segment, int doc) throws IOException { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java index 2c5eedb6d8bbe..07494f97cfd6d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java @@ -467,7 +467,7 @@ private static class ValuesAggregator implements Releasable { int maxPageSize, DriverContext driverContext ) { - this.extractor = new ValuesSourceReaderOperator(factories, docChannel, groupingField); + this.extractor = new ValuesSourceReaderOperator(BlockFactory.getNonBreakingInstance(), factories, docChannel, groupingField); this.aggregator = new HashAggregationOperator( aggregatorFactories, () -> BlockHash.build(List.of(new GroupSpec(channelIndex, groupingElementType)), driverContext, maxPageSize, false), diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java index 6951cdf4d56ca..bde2ae0a747e4 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java @@ -141,6 +141,7 @@ public void testQueryOperator() throws IOException { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/101413") public void testGroupingWithOrdinals() throws Exception { DriverContext driverContext = driverContext(); BlockFactory blockFactory = driverContext.blockFactory(); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java index 83f4a6895b154..894b94476c08d 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java @@ -29,7 +29,7 @@ import org.elasticsearch.compute.operator.NullInsertingSourceOperator; import org.elasticsearch.compute.operator.Operator; import org.elasticsearch.compute.operator.PositionMergingSourceOperator; -import org.elasticsearch.compute.operator.ResultPageSinkOperator; +import org.elasticsearch.compute.operator.TestResultPageSinkOperator; import java.util.ArrayList; import java.util.List; @@ -43,12 +43,6 @@ import static org.hamcrest.Matchers.hasSize; public abstract class AggregatorFunctionTestCase extends ForkingOperatorTestCase { - - @Override - protected DriverContext driverContext() { - return breakingDriverContext(); - } - protected abstract AggregatorFunctionSupplier aggregatorFunction(BigArrays bigArrays, List inputChannels); protected final int aggregatorIntermediateBlockCount() { @@ -110,7 +104,7 @@ public final void testIgnoresNulls() { driverContext, new NullInsertingSourceOperator(new CannedSourceOperator(input.iterator()), blockFactory), List.of(simple(nonBreakingBigArrays().withCircuitBreaking()).get(driverContext)), - new ResultPageSinkOperator(results::add), + new TestResultPageSinkOperator(results::add), () -> {} ) ) { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java index ba930e943e79a..753b5878de2ae 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java @@ -48,12 +48,6 @@ import static org.hamcrest.Matchers.hasSize; public abstract class GroupingAggregatorFunctionTestCase extends ForkingOperatorTestCase { - - @Override - protected DriverContext driverContext() { - return breakingDriverContext(); - } - protected abstract AggregatorFunctionSupplier aggregatorFunction(BigArrays bigArrays, List inputChannels); protected final int aggregatorIntermediateBlockCount() { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/SumDoubleAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/SumDoubleAggregatorFunctionTests.java index b08bda4a71d38..ea428d7d87cad 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/SumDoubleAggregatorFunctionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/SumDoubleAggregatorFunctionTests.java @@ -14,9 +14,9 @@ import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.Driver; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.compute.operator.ResultPageSinkOperator; import org.elasticsearch.compute.operator.SequenceDoubleBlockSourceOperator; import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.compute.operator.TestResultPageSinkOperator; import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; @@ -57,7 +57,7 @@ public void testOverflowSucceeds() { driverContext, new SequenceDoubleBlockSourceOperator(driverContext.blockFactory(), DoubleStream.of(Double.MAX_VALUE - 1, 2)), List.of(simple(nonBreakingBigArrays()).get(driverContext)), - new ResultPageSinkOperator(results::add), + new TestResultPageSinkOperator(results::add), () -> {} ) ) { @@ -78,7 +78,7 @@ public void testSummationAccuracy() { DoubleStream.of(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7) ), List.of(simple(nonBreakingBigArrays()).get(driverContext)), - new ResultPageSinkOperator(results::add), + new TestResultPageSinkOperator(results::add), () -> {} ) ) { @@ -104,7 +104,7 @@ public void testSummationAccuracy() { driverContext, new SequenceDoubleBlockSourceOperator(driverContext.blockFactory(), DoubleStream.of(values)), List.of(simple(nonBreakingBigArrays()).get(driverContext)), - new ResultPageSinkOperator(results::add), + new TestResultPageSinkOperator(results::add), () -> {} ) ) { @@ -126,7 +126,7 @@ public void testSummationAccuracy() { driverContext, new SequenceDoubleBlockSourceOperator(driverContext.blockFactory(), DoubleStream.of(largeValues)), List.of(simple(nonBreakingBigArrays()).get(driverContext)), - new ResultPageSinkOperator(results::add), + new TestResultPageSinkOperator(results::add), () -> {} ) ) { @@ -145,7 +145,7 @@ public void testSummationAccuracy() { driverContext, new SequenceDoubleBlockSourceOperator(driverContext.blockFactory(), DoubleStream.of(largeValues)), List.of(simple(nonBreakingBigArrays()).get(driverContext)), - new ResultPageSinkOperator(results::add), + new TestResultPageSinkOperator(results::add), () -> {} ) ) { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneCountOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneCountOperatorTests.java index 0696d02d87af6..d6edc903607cc 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneCountOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneCountOperatorTests.java @@ -16,6 +16,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; +import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.LongBlock; @@ -24,10 +25,11 @@ import org.elasticsearch.compute.operator.Driver; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.OperatorTestCase; -import org.elasticsearch.compute.operator.PageConsumerOperator; +import org.elasticsearch.compute.operator.TestResultPageSinkOperator; import org.elasticsearch.core.IOUtils; import org.elasticsearch.index.cache.query.TrivialQueryCachingPolicy; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.indices.CrankyCircuitBreakerService; import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.search.internal.SearchContext; import org.junit.After; @@ -37,6 +39,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -115,25 +118,53 @@ protected String expectedDescriptionOfSimple() { // TODO tests for the other data partitioning configurations public void testSimple() { + testSimple(this::driverContext); + } + + public void testSimpleWithCranky() { + try { + testSimple(this::crankyDriverContext); + logger.info("cranky didn't break"); + } catch (CircuitBreakingException e) { + logger.info("broken", e); + assertThat(e.getMessage(), equalTo(CrankyCircuitBreakerService.ERROR_MESSAGE)); + } + } + + private void testSimple(Supplier contexts) { int size = between(1_000, 20_000); int limit = randomBoolean() ? between(10, size) : Integer.MAX_VALUE; - testCount(size, limit); + testCount(contexts, size, limit); } public void testEmpty() { + testEmpty(this::driverContext); + } + + public void testEmptyWithCranky() { + try { + testEmpty(this::crankyDriverContext); + logger.info("cranky didn't break"); + } catch (CircuitBreakingException e) { + logger.info("broken", e); + assertThat(e.getMessage(), equalTo(CrankyCircuitBreakerService.ERROR_MESSAGE)); + } + } + + private void testEmpty(Supplier contexts) { int limit = randomBoolean() ? between(10, 10000) : Integer.MAX_VALUE; - testCount(0, limit); + testCount(contexts, 0, limit); } - private void testCount(int size, int limit) { + private void testCount(Supplier contexts, int size, int limit) { DataPartitioning dataPartitioning = randomFrom(DataPartitioning.values()); - LuceneCountOperator.Factory factory = simple(nonBreakingBigArrays(), dataPartitioning, size, limit); + LuceneCountOperator.Factory factory = simple(contexts.get().bigArrays(), dataPartitioning, size, limit); List results = new CopyOnWriteArrayList<>(); List drivers = new ArrayList<>(); int taskConcurrency = between(1, 8); for (int i = 0; i < taskConcurrency; i++) { - DriverContext ctx = driverContext(); - drivers.add(new Driver(ctx, factory.get(ctx), List.of(), new PageConsumerOperator(results::add), () -> {})); + DriverContext ctx = contexts.get(); + drivers.add(new Driver(ctx, factory.get(ctx), List.of(), new TestResultPageSinkOperator(results::add), () -> {})); } OperatorTestCase.runDriver(drivers); assertThat(results.size(), lessThanOrEqualTo(taskConcurrency)); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java index 131082859bf4c..41fe1a93d9c8b 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java @@ -16,6 +16,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; +import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; @@ -24,7 +25,7 @@ import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.Operator; import org.elasticsearch.compute.operator.OperatorTestCase; -import org.elasticsearch.compute.operator.PageConsumerOperator; +import org.elasticsearch.compute.operator.TestResultPageSinkOperator; import org.elasticsearch.core.IOUtils; import org.elasticsearch.index.cache.query.TrivialQueryCachingPolicy; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -35,6 +36,7 @@ import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.support.NestedScope; +import org.elasticsearch.indices.CrankyCircuitBreakerService; import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.search.internal.SearchContext; import org.junit.After; @@ -46,6 +48,7 @@ import java.util.function.Function; import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.lessThan; @@ -133,21 +136,53 @@ protected String expectedDescriptionOfSimple() { public void testShardDataPartitioning() { int size = between(1_000, 20_000); int limit = between(10, size); - testSimple(size, limit); + testSimple(driverContext(), size, limit); } public void testEmpty() { - testSimple(0, between(10, 10_000)); + testSimple(driverContext(), 0, between(10, 10_000)); } - private void testSimple(int size, int limit) { - DriverContext ctx = driverContext(); - LuceneSourceOperator.Factory factory = simple(nonBreakingBigArrays(), DataPartitioning.SHARD, size, limit); + public void testWithCranky() { + try { + testSimple(crankyDriverContext(), between(1, 10_000), 100); + logger.info("cranky didn't break"); + } catch (CircuitBreakingException e) { + logger.info("broken", e); + assertThat(e.getMessage(), equalTo(CrankyCircuitBreakerService.ERROR_MESSAGE)); + } + } + + public void testEmptyWithCranky() { + try { + testSimple(crankyDriverContext(), 0, between(10, 10_000)); + logger.info("cranky didn't break"); + } catch (CircuitBreakingException e) { + logger.info("broken", e); + assertThat(e.getMessage(), equalTo(CrankyCircuitBreakerService.ERROR_MESSAGE)); + } + } + + public void testShardDataPartitioningWithCranky() { + int size = between(1_000, 20_000); + int limit = between(10, size); + try { + testSimple(crankyDriverContext(), size, limit); + logger.info("cranky didn't break"); + } catch (CircuitBreakingException e) { + logger.info("broken", e); + assertThat(e.getMessage(), equalTo(CrankyCircuitBreakerService.ERROR_MESSAGE)); + } + } + + private void testSimple(DriverContext ctx, int size, int limit) { + LuceneSourceOperator.Factory factory = simple(ctx.bigArrays(), DataPartitioning.SHARD, size, limit); Operator.OperatorFactory readS = ValuesSourceReaderOperatorTests.factory(reader, S_FIELD); List results = new ArrayList<>(); + OperatorTestCase.runDriver( - new Driver(ctx, factory.get(ctx), List.of(readS.get(ctx)), new PageConsumerOperator(page -> results.add(page)), () -> {}) + new Driver(ctx, factory.get(ctx), List.of(readS.get(ctx)), new TestResultPageSinkOperator(results::add), () -> {}) ); OperatorTestCase.assertDriverContext(ctx); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperatorTests.java index f0eb49c233e7a..d1b9e706750df 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperatorTests.java @@ -15,6 +15,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; +import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; @@ -23,7 +24,7 @@ import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.Operator; import org.elasticsearch.compute.operator.OperatorTestCase; -import org.elasticsearch.compute.operator.PageConsumerOperator; +import org.elasticsearch.compute.operator.TestResultPageSinkOperator; import org.elasticsearch.core.IOUtils; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -33,6 +34,7 @@ import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.support.NestedScope; +import org.elasticsearch.indices.CrankyCircuitBreakerService; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortBuilder; @@ -136,23 +138,50 @@ protected String expectedDescriptionOfSimple() { // TODO tests for the other data partitioning configurations public void testShardDataPartitioning() { + testShardDataPartitioning(driverContext()); + } + + public void testShardDataPartitioningWithCranky() { + try { + testShardDataPartitioning(crankyDriverContext()); + logger.info("cranky didn't break"); + } catch (CircuitBreakingException e) { + logger.info("broken", e); + assertThat(e.getMessage(), equalTo(CrankyCircuitBreakerService.ERROR_MESSAGE)); + } + } + + private void testShardDataPartitioning(DriverContext context) { int size = between(1_000, 20_000); int limit = between(10, size); - testSimple(size, limit); + testSimple(context, size, limit); } public void testEmpty() { - testSimple(0, between(10, 10_000)); + testEmpty(driverContext()); + } + + public void testEmptyWithCranky() { + try { + testEmpty(crankyDriverContext()); + logger.info("cranky didn't break"); + } catch (CircuitBreakingException e) { + logger.info("broken", e); + assertThat(e.getMessage(), equalTo(CrankyCircuitBreakerService.ERROR_MESSAGE)); + } + } + + private void testEmpty(DriverContext context) { + testSimple(context, 0, between(10, 10_000)); } - private void testSimple(int size, int limit) { - DriverContext ctx = driverContext(); - LuceneTopNSourceOperator.Factory factory = simple(nonBreakingBigArrays(), DataPartitioning.SHARD, size, limit); + private void testSimple(DriverContext ctx, int size, int limit) { + LuceneTopNSourceOperator.Factory factory = simple(ctx.bigArrays(), DataPartitioning.SHARD, size, limit); Operator.OperatorFactory readS = ValuesSourceReaderOperatorTests.factory(reader, S_FIELD); List results = new ArrayList<>(); OperatorTestCase.runDriver( - new Driver(ctx, factory.get(ctx), List.of(readS.get(ctx)), new PageConsumerOperator(page -> results.add(page)), () -> {}) + new Driver(ctx, factory.get(ctx), List.of(readS.get(ctx)), new TestResultPageSinkOperator(results::add), () -> {}) ); OperatorTestCase.assertDriverContext(ctx); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java index a6e5e3bf4744d..269a478560bac 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java @@ -223,7 +223,6 @@ public void testLoadAllInOnePageShuffled() { } private void loadSimpleAndAssert(DriverContext driverContext, List input) { - List results = new ArrayList<>(); List operators = List.of( factory(reader, new NumberFieldMapper.NumberFieldType("key", NumberFieldMapper.NumberType.INTEGER)).get(driverContext), factory(reader, new NumberFieldMapper.NumberFieldType("long", NumberFieldMapper.NumberType.LONG)).get(driverContext), @@ -236,17 +235,7 @@ private void loadSimpleAndAssert(DriverContext driverContext, List input) factory(reader, new NumberFieldMapper.NumberFieldType("double", NumberFieldMapper.NumberType.DOUBLE)).get(driverContext), factory(reader, new NumberFieldMapper.NumberFieldType("mv_double", NumberFieldMapper.NumberType.DOUBLE)).get(driverContext) ); - try ( - Driver d = new Driver( - driverContext, - new CannedSourceOperator(input.iterator()), - operators, - new PageConsumerOperator(page -> results.add(page)), - () -> {} - ) - ) { - runDriver(d); - } + List results = drive(operators, input.iterator(), driverContext); assertThat(results, hasSize(input.size())); for (Page p : results) { assertThat(p.getBlockCount(), equalTo(11)); @@ -368,20 +357,24 @@ public void testValuesSourceReaderOperatorWithNulls() throws IOException { factory(reader, kwFt).get(driverContext) ), new PageConsumerOperator(page -> { - logger.debug("New page: {}", page); - IntBlock intValuesBlock = page.getBlock(1); - LongBlock longValuesBlock = page.getBlock(2); - DoubleBlock doubleValuesBlock = page.getBlock(3); - BytesRefBlock keywordValuesBlock = page.getBlock(4); + try { + logger.debug("New page: {}", page); + IntBlock intValuesBlock = page.getBlock(1); + LongBlock longValuesBlock = page.getBlock(2); + DoubleBlock doubleValuesBlock = page.getBlock(3); + BytesRefBlock keywordValuesBlock = page.getBlock(4); - for (int i = 0; i < page.getPositionCount(); i++) { - assertFalse(intValuesBlock.isNull(i)); - long j = intValuesBlock.getInt(i); - // Every 100 documents we set fields to null - boolean fieldIsEmpty = j % 100 == 0; - assertEquals(fieldIsEmpty, longValuesBlock.isNull(i)); - assertEquals(fieldIsEmpty, doubleValuesBlock.isNull(i)); - assertEquals(fieldIsEmpty, keywordValuesBlock.isNull(i)); + for (int i = 0; i < page.getPositionCount(); i++) { + assertFalse(intValuesBlock.isNull(i)); + long j = intValuesBlock.getInt(i); + // Every 100 documents we set fields to null + boolean fieldIsEmpty = j % 100 == 0; + assertEquals(fieldIsEmpty, longValuesBlock.isNull(i)); + assertEquals(fieldIsEmpty, doubleValuesBlock.isNull(i)); + assertEquals(fieldIsEmpty, keywordValuesBlock.isNull(i)); + } + } finally { + page.releaseBlocks(); } }), () -> {} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AnyOperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AnyOperatorTestCase.java index 00c054ef0031a..290756e81cfae 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AnyOperatorTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AnyOperatorTestCase.java @@ -16,6 +16,7 @@ import org.elasticsearch.compute.aggregation.GroupingAggregatorFunction; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.MockBlockFactory; +import org.elasticsearch.indices.CrankyCircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.test.ESTestCase; import org.junit.After; @@ -34,7 +35,7 @@ public abstract class AnyOperatorTestCase extends ESTestCase { * The operator configured a "simple" or basic way, used for smoke testing * descriptions and {@link BigArrays} and scatter/gather. */ - protected abstract Operator.OperatorFactory simple(BigArrays bigArrays); + protected abstract Operator.OperatorFactory simple(BigArrays bigArrays); // TODO remove BigArrays - that's part of the context /** * The description of the operator produced by {@link #simple}. @@ -100,23 +101,30 @@ protected final BigArrays nonBreakingBigArrays() { /** * A {@link DriverContext} with a nonBreakingBigArrays. */ - protected DriverContext driverContext() { // TODO make this final and return a breaking block factory + protected DriverContext driverContext() { // TODO make this final once all operators support memory tracking + BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, ByteSizeValue.ofGb(1)).withCircuitBreaking(); + CircuitBreaker breaker = bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST); + breakers.add(breaker); + BlockFactory factory = new MockBlockFactory(breaker, bigArrays); + blockFactories.add(factory); + return new DriverContext(bigArrays, factory); + } + + protected final DriverContext nonBreakingDriverContext() { // TODO drop this once the driverContext method isn't overrideable return new DriverContext(nonBreakingBigArrays(), BlockFactory.getNonBreakingInstance()); } private final List breakers = new ArrayList<>(); private final List blockFactories = new ArrayList<>(); - /** - * A {@link DriverContext} with a breaking {@link BigArrays} and {@link BlockFactory}. - */ - protected DriverContext breakingDriverContext() { // TODO move this to driverContext once everyone supports breaking - BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, ByteSizeValue.ofGb(1)).withCircuitBreaking(); + protected final DriverContext crankyDriverContext() { + CrankyCircuitBreakerService cranky = new CrankyCircuitBreakerService(); + BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, cranky).withCircuitBreaking(); CircuitBreaker breaker = bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST); breakers.add(breaker); - BlockFactory factory = new MockBlockFactory(breaker, bigArrays); - blockFactories.add(factory); - return new DriverContext(bigArrays, factory); + BlockFactory blockFactory = new MockBlockFactory(breaker, bigArrays); + blockFactories.add(blockFactory); + return new DriverContext(bigArrays, blockFactory); } @After diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/CannedSourceOperator.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/CannedSourceOperator.java index f5dd2680e0ac7..47febc09e45f5 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/CannedSourceOperator.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/CannedSourceOperator.java @@ -7,8 +7,11 @@ package org.elasticsearch.compute.operator; +import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.Page; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; import java.util.ArrayList; import java.util.Iterator; @@ -42,19 +45,32 @@ public static Page mergePages(List pages) { int totalPositions = pages.stream().mapToInt(Page::getPositionCount).sum(); Page first = pages.get(0); Block.Builder[] builders = new Block.Builder[first.getBlockCount()]; - for (int b = 0; b < builders.length; b++) { - builders[b] = first.getBlock(b).elementType().newBlockBuilder(totalPositions); - } - for (Page p : pages) { + try { for (int b = 0; b < builders.length; b++) { - builders[b].copyFrom(p.getBlock(b), 0, p.getPositionCount()); + builders[b] = first.getBlock(b).elementType().newBlockBuilder(totalPositions); } + for (Page p : pages) { + for (int b = 0; b < builders.length; b++) { + builders[b].copyFrom(p.getBlock(b), 0, p.getPositionCount()); + } + } + Block[] blocks = new Block[builders.length]; + Page result = null; + try { + for (int b = 0; b < blocks.length; b++) { + blocks[b] = builders[b].build(); + } + result = new Page(blocks); + } finally { + if (result == null) { + Releasables.close(blocks); + } + } + return result; + } finally { + Iterable releasePages = () -> Iterators.map(pages.iterator(), p -> p::releaseBlocks); + Releasables.closeExpectNoException(Releasables.wrap(builders), Releasables.wrap(releasePages)); } - Block[] blocks = new Block[builders.length]; - for (int b = 0; b < blocks.length; b++) { - blocks[b] = builders[b].build(); - } - return new Page(blocks); } /** diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/EvalOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/EvalOperatorTests.java index 95a5647717851..e7f5db7579869 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/EvalOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/EvalOperatorTests.java @@ -118,9 +118,4 @@ public void testReadFromBlock() { protected ByteSizeValue smallEnoughToCircuitBreak() { return ByteSizeValue.ofBytes(between(1, 8000)); } - - @Override - protected DriverContext driverContext() { // TODO remove this when the parent uses a breaking block factory - return breakingDriverContext(); - } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FilterOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FilterOperatorTests.java index fa4c7bea7c9cc..e16f643e1ca4d 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FilterOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FilterOperatorTests.java @@ -119,9 +119,4 @@ public void testReadFromBlock() { protected ByteSizeValue smallEnoughToCircuitBreak() { return ByteSizeValue.ofBytes(between(1, 600)); } - - @Override - protected DriverContext driverContext() { // TODO remove this when the parent uses a breaking block factory - return breakingDriverContext(); - } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java index 1a2c87aff1591..9403d22f2b4c4 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java @@ -71,7 +71,7 @@ public final void testInitialFinal() { simpleWithMode(bigArrays, AggregatorMode.INITIAL).get(driverContext), simpleWithMode(bigArrays, AggregatorMode.FINAL).get(driverContext) ), - new ResultPageSinkOperator(page -> results.add(page)), + new TestResultPageSinkOperator(page -> results.add(page)), () -> {} ) ) { @@ -93,7 +93,7 @@ public final void testManyInitialFinal() { driverContext, new CannedSourceOperator(partials.iterator()), List.of(simpleWithMode(bigArrays, AggregatorMode.FINAL).get(driverContext)), - new ResultPageSinkOperator(results::add), + new TestResultPageSinkOperator(results::add), () -> {} ) ) { @@ -119,7 +119,7 @@ public final void testInitialIntermediateFinal() { simpleWithMode(bigArrays, AggregatorMode.INTERMEDIATE).get(driverContext), simpleWithMode(bigArrays, AggregatorMode.FINAL).get(driverContext) ), - new ResultPageSinkOperator(page -> results.add(page)), + new TestResultPageSinkOperator(page -> results.add(page)), () -> {} ) ) { @@ -148,7 +148,7 @@ public final void testManyInitialManyPartialFinal() { driverContext, new CannedSourceOperator(intermediates.iterator()), List.of(simpleWithMode(bigArrays, AggregatorMode.FINAL).get(driverContext)), - new ResultPageSinkOperator(results::add), + new TestResultPageSinkOperator(results::add), () -> {} ) ) { @@ -255,7 +255,7 @@ List createDriversForInput(BigArrays bigArrays, List input, List

{} ) ); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/HashAggregationOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/HashAggregationOperatorTests.java index afa307c494431..b1ef784ca339c 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/HashAggregationOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/HashAggregationOperatorTests.java @@ -31,12 +31,6 @@ import static org.hamcrest.Matchers.hasSize; public class HashAggregationOperatorTests extends ForkingOperatorTestCase { - - @Override - protected DriverContext driverContext() { - return breakingDriverContext(); - } - @Override protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { long max = randomLongBetween(1, Long.MAX_VALUE / size); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LimitOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LimitOperatorTests.java index 76f99389a697b..8c85f5927196f 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LimitOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LimitOperatorTests.java @@ -23,11 +23,6 @@ import static org.hamcrest.Matchers.sameInstance; public class LimitOperatorTests extends OperatorTestCase { - @Override - protected DriverContext driverContext() { - return breakingDriverContext(); - } - @Override protected LimitOperator.Factory simple(BigArrays bigArrays) { return new LimitOperator.Factory(100); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorTests.java index a105818a2bf03..3572dc620287d 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorTests.java @@ -257,9 +257,4 @@ protected Page createPage(int positionOffset, int length) { List results = drive(new MvExpandOperator(0, randomIntBetween(1, 1000)), input.iterator(), context); assertSimpleOutput(origInput, results); } - - @Override - protected DriverContext driverContext() { - return breakingDriverContext(); - } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java index 50b97021c216b..fe6061ee90779 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java @@ -124,14 +124,11 @@ public final void testSimpleWithCranky() { DriverContext inputFactoryContext = driverContext(); List input = CannedSourceOperator.collectPages(simpleInput(inputFactoryContext.blockFactory(), between(1_000, 10_000))); - CrankyCircuitBreakerService cranky = new CrankyCircuitBreakerService(); - BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, cranky).withCircuitBreaking(); - BlockFactory blockFactory = BlockFactory.getInstance(cranky.getBreaker(CircuitBreaker.REQUEST), bigArrays); - DriverContext driverContext = new DriverContext(bigArrays, blockFactory); + DriverContext driverContext = crankyDriverContext(); boolean driverStarted = false; try { - Operator operator = simple(bigArrays).get(driverContext); + Operator operator = simple(driverContext.bigArrays()).get(driverContext); driverStarted = true; drive(operator, input.iterator(), driverContext); // Either we get lucky and cranky doesn't throw and the test completes or we don't and it throws @@ -145,7 +142,6 @@ public final void testSimpleWithCranky() { } // Note the lack of try/finally here - we're asserting that when the driver throws an exception we clear the breakers. - assertThat(bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L)); assertThat(inputFactoryContext.breaker().getUsed(), equalTo(0L)); } @@ -248,7 +244,7 @@ protected final List drive(List operators, Iterator input, driverContext, new CannedSourceOperator(input), operators, - new ResultPageSinkOperator(results::add), + new TestResultPageSinkOperator(results::add), () -> {} ) ) { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ProjectOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ProjectOperatorTests.java index 1acdbc4895c94..1aff6be7594aa 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ProjectOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ProjectOperatorTests.java @@ -28,11 +28,6 @@ import static org.mockito.Mockito.when; public class ProjectOperatorTests extends OperatorTestCase { - @Override - protected DriverContext driverContext() { - return breakingDriverContext(); - } - public void testProjectionOnEmptyPage() { var page = new Page(0); var projection = new ProjectOperator(randomProjection(10)); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/StringExtractOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/StringExtractOperatorTests.java index 5dff00ec930c4..55470d1dcae12 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/StringExtractOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/StringExtractOperatorTests.java @@ -126,4 +126,9 @@ public void close() {} assertThat(brb.getBytesRef(idx + 1, spare).utf8ToString(), equalTo("foo4")); assertThat(brb.getBytesRef(idx + 2, spare).utf8ToString(), equalTo("foo5")); } + + @Override + protected DriverContext driverContext() { + return nonBreakingDriverContext(); + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ResultPageSinkOperator.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/TestResultPageSinkOperator.java similarity index 85% rename from x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ResultPageSinkOperator.java rename to x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/TestResultPageSinkOperator.java index 69e7a78ae5c97..aaa3a6ac8a3c8 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ResultPageSinkOperator.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/TestResultPageSinkOperator.java @@ -17,9 +17,9 @@ * Page Consumer operator that deep copies the input page, closes it, and then passes the copy * to the underlying page consumer. */ -public class ResultPageSinkOperator extends PageConsumerOperator { +public class TestResultPageSinkOperator extends PageConsumerOperator { - public ResultPageSinkOperator(Consumer pageConsumer) { + public TestResultPageSinkOperator(Consumer pageConsumer) { super(page -> { Page copy = BlockTestUtils.deepCopyOf(page, BlockFactory.getNonBreakingInstance()); page.releaseBlocks(); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNOperatorTests.java index 049bc449069b8..6c5bab9b8f784 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNOperatorTests.java @@ -1359,11 +1359,6 @@ public void testCloseWithoutCompleting() { } } - @Override - protected DriverContext driverContext() { // TODO remove this when the parent uses a breaking block factory - return breakingDriverContext(); - } - @SuppressWarnings({ "unchecked", "rawtypes" }) private static void readAsRows(List>> values, Page page) { if (page.getBlockCount() == 0) { diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/HeapAttackIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/HeapAttackIT.java index eed96a007a1b9..31d0a7646e1b7 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/HeapAttackIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/HeapAttackIT.java @@ -290,7 +290,6 @@ public void testFetchManyBigFields() throws IOException { fetchManyBigFields(100); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/100528") public void testFetchTooManyBigFields() throws IOException { initManyBigFieldsIndex(500); assertCircuitBreaks(() -> fetchManyBigFields(500)); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec index f4eaa44f15408..8091f7a18463e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec @@ -48,6 +48,13 @@ a:keyword | b:keyword | c:keyword | d:keyword foo 1 bar 2 baz | foo,bar,baz | 1 | 2 ; +appendSeparatorUppercase +row a = "foo 1 bar 2 baz" | dissect a "%{+b} %{c} %{+b} %{d} %{+b}" APPEND_SEPARATOR=","; + +a:keyword | b:keyword | c:keyword | d:keyword +foo 1 bar 2 baz | foo,bar,baz | 1 | 2 +; + namedSkip row a = "foo bar baz" | dissect a "%{b} %{?c} %{d}"; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java index bf246b5ac02d4..98c1397d97860 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java @@ -255,7 +255,9 @@ private void doLookup( extractField instanceof Alias a ? ((NamedExpression) a.child()).name() : extractField.name(), EsqlDataTypes.isUnsupported(extractField.dataType()) ); - intermediateOperators.add(new ValuesSourceReaderOperator(sources, 0, extractField.name())); + intermediateOperators.add( + new ValuesSourceReaderOperator(BlockFactory.getNonBreakingInstance(), sources, 0, extractField.name()) + ); } // drop docs block intermediateOperators.add(droppingBlockOperator(extractFields.size() + 2, 0)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index 418e11f248ab7..3b1ab30f0bd80 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -110,7 +110,7 @@ public PlanFactory visitDissectCommand(EsqlBaseParser.DissectCommandContext ctx) Map options = visitCommandOptions(ctx.commandOptions()); String appendSeparator = ""; for (Map.Entry item : options.entrySet()) { - if (item.getKey().equals("append_separator") == false) { + if (item.getKey().equalsIgnoreCase("append_separator") == false) { throw new ParsingException(source(ctx), "Invalid option for dissect: [{}]", item.getKey()); } if (item.getValue() instanceof String == false) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java index fba2f00f0b314..4c05a70e607f9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java @@ -8,11 +8,8 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; @@ -33,21 +30,13 @@ import org.elasticsearch.compute.operator.exchange.ExchangeSinkOperator; import org.elasticsearch.compute.operator.exchange.ExchangeSourceOperator; import org.elasticsearch.compute.operator.topn.TopNOperatorStatus; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; import org.elasticsearch.xpack.esql.EsqlInfoTransportAction; @@ -88,28 +77,25 @@ public class EsqlPlugin extends Plugin implements ActionPlugin { ); @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - CircuitBreaker circuitBreaker = indicesService.getBigArrays().breakerService().getBreaker("request"); + public Collection createComponents(PluginServices services) { + CircuitBreaker circuitBreaker = services.indicesService().getBigArrays().breakerService().getBreaker("request"); Objects.requireNonNull(circuitBreaker, "request circuit breaker wasn't set"); - BlockFactory blockFactory = new BlockFactory(circuitBreaker, indicesService.getBigArrays().withCircuitBreaking()); + BlockFactory blockFactory = new BlockFactory(circuitBreaker, services.indicesService().getBigArrays().withCircuitBreaking()); return List.of( - new PlanExecutor(new IndexResolver(client, clusterService.getClusterName().value(), EsqlDataTypeRegistry.INSTANCE, Set::of)), - new ExchangeService(clusterService.getSettings(), threadPool, EsqlPlugin.ESQL_THREAD_POOL_NAME, blockFactory), + new PlanExecutor( + new IndexResolver( + services.client(), + services.clusterService().getClusterName().value(), + EsqlDataTypeRegistry.INSTANCE, + Set::of + ) + ), + new ExchangeService( + services.clusterService().getSettings(), + services.threadPool(), + EsqlPlugin.ESQL_THREAD_POOL_NAME, + blockFactory + ), blockFactory ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 5047688610120..985127731a0d1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -613,12 +613,14 @@ public void testDissectPattern() { assertEquals("", dissect.parser().appendSeparator()); assertEquals(List.of(referenceAttribute("foo", KEYWORD)), dissect.extractedFields()); - cmd = processingCommand("dissect a \"%{foo}\" append_separator=\",\""); - assertEquals(Dissect.class, cmd.getClass()); - dissect = (Dissect) cmd; - assertEquals("%{foo}", dissect.parser().pattern()); - assertEquals(",", dissect.parser().appendSeparator()); - assertEquals(List.of(referenceAttribute("foo", KEYWORD)), dissect.extractedFields()); + for (String separatorName : List.of("append_separator", "APPEND_SEPARATOR", "AppEnd_SeparAtor")) { + cmd = processingCommand("dissect a \"%{foo}\" " + separatorName + "=\",\""); + assertEquals(Dissect.class, cmd.getClass()); + dissect = (Dissect) cmd; + assertEquals("%{foo}", dissect.parser().pattern()); + assertEquals(",", dissect.parser().appendSeparator()); + assertEquals(List.of(referenceAttribute("foo", KEYWORD)), dissect.extractedFields()); + } for (Tuple queryWithUnexpectedCmd : List.of( Tuple.tuple("from a | dissect foo \"\"", "[]"), diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index 21f0e846ebfa6..fb8d68541d4d1 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -23,30 +23,19 @@ import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.indices.ExecutorNames; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndexDescriptor.Type; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; @@ -100,28 +89,13 @@ public class Fleet extends Plugin implements SystemIndexPlugin { private static final int FLEET_ACTIONS_RESULTS_MAPPINGS_VERSION = 1; @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { FleetTemplateRegistry registry = new FleetTemplateRegistry( - environment.settings(), - clusterService, - threadPool, - client, - xContentRegistry + services.environment().settings(), + services.clusterService(), + services.threadPool(), + services.client(), + services.xContentRegistry() ); registry.initialize(); return List.of(); diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java index 943394f326653..c4c91dac9a513 100644 --- a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java +++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java @@ -11,31 +11,18 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.ssl.X509KeyPairSettings; @@ -86,35 +73,24 @@ public class IdentityProviderPlugin extends Plugin implements ActionPlugin { private Settings settings; @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - settings = environment.settings(); + public Collection createComponents(PluginServices services) { + settings = services.environment().settings(); enabled = ENABLED_SETTING.get(settings); if (enabled == false) { return List.of(); } SamlInit.initialize(); - final SamlServiceProviderIndex index = new SamlServiceProviderIndex(client, clusterService); - final SecurityContext securityContext = new SecurityContext(settings, threadPool.getThreadContext()); + final SamlServiceProviderIndex index = new SamlServiceProviderIndex(services.client(), services.clusterService()); + final SecurityContext securityContext = new SecurityContext(settings, services.threadPool().getThreadContext()); final ServiceProviderDefaults serviceProviderDefaults = ServiceProviderDefaults.forSettings(settings); - final ApplicationActionsResolver actionsResolver = new ApplicationActionsResolver(settings, serviceProviderDefaults, client); - final UserPrivilegeResolver userPrivilegeResolver = new UserPrivilegeResolver(client, securityContext, actionsResolver); + final ApplicationActionsResolver actionsResolver = new ApplicationActionsResolver( + settings, + serviceProviderDefaults, + services.client() + ); + final UserPrivilegeResolver userPrivilegeResolver = new UserPrivilegeResolver(services.client(), securityContext, actionsResolver); final SamlServiceProviderFactory serviceProviderFactory = new SamlServiceProviderFactory(serviceProviderDefaults); final SamlServiceProviderResolver registeredServiceProviderResolver = new SamlServiceProviderResolver( @@ -123,13 +99,13 @@ public Collection createComponents( serviceProviderFactory ); final WildcardServiceProviderResolver wildcardServiceProviderResolver = WildcardServiceProviderResolver.create( - environment, - resourceWatcherService, - scriptService, + services.environment(), + services.resourceWatcherService(), + services.scriptService(), serviceProviderFactory ); final SamlIdentityProvider idp = SamlIdentityProvider.builder(registeredServiceProviderResolver, wildcardServiceProviderResolver) - .fromSettings(environment) + .fromSettings(services.environment()) .serviceProviderDefaults(serviceProviderDefaults) .build(); diff --git a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle index a1a2a5129e3f7..111496669afe3 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle @@ -1,5 +1,6 @@ import org.elasticsearch.gradle.internal.test.RestIntegTestTask import org.elasticsearch.gradle.internal.info.BuildParams +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.standalone-rest-test' @@ -16,12 +17,12 @@ tasks.register('leader-cluster', RestIntegTestTask) { mustRunAfter("precommit") systemProperty 'tests.target_cluster', 'leader' /* To support taking index snapshots, we have to set path.repo setting */ - systemProperty 'tests.path.repo', repoDir.absolutePath + nonInputProperties.systemProperty 'tests.path.repo', repoDir.absolutePath } testClusters.matching { it.name == 'leader-cluster' }.configureEach { testDistribution = 'DEFAULT' - setting 'path.repo', repoDir.absolutePath + setting 'path.repo', repoDir.absolutePath, IGNORE_VALUE setting 'xpack.ccr.enabled', 'true' setting 'xpack.security.enabled', 'false' setting 'xpack.watcher.enabled', 'false' @@ -39,12 +40,12 @@ tasks.register('follow-cluster', RestIntegTestTask) { nonInputProperties.systemProperty 'tests.leader_remote_cluster_seed', "${-> testClusters.'leader-cluster'.getAllTransportPortURI().get(0)}" /* To support taking index snapshots, we have to set path.repo setting */ - systemProperty 'tests.path.repo', repoDir.absolutePath + nonInputProperties.systemProperty 'tests.path.repo', repoDir.absolutePath } testClusters.matching{ it.name == 'follow-cluster' }.configureEach { testDistribution = 'DEFAULT' - setting 'path.repo', repoDir.absolutePath + setting 'path.repo', repoDir.absolutePath, IGNORE_VALUE setting 'xpack.ccr.enabled', 'true' setting 'xpack.security.enabled', 'false' setting 'xpack.watcher.enabled', 'false' @@ -52,11 +53,11 @@ testClusters.matching{ it.name == 'follow-cluster' }.configureEach { setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.poll_interval', '1000ms' setting 'cluster.remote.leader_cluster.seeds', - { "\"${testClusters.'leader-cluster'.getAllTransportPortURI().get(0)}\"" } + { "\"${testClusters.'leader-cluster'.getAllTransportPortURI().get(0)}\"" }, IGNORE_VALUE } tasks.named("check").configure { dependsOn 'follow-cluster' } // Security is explicitly disabled for follow-cluster and leader-cluster, do not run these in FIPS mode tasks.withType(Test).configureEach { enabled = BuildParams.inFipsJvm == false -} \ No newline at end of file +} diff --git a/x-pack/plugin/ilm/qa/multi-node/build.gradle b/x-pack/plugin/ilm/qa/multi-node/build.gradle index 523bf5d04ae4f..ef20d57d83f59 100644 --- a/x-pack/plugin/ilm/qa/multi-node/build.gradle +++ b/x-pack/plugin/ilm/qa/multi-node/build.gradle @@ -1,4 +1,5 @@ import org.elasticsearch.gradle.internal.info.BuildParams +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'elasticsearch.legacy-java-rest-test' @@ -12,14 +13,14 @@ File repoDir = file("$buildDir/testclusters/repo") tasks.named("javaRestTest").configure { /* To support taking index snapshots, we have to set path.repo setting */ - systemProperty 'tests.path.repo', repoDir + nonInputProperties.systemProperty 'tests.path.repo', repoDir } testClusters.configureEach { testDistribution = 'DEFAULT' numberOfNodes = 4 - setting 'path.repo', repoDir.absolutePath + setting 'path.repo', repoDir.absolutePath, IGNORE_VALUE setting 'xpack.searchable.snapshot.shared_cache.size', '16MB' setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' setting 'xpack.security.enabled', 'false' diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/UpdateSettingsStepTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/UpdateSettingsStepTests.java index 06ac4673264f3..4e8df685a25a7 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/UpdateSettingsStepTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/UpdateSettingsStepTests.java @@ -7,30 +7,18 @@ package org.elasticsearch.xpack.ilm; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexModule; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.ilm.Step.StepKey; import org.elasticsearch.xpack.core.ilm.UpdateSettingsStep; import org.junit.After; @@ -39,7 +27,6 @@ import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.xpack.ilm.UpdateSettingsStepTests.SettingsTestingService.INVALID_VALUE; @@ -62,22 +49,7 @@ public void onIndexModule(IndexModule module) { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { return List.of(service); } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java index 6409c2b72e1f1..53e4d3de463fd 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java @@ -10,37 +10,25 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.core.IOUtils; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.health.HealthIndicatorService; import org.elasticsearch.index.IndexModule; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.HealthPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersAction; @@ -154,48 +142,39 @@ protected XPackLicenseState getLicenseState() { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { final List components = new ArrayList<>(); ILMHistoryTemplateRegistry ilmTemplateRegistry = new ILMHistoryTemplateRegistry( settings, - clusterService, - threadPool, - client, - xContentRegistry + services.clusterService(), + services.threadPool(), + services.client(), + services.xContentRegistry() ); ilmTemplateRegistry.initialize(); - ilmHistoryStore.set(new ILMHistoryStore(new OriginSettingClient(client, INDEX_LIFECYCLE_ORIGIN), clusterService, threadPool)); + ilmHistoryStore.set( + new ILMHistoryStore( + new OriginSettingClient(services.client(), INDEX_LIFECYCLE_ORIGIN), + services.clusterService(), + services.threadPool() + ) + ); /* * Here we use threadPool::absoluteTimeInMillis rather than System::currentTimeInMillis because snapshot start time is set using * ThreadPool.absoluteTimeInMillis(). ThreadPool.absoluteTimeInMillis() returns a cached time that can be several hundred * milliseconds behind System.currentTimeMillis(). The result is that a snapshot taken after a policy is created can have a start * time that is before the policy's (or action's) start time if System::currentTimeInMillis is used here. */ - LongSupplier nowSupplier = threadPool::absoluteTimeInMillis; + LongSupplier nowSupplier = services.threadPool()::absoluteTimeInMillis; indexLifecycleInitialisationService.set( new IndexLifecycleService( settings, - client, - clusterService, - threadPool, + services.client(), + services.clusterService(), + services.threadPool(), getClock(), nowSupplier, - xContentRegistry, + services.xContentRegistry(), ilmHistoryStore.get(), getLicenseState() ) @@ -204,15 +183,17 @@ public Collection createComponents( ilmHealthIndicatorService.set( new IlmHealthIndicatorService( - clusterService, + services.clusterService(), new IlmHealthIndicatorService.StagnatingIndicesFinder( - clusterService, + services.clusterService(), IlmHealthIndicatorService.RULES_BY_ACTION_CONFIG.values(), System::currentTimeMillis ) ) ); - reservedLifecycleAction.set(new ReservedLifecycleAction(xContentRegistry, client, XPackPlugin.getSharedLicenseState())); + reservedLifecycleAction.set( + new ReservedLifecycleAction(services.xContentRegistry(), services.client(), XPackPlugin.getSharedLicenseState()) + ); return components; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index f38f90f773b7f..2f0f95cf8a911 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -10,11 +10,8 @@ import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; @@ -23,24 +20,15 @@ import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.InferenceServicePlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.ScalingExecutorBuilder; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.inference.action.DeleteInferenceModelAction; import org.elasticsearch.xpack.inference.action.GetInferenceModelAction; @@ -108,25 +96,12 @@ public List getRestHandlers( } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - httpManager.set(HttpClientManager.create(settings, threadPool, clusterService)); - httpRequestSenderFactory.set(new HttpRequestSenderFactory(threadPool, httpManager.get(), clusterService, settings)); - ModelRegistry modelRegistry = new ModelRegistry(client); + public Collection createComponents(PluginServices services) { + httpManager.set(HttpClientManager.create(settings, services.threadPool(), services.clusterService())); + httpRequestSenderFactory.set( + new HttpRequestSenderFactory(services.threadPool(), httpManager.get(), services.clusterService(), settings) + ); + ModelRegistry modelRegistry = new ModelRegistry(services.client()); return List.of(modelRegistry); } diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 75202749d8dca..61db9dcd54a02 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -153,12 +153,13 @@ public BlockLoader.BytesRefBuilder builder(BlockLoader.BuilderFactory factory, i } @Override - public BlockLoader.Builder readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - BlockLoader.BytesRefBuilder builder = builder(factory, docs.count()); - for (int i = 0; i < docs.count(); i++) { - builder.appendBytesRef(bytes); + public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { + try (BlockLoader.BytesRefBuilder builder = builder(factory, docs.count())) { + for (int i = 0; i < docs.count(); i++) { + builder.appendBytesRef(bytes); + } + return builder.build(); } - return builder; } @Override diff --git a/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle b/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle index 368aab4615fc7..1cc37f5c4ffc0 100644 --- a/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle @@ -1,6 +1,7 @@ import org.elasticsearch.gradle.internal.test.RestIntegTestTask import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.VersionProperties +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.standalone-rest-test' @@ -39,7 +40,7 @@ testClusters.register('mixed-cluster') { setting 'xpack.license.self_generated.type', 'trial' setting 'cluster.remote.my_remote_cluster.seeds', { remoteCluster.get().getAllTransportPortURI().collect { "\"$it\"" }.toString() - } + }, IGNORE_VALUE setting 'cluster.remote.connections_per_cluster', "1" user username: "test_user", password: "x-pack-test-password" diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index cb38e23c7f8eb..f4bce4906c0b0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -29,7 +29,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -45,12 +44,10 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.analysis.CharFilterFactory; import org.elasticsearch.index.analysis.TokenizerFactory; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.indices.AssociatedIndexDescriptor; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; import org.elasticsearch.indices.breaker.BreakerSettings; @@ -73,15 +70,11 @@ import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.plugins.ShutdownAwarePlugin; import org.elasticsearch.plugins.SystemIndexPlugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.ScalingExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.ContextParser; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; @@ -884,22 +877,14 @@ public List> getRescorers() { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { + Client client = services.client(); + ClusterService clusterService = services.clusterService(); + ThreadPool threadPool = services.threadPool(); + Environment environment = services.environment(); + NamedXContentRegistry xContentRegistry = services.xContentRegistry(); + IndexNameExpressionResolver indexNameExpressionResolver = services.indexNameExpressionResolver(); + if (enabled == false) { // Holders for @link(MachineLearningFeatureSetUsage) which needs access to job manager and ML extension, // both empty if ML is disabled diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTracker.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTracker.java index 2dcfdf7603c7f..5c89c29a70cdd 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTracker.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTracker.java @@ -141,9 +141,9 @@ static void getMemoryAndProcessors( Long jobMemory = mlMemoryTracker.getAnomalyDetectorJobMemoryRequirement(jobId); if (jobMemory == null) { - // TODO: this indicates a bug, should we indicate that the result is incomplete? - logger.debug("could not find memory requirement for job [{}], skipping", jobId); - continue; + logger.debug("could not find memory requirement for job [{}], returning no-scale", jobId); + listener.onResponse(noScaleStats(numberMlNodes)); + return; } if (AWAITING_LAZY_ASSIGNMENT.equals(task.getAssignment())) { @@ -169,9 +169,9 @@ static void getMemoryAndProcessors( Long jobMemory = mlMemoryTracker.getDataFrameAnalyticsJobMemoryRequirement(jobId); if (jobMemory == null) { - // TODO: this indicates a bug, should we indicate that the result is incomplete? - logger.debug("could not find memory requirement for job [{}], skipping", jobId); - continue; + logger.debug("could not find memory requirement for job [{}], returning no-scale", jobId); + listener.onResponse(noScaleStats(numberMlNodes)); + return; } if (AWAITING_LAZY_ASSIGNMENT.equals(task.getAssignment())) { @@ -291,6 +291,10 @@ static void getMemoryAndProcessors( */ public static MlAutoscalingStats noScaleStats(ClusterState clusterState) { int numberMlNodes = (int) clusterState.nodes().stream().filter(node -> node.getRoles().contains(DiscoveryNodeRole.ML_ROLE)).count(); + return noScaleStats(numberMlNodes); + } + + private static MlAutoscalingStats noScaleStats(int numberMlNodes) { return new MlAutoscalingStats( numberMlNodes, 0, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java index 550742ef04b5a..139eceb09bd02 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java @@ -170,6 +170,7 @@ public void close() { killProcess(false, false); stateStreamer.cancel(); } + dataCountsReporter.writeUnreportedCounts(); autodetectResultProcessor.awaitCompletion(); } finally { onFinishHandler.accept(null, true); @@ -180,7 +181,6 @@ public void close() { try { future.get(); autodetectWorkerExecutor.shutdownNow(); - dataCountsReporter.writeUnreportedCounts(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } catch (ExecutionException e) { diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java index 47e24b60896da..bab8e5b22c37a 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java @@ -12,17 +12,12 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.license.License; import org.elasticsearch.license.LicenseService; import org.elasticsearch.license.LicensedFeature; @@ -30,14 +25,9 @@ import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ReloadablePlugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; @@ -118,22 +108,11 @@ protected LicenseService getLicenseService() { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { + Client client = services.client(); + ClusterService clusterService = services.clusterService(); + ThreadPool threadPool = services.threadPool(); + final ClusterSettings clusterSettings = clusterService.getClusterSettings(); final CleanerService cleanerService = new CleanerService(settings, clusterSettings, threadPool); final SSLService dynamicSSLService = getSslService().createDynamicSSLService(); @@ -157,7 +136,14 @@ public Collection createComponents( Set collectors = new HashSet<>(); collectors.add(new IndexStatsCollector(clusterService, getLicenseState(), client)); collectors.add( - new ClusterStatsCollector(settings, clusterService, getLicenseState(), client, getLicenseService(), expressionResolver) + new ClusterStatsCollector( + settings, + clusterService, + getLicenseState(), + client, + getLicenseService(), + services.indexNameExpressionResolver() + ) ); collectors.add(new ShardsCollector(clusterService, getLicenseState())); collectors.add(new NodeStatsCollector(clusterService, getLicenseState(), client)); @@ -175,7 +161,7 @@ public Collection createComponents( clusterService, threadPool, client, - xContentRegistry + services.xContentRegistry() ); templateRegistry.initialize(); diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java index 5fc5d20461fe8..955cf0396326b 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java @@ -14,23 +14,17 @@ import org.elasticsearch.Build; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; @@ -41,7 +35,6 @@ import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.translog.TranslogStats; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.license.License; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.LicensedFeature; @@ -52,15 +45,10 @@ import org.elasticsearch.plugins.IndexStorePlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.RepositoryPlugin; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotRestoreException; import org.elasticsearch.snapshots.sourceonly.SourceOnlySnapshotRepository; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; @@ -75,7 +63,6 @@ import java.util.Optional; import java.util.function.BiConsumer; import java.util.function.Function; -import java.util.function.Supplier; public class OldLuceneVersions extends Plugin implements IndexStorePlugin, ClusterPlugin, RepositoryPlugin, ActionPlugin, EnginePlugin { @@ -90,24 +77,12 @@ public class OldLuceneVersions extends Plugin implements IndexStorePlugin, Clust private final SetOnce failShardsListener = new SetOnce<>(); @Override - public Collection createComponents( - final Client client, - final ClusterService clusterService, - final ThreadPool threadPool, - final ResourceWatcherService resourceWatcherService, - final ScriptService scriptService, - final NamedXContentRegistry xContentRegistry, - final Environment environment, - final NodeEnvironment nodeEnvironment, - final NamedWriteableRegistry registry, - final IndexNameExpressionResolver resolver, - final Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { + ClusterService clusterService = services.clusterService(); + ThreadPool threadPool = services.threadPool(); + this.failShardsListener.set(new FailShardsOnInvalidLicenseClusterListener(getLicenseState(), clusterService.getRerouteService())); - if (DiscoveryNode.isMasterNode(environment.settings())) { + if (DiscoveryNode.isMasterNode(services.environment().settings())) { // We periodically look through the indices and identify if there are any archive indices, // then marking the feature as used. We do this on each master node so that if one master fails, the // continue reporting usage state. diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingPlugin.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingPlugin.java index d37a5be7543bd..f98c22b3bde30 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingPlugin.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingPlugin.java @@ -15,30 +15,20 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.ScalingExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; @@ -81,24 +71,13 @@ public ProfilingPlugin(Settings settings) { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { + Client client = services.client(); + ClusterService clusterService = services.clusterService(); + ThreadPool threadPool = services.threadPool(); + logger.info("Profiling is {}", enabled ? "enabled" : "disabled"); - registry.set(new ProfilingIndexTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry)); + registry.set(new ProfilingIndexTemplateRegistry(settings, clusterService, threadPool, client, services.xContentRegistry())); indexStateResolver.set(new IndexStateResolver(PROFILING_CHECK_OUTDATED_INDICES.get(settings))); clusterService.getClusterSettings().addSettingsUpdateConsumer(PROFILING_CHECK_OUTDATED_INDICES, this::updateCheckOutdatedIndices); diff --git a/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle b/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle index b8c0127d0586a..595d05e05b410 100644 --- a/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle @@ -1,3 +1,5 @@ +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE + apply plugin: 'elasticsearch.legacy-java-rest-test' apply plugin: 'elasticsearch.legacy-yaml-rest-test' apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' @@ -15,12 +17,12 @@ restResources { } tasks.withType(Test).configureEach { - systemProperty 'tests.path.repo', repoDir + nonInputProperties.systemProperty 'tests.path.repo', repoDir } testClusters.configureEach { testDistribution = 'DEFAULT' - setting 'path.repo', repoDir.absolutePath + setting 'path.repo', repoDir.absolutePath, IGNORE_VALUE setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.searchable.snapshot.shared_cache.size', '16MB' diff --git a/x-pack/plugin/searchable-snapshots/qa/url/build.gradle b/x-pack/plugin/searchable-snapshots/qa/url/build.gradle index 160eb8d28cbcc..12fc0873958e1 100644 --- a/x-pack/plugin/searchable-snapshots/qa/url/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/url/build.gradle @@ -29,13 +29,13 @@ File repositoryDir = fixture.fsRepositoryDir as File tasks.named("javaRestTest").configure { dependsOn fixture.getTasks().named("postProcessFixture") - systemProperty 'test.url.fs.repo.dir', repositoryDir.absolutePath + nonInputProperties.systemProperty 'test.url.fs.repo.dir', repositoryDir.absolutePath nonInputProperties.systemProperty 'test.url.http', "${-> fixtureAddress('nginx-fixture')}" } testClusters.matching { it.name == "javaRestTest" }.configureEach { testDistribution = 'DEFAULT' - setting 'path.repo', repositoryDir.absolutePath + setting 'path.repo', repositoryDir.absolutePath, IGNORE_VALUE setting 'repositories.url.allowed_urls', { "${-> fixtureAddress('nginx-fixture')}" }, IGNORE_VALUE setting 'xpack.license.self_generated.type', 'trial' diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index f74dba71ebb83..98f6da9ba6a58 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -23,13 +23,11 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RerouteService; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; @@ -41,7 +39,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasables; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; @@ -51,7 +48,6 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogStats; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.XPackLicenseState; @@ -67,15 +63,11 @@ import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.snapshots.SearchableSnapshotsSettings; import org.elasticsearch.snapshots.sourceonly.SourceOnlySnapshotRepository; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.ScalingExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; @@ -319,24 +311,14 @@ public List> getSettings() { } @Override - public Collection createComponents( - final Client client, - final ClusterService clusterService, - final ThreadPool threadPool, - final ResourceWatcherService resourceWatcherService, - final ScriptService scriptService, - final NamedXContentRegistry xContentRegistry, - final Environment environment, - final NodeEnvironment nodeEnvironment, - final NamedWriteableRegistry registry, - final IndexNameExpressionResolver resolver, - final Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { + Client client = services.client(); + ClusterService clusterService = services.clusterService(); + ThreadPool threadPool = services.threadPool(); + NodeEnvironment nodeEnvironment = services.nodeEnvironment(); + final List components = new ArrayList<>(); - this.repositoriesServiceSupplier = repositoriesServiceSupplier; + this.repositoriesServiceSupplier = services.repositoriesServiceSupplier(); this.threadPool.set(threadPool); this.failShardsListener.set(new FailShardsOnInvalidLicenseClusterListener(getLicenseState(), clusterService.getRerouteService())); if (DiscoveryNode.canContainData(settings)) { @@ -364,7 +346,7 @@ public Collection createComponents( PersistentCache.cleanUp(settings, nodeEnvironment); } - if (DiscoveryNode.isMasterNode(environment.settings())) { + if (DiscoveryNode.isMasterNode(services.environment().settings())) { // Tracking usage of searchable snapshots is too costly to do on each individually mounted snapshot. // Instead, we periodically look through the indices and identify if any are searchable snapshots, // then marking the feature as used. We do this on each master node so that if one master fails, the diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index fd38b40683f71..42b4c8c459eb0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -28,7 +28,6 @@ import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Strings; @@ -53,7 +52,6 @@ import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.Nullable; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.NodeMetadata; import org.elasticsearch.http.HttpPreRequest; import org.elasticsearch.http.HttpServerTransport; @@ -61,7 +59,6 @@ import org.elasticsearch.http.netty4.internal.HttpHeadersAuthenticatorUtils; import org.elasticsearch.http.netty4.internal.HttpValidator; import org.elasticsearch.index.IndexModule; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.ingest.Processor; @@ -81,7 +78,6 @@ import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.plugins.interceptor.RestServerActionPlugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; @@ -90,7 +86,6 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.internal.ShardSearchRequest; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.telemetry.tracing.Tracer; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.FixedExecutorBuilder; @@ -624,33 +619,18 @@ protected XPackLicenseState getLicenseState() { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { try { return createComponents( - client, - threadPool, - clusterService, - resourceWatcherService, - scriptService, - xContentRegistry, - environment, - nodeEnvironment.nodeMetadata(), - expressionResolver + services.client(), + services.threadPool(), + services.clusterService(), + services.resourceWatcherService(), + services.scriptService(), + services.xContentRegistry(), + services.environment(), + services.nodeEnvironment().nodeMetadata(), + services.indexNameExpressionResolver() ); } catch (final Exception e) { throw new IllegalStateException("security initialization failed", e); diff --git a/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownTasksIT.java b/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownTasksIT.java index 6c5e1e6af69ce..15e16d2a86910 100644 --- a/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownTasksIT.java +++ b/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownTasksIT.java @@ -21,15 +21,11 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.SettingsModule; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.persistent.AllocatedPersistentTask; import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.persistent.PersistentTaskState; @@ -38,12 +34,8 @@ import org.elasticsearch.persistent.PersistentTasksService; import org.elasticsearch.plugins.PersistentTaskPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; @@ -57,7 +49,6 @@ import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Supplier; import java.util.stream.Collectors; import static org.hamcrest.Matchers.contains; @@ -134,23 +125,8 @@ public static class TaskPlugin extends Plugin implements PersistentTaskPlugin { TaskExecutor taskExecutor; @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - taskExecutor = new TaskExecutor(client, clusterService, threadPool); + public Collection createComponents(PluginServices services) { + taskExecutor = new TaskExecutor(services.client(), services.clusterService(), services.threadPool()); return Collections.singletonList(taskExecutor); } diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java index 754ba7970eaa2..8c85bdb11dfa2 100644 --- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java +++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java @@ -9,29 +9,16 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.Arrays; import java.util.Collection; @@ -41,24 +28,9 @@ public class ShutdownPlugin extends Plugin implements ActionPlugin { @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { - NodeSeenService nodeSeenService = new NodeSeenService(clusterService); + NodeSeenService nodeSeenService = new NodeSeenService(services.clusterService()); return Collections.singletonList(nodeSeenService); } diff --git a/x-pack/plugin/slm/qa/multi-node/build.gradle b/x-pack/plugin/slm/qa/multi-node/build.gradle index 999f6e6a79c7c..b02fe7cd44fbd 100644 --- a/x-pack/plugin/slm/qa/multi-node/build.gradle +++ b/x-pack/plugin/slm/qa/multi-node/build.gradle @@ -1,4 +1,5 @@ import org.elasticsearch.gradle.internal.info.BuildParams +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'elasticsearch.legacy-java-rest-test' @@ -12,14 +13,14 @@ File repoDir = file("$buildDir/testclusters/repo") tasks.named("javaRestTest").configure { /* To support taking index snapshots, we have to set path.repo setting */ - systemProperty 'tests.path.repo', repoDir + nonInputProperties.systemProperty 'tests.path.repo', repoDir } testClusters.configureEach { testDistribution = 'DEFAULT' numberOfNodes = 4 - setting 'path.repo', repoDir.absolutePath + setting 'path.repo', repoDir.absolutePath, IGNORE_VALUE setting 'xpack.searchable.snapshot.shared_cache.size', '16MB' setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' setting 'xpack.security.enabled', 'false' diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycle.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycle.java index 3664b4efbcf9a..74094c83d4bcb 100644 --- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycle.java +++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycle.java @@ -15,31 +15,22 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.core.IOUtils; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.health.HealthIndicatorService; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.HealthPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.core.XPackPlugin; @@ -83,6 +74,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.function.Supplier; @@ -123,22 +115,10 @@ protected XPackLicenseState getLicenseState() { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { + Client client = services.client(); + ClusterService clusterService = services.clusterService(); + ThreadPool threadPool = services.threadPool(); final List components = new ArrayList<>(); SnapshotLifecycleTemplateRegistry templateRegistry = new SnapshotLifecycleTemplateRegistry( @@ -146,7 +126,7 @@ public Collection createComponents( clusterService, threadPool, client, - xContentRegistry + services.xContentRegistry() ); templateRegistry.initialize(); snapshotHistoryStore.set(new SnapshotHistoryStore(new OriginSettingClient(client, INDEX_LIFECYCLE_ORIGIN), clusterService)); @@ -167,7 +147,7 @@ public Collection createComponents( ) ); snapshotRetentionService.get().init(clusterService); - components.addAll(Arrays.asList(snapshotLifecycleService.get(), snapshotHistoryStore.get(), snapshotRetentionService.get())); + Collections.addAll(components, snapshotLifecycleService.get(), snapshotHistoryStore.get(), snapshotRetentionService.get()); slmHealthIndicatorService.set(new SlmHealthIndicatorService(clusterService)); return components; diff --git a/x-pack/plugin/snapshot-based-recoveries/qa/fs/build.gradle b/x-pack/plugin/snapshot-based-recoveries/qa/fs/build.gradle index 0c0c1930f8601..d116a2d11e22d 100644 --- a/x-pack/plugin/snapshot-based-recoveries/qa/fs/build.gradle +++ b/x-pack/plugin/snapshot-based-recoveries/qa/fs/build.gradle @@ -6,6 +6,8 @@ * Side Public License, v 1. */ +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE + apply plugin: 'elasticsearch.legacy-java-rest-test' apply plugin: 'elasticsearch.rest-resources' @@ -25,7 +27,7 @@ tasks.withType(Test).configureEach { doFirst { delete(repoDir) } - systemProperty 'tests.path.repo', repoDir + nonInputProperties.systemProperty 'tests.path.repo', repoDir } testClusters.matching { it.name == "javaRestTest" }.configureEach { @@ -33,6 +35,6 @@ testClusters.matching { it.name == "javaRestTest" }.configureEach { numberOfNodes = 3 setting 'xpack.license.self_generated.type', 'trial' - setting 'path.repo', repoDir.absolutePath + setting 'path.repo', repoDir.absolutePath, IGNORE_VALUE setting 'xpack.security.enabled', 'false' } diff --git a/x-pack/plugin/snapshot-based-recoveries/qa/license-enforcing/build.gradle b/x-pack/plugin/snapshot-based-recoveries/qa/license-enforcing/build.gradle index 1c9a877507791..d2121daa1ba93 100644 --- a/x-pack/plugin/snapshot-based-recoveries/qa/license-enforcing/build.gradle +++ b/x-pack/plugin/snapshot-based-recoveries/qa/license-enforcing/build.gradle @@ -27,7 +27,7 @@ tasks.withType(Test).configureEach { doFirst { delete(repoDir) } - systemProperty 'tests.path.repo', repoDir + nonInputProperties.systemProperty 'tests.path.repo', repoDir } testClusters.matching { it.name == "javaRestTest" }.configureEach { @@ -37,6 +37,6 @@ testClusters.matching { it.name == "javaRestTest" }.configureEach { // This project tests that enterprise licensing is enforced, // therefore we use a basic license setting 'xpack.license.self_generated.type', 'basic' - setting 'path.repo', repoDir.absolutePath + setting 'path.repo', repoDir.absolutePath, IGNORE_VALUE setting 'xpack.security.enabled', 'false' } diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle index 6b32e11ef81d4..17df249b08cf6 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle @@ -1,3 +1,5 @@ +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE + apply plugin: 'elasticsearch.legacy-yaml-rest-test' apply plugin: 'elasticsearch.rest-resources' @@ -9,12 +11,12 @@ dependencies { final File repoDir = file("$buildDir/testclusters/repo") tasks.named("yamlRestTest").configure { - systemProperty 'tests.path.repo', repoDir + nonInputProperties.systemProperty 'tests.path.repo', repoDir } testClusters.matching { it.name == "yamlRestTest" }.configureEach { testDistribution = 'DEFAULT' - setting 'path.repo', repoDir.absolutePath + setting 'path.repo', repoDir.absolutePath, IGNORE_VALUE setting 'xpack.security.enabled', 'false' } diff --git a/x-pack/plugin/sql/qa/server/multi-cluster-with-security/build.gradle b/x-pack/plugin/sql/qa/server/multi-cluster-with-security/build.gradle index b56d8ff211990..b42ae29e257f0 100644 --- a/x-pack/plugin/sql/qa/server/multi-cluster-with-security/build.gradle +++ b/x-pack/plugin/sql/qa/server/multi-cluster-with-security/build.gradle @@ -1,4 +1,5 @@ import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'elasticsearch.legacy-java-rest-test' @@ -25,7 +26,7 @@ def javaRestTestClusterReg = testClusters.register('javaRestTest') { setting 'xpack.watcher.enabled', 'false' setting 'cluster.remote.my_remote_cluster.seeds', { remoteClusterReg.get().getAllTransportPortURI().collect { "\"$it\"" }.toString() - } + }, IGNORE_VALUE setting 'cluster.remote.connections_per_cluster', "1" setting 'xpack.security.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPlugin.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPlugin.java index c5f8eee643858..c6e0b5067ee08 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPlugin.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPlugin.java @@ -11,30 +11,20 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.license.License; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.LicensedFeature; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; @@ -88,24 +78,13 @@ protected XPackLicenseState getLicenseState() { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - - return createComponents(client, environment.settings(), clusterService, namedWriteableRegistry); + public Collection createComponents(PluginServices services) { + return createComponents( + services.client(), + services.environment().settings(), + services.clusterService(), + services.namedWriteableRegistry() + ); } /** diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java index a665170f8dcdb..1fac8a28aa5da 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java @@ -6,29 +6,14 @@ */ package org.elasticsearch.xpack.stack; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.function.Supplier; public class StackPlugin extends Plugin implements ActionPlugin { private final Settings settings; @@ -43,36 +28,21 @@ public List> getSettings() { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { LegacyStackTemplateRegistry legacyStackTemplateRegistry = new LegacyStackTemplateRegistry( settings, - clusterService, - threadPool, - client, - xContentRegistry + services.clusterService(), + services.threadPool(), + services.client(), + services.xContentRegistry() ); legacyStackTemplateRegistry.initialize(); StackTemplateRegistry stackTemplateRegistry = new StackTemplateRegistry( settings, - clusterService, - threadPool, - client, - xContentRegistry + services.clusterService(), + services.threadPool(), + services.client(), + services.xContentRegistry() ); stackTemplateRegistry.initialize(); return List.of(legacyStackTemplateRegistry, stackTemplateRegistry); diff --git a/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/build.gradle b/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/build.gradle index 7c0eaf1450567..ef34db62e5e03 100644 --- a/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/build.gradle +++ b/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/build.gradle @@ -1,6 +1,7 @@ import org.elasticsearch.gradle.internal.test.RestIntegTestTask import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.VersionProperties +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.standalone-rest-test' @@ -43,7 +44,7 @@ testClusters.register('mixed-cluster') { setting 'xpack.license.self_generated.type', 'trial' setting 'cluster.remote.my_remote_cluster.seeds', { remoteCluster.get().getAllTransportPortURI().collect { "\"$it\"" }.toString() - } + }, IGNORE_VALUE setting 'cluster.remote.connections_per_cluster', "1" user username: "test_user", password: "x-pack-test-password" diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointIT.java index fde13c9d65324..bb159856b965d 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointIT.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointIT.java @@ -25,10 +25,9 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.startsWith; /** * Test suite for checkpointing using transform getcheckpoint API @@ -102,11 +101,11 @@ public void testGetCheckpoint() throws Exception { public void testGetCheckpointTimeoutExceeded() throws Exception { final String indexNamePrefix = "test_index-"; - final int indices = 5; - final int shards = 10; + final int indices = 100; + final int shards = 5; for (int i = 0; i < indices; ++i) { - indicesAdmin().prepareCreate(indexNamePrefix + i).setSettings(indexSettings(shards, 1)).get(); + indicesAdmin().prepareCreate(indexNamePrefix + i).setSettings(indexSettings(shards, 0)).get(); } final GetCheckpointAction.Request request = new GetCheckpointAction.Request( @@ -115,20 +114,25 @@ public void testGetCheckpointTimeoutExceeded() throws Exception { TimeValue.ZERO ); - CountDownLatch countDown = new CountDownLatch(1); + CountDownLatch latch = new CountDownLatch(1); SetOnce finalException = new SetOnce<>(); - client().execute(GetCheckpointAction.INSTANCE, request, ActionListener.wrap(r -> countDown.countDown(), e -> { + client().execute(GetCheckpointAction.INSTANCE, request, ActionListener.wrap(r -> latch.countDown(), e -> { finalException.set(e); - countDown.countDown(); + latch.countDown(); })); - countDown.await(10, TimeUnit.SECONDS); + latch.await(10, TimeUnit.SECONDS); Exception e = finalException.get(); - assertThat(e, is(notNullValue())); - assertThat(e, is(instanceOf(ElasticsearchTimeoutException.class))); - assertThat( - e.getMessage(), - is(equalTo("Transform checkpointing timed out on node [node_s_0] after [0ms] having processed [0] of [50] shards")) - ); + if (e != null) { + assertThat(e, is(instanceOf(ElasticsearchTimeoutException.class))); + assertThat( + "Message was: " + e.getMessage(), + e.getMessage(), + startsWith("Transform checkpointing timed out on node [node_s_0] after [0ms]") + ); + } else { + // Due to system clock usage, the timeout does not always occur where it should. + // We cannot mock the clock so we just have to live with it. + } } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java index 81a719e24f633..61cc0e2c072ad 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java @@ -24,9 +24,7 @@ import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; @@ -34,25 +32,17 @@ import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.indices.AssociatedIndexDescriptor; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.persistent.PersistentTasksExecutor; import org.elasticsearch.plugins.PersistentTaskPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.NamedXContentRegistry.Entry; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.action.SetResetModeActionRequest; @@ -229,27 +219,15 @@ public List getRestHandlers( } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { + Client client = services.client(); + ClusterService clusterService = services.clusterService(); + TransformConfigManager configManager = new IndexBasedTransformConfigManager( clusterService, - expressionResolver, + services.indexNameExpressionResolver(), client, - xContentRegistry + services.xContentRegistry() ); TransformAuditor auditor = new TransformAuditor( client, @@ -265,12 +243,12 @@ public Collection createComponents( configManager, auditor ); - TransformScheduler scheduler = new TransformScheduler(clock, threadPool, settings); + TransformScheduler scheduler = new TransformScheduler(clock, services.threadPool(), settings); scheduler.start(); transformServices.set(new TransformServices(configManager, checkpointService, auditor, scheduler)); - return Arrays.asList( + return List.of( transformServices.get(), new TransformClusterStateListener(clusterService, client), new TransformExtensionHolder(getTransformExtension()) diff --git a/x-pack/plugin/vector-tile/qa/multi-cluster/build.gradle b/x-pack/plugin/vector-tile/qa/multi-cluster/build.gradle index ea20c8eacd172..30b032c1cae1a 100644 --- a/x-pack/plugin/vector-tile/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/vector-tile/qa/multi-cluster/build.gradle @@ -6,7 +6,7 @@ */ import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask - +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'elasticsearch.legacy-java-rest-test' dependencies { @@ -27,7 +27,7 @@ def localCluster = testClusters.register('local') { setting 'xpack.security.enabled', 'true' user username: 'admin', password: 'admin-password', role: 'superuser' setting 'cluster.remote.other.seeds', - { "\"${remoteCluster.get().getAllTransportPortURI().join(",")}\"" } + { "\"${remoteCluster.get().getAllTransportPortURI().join(",")}\"" }, IGNORE_VALUE } diff --git a/x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePlugin.java b/x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePlugin.java index 1fee6eda5ad6d..df089381e70b2 100644 --- a/x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePlugin.java +++ b/x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePlugin.java @@ -11,32 +11,22 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.coordination.CoordinationMetadata.VotingConfiguration; import org.elasticsearch.cluster.coordination.CoordinationState.VoteCollection; import org.elasticsearch.cluster.coordination.ElectionStrategy; import org.elasticsearch.cluster.coordination.Join; import org.elasticsearch.cluster.coordination.PublicationTransportHandler; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; -import org.elasticsearch.cluster.routing.allocation.AllocationService; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.discovery.DiscoveryModule; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.ClusterCoordinationPlugin; import org.elasticsearch.plugins.NetworkPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportException; @@ -45,8 +35,6 @@ import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportResponseHandler; -import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; @@ -84,23 +72,8 @@ public static boolean isFullMasterNode(DiscoveryNode discoveryNode) { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { - this.threadPool.set(threadPool); + public Collection createComponents(PluginServices services) { + this.threadPool.set(services.threadPool()); return Collections.emptyList(); } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index e5fd1fcbf2035..a46b42e2153bd 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -24,10 +24,8 @@ import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; @@ -39,26 +37,21 @@ import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexModule; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ReloadablePlugin; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.plugins.SystemIndexPlugin; -import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.TemplateScript; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.XPackPlugin; @@ -305,26 +298,18 @@ protected Clock getClock() { } @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver expressionResolver, - Supplier repositoriesServiceSupplier, - TelemetryProvider telemetryProvider, - AllocationService allocationService, - IndicesService indicesService - ) { + public Collection createComponents(PluginServices services) { if (enabled == false) { return Collections.emptyList(); } + Client client = services.client(); + ClusterService clusterService = services.clusterService(); + ThreadPool threadPool = services.threadPool(); + Environment environment = services.environment(); + ScriptService scriptService = services.scriptService(); + NamedXContentRegistry xContentRegistry = services.xContentRegistry(); + // only initialize these classes if Watcher is enabled, and only after the plugin security policy for Watcher is in place BodyPartSource.init(); Account.init(); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java index e1f1933242414..d3af489a77a2a 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java @@ -14,8 +14,8 @@ import org.elasticsearch.index.engine.InternalEngineFactory; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.TestIndexNameExpressionResolver; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.internal.DocumentParsingObserver; -import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.threadpool.ExecutorBuilder; @@ -74,10 +74,7 @@ public void testWatcherDisabledTests() throws Exception { watcher.onIndexModule(indexModule); // also no component creation if not enabled - assertThat( - watcher.createComponents(null, null, null, null, null, null, null, null, null, null, null, TelemetryProvider.NOOP, null, null), - hasSize(0) - ); + assertThat(watcher.createComponents(mock(Plugin.PluginServices.class)), hasSize(0)); watcher.close(); } diff --git a/x-pack/qa/kerberos-tests/build.gradle b/x-pack/qa/kerberos-tests/build.gradle index 30fac49096679..62d6f0a1e34b8 100644 --- a/x-pack/qa/kerberos-tests/build.gradle +++ b/x-pack/qa/kerberos-tests/build.gradle @@ -12,6 +12,13 @@ dependencies { javaRestTestImplementation(testArtifact(project(xpackModule('security')))) } +normalization { + runtimeClasspath { + ignore 'krb5.conf' + ignore '*.keytab' + } +} + tasks.register("copyKeytabToGeneratedResources", Copy) { from project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("peppa", "peppa.keytab") into "$buildDir/generated-resources/keytabs" @@ -28,6 +35,7 @@ tasks.register("copyConfToGeneratedResources", Copy) { String realm = "BUILD.ELASTIC.CO" tasks.named("javaRestTest").configure { + dependsOn "copyKeytabToGeneratedResources", "copyConfToGeneratedResources" usesDefaultDistribution() Path peppaKeytab = Paths.get("${project.buildDir}", "generated-resources", "keytabs", "peppa.keytab") Path krb5Conf = Paths.get("${project.buildDir}", "generated-resources", "conf", "krb5.conf") @@ -35,12 +43,9 @@ tasks.named("javaRestTest").configure { nonInputProperties.systemProperty 'test.userkt.keytab', "${peppaKeytab}" nonInputProperties.systemProperty 'test.userpwd', "george@${realm}" nonInputProperties.systemProperty 'test.krb5.conf', "${krb5Conf}" + nonInputProperties.systemProperty 'java.security.krb5.conf', "${krb5Conf}" systemProperty 'test.userpwd.password', "dino_but_longer_than_14_chars" - jvmArgs([ - "-Djava.security.krb5.conf=${krb5Conf}", - "-Dsun.security.krb5.debug=true" - ]) - dependsOn "copyKeytabToGeneratedResources", "copyConfToGeneratedResources" + systemProperty 'sun.security.krb5.debug', true classpath += files("$buildDir/generated-resources/keytabs") classpath += files("$buildDir/generated-resources/conf") } diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle index f9d60181fc815..cce18a4bd1579 100644 --- a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle +++ b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle @@ -1,5 +1,6 @@ import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.test.RestIntegTestTask +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-resources' @@ -40,11 +41,11 @@ def queryingCluster = testClusters.register('querying-cluster') { setting 'cluster.remote.my_remote_cluster.mode', 'proxy' setting 'cluster.remote.my_remote_cluster.proxy_address', { "\"${fulfillingCluster.get().getAllTransportPortURI().get(0)}\"" - } + }, IGNORE_VALUE } else { setting 'cluster.remote.my_remote_cluster.seeds', { fulfillingCluster.get().getAllTransportPortURI().collect { "\"$it\"" }.toString() - } + }, IGNORE_VALUE } } diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-full-license/build.gradle b/x-pack/qa/multi-cluster-search-security/legacy-with-full-license/build.gradle index 35b7a981fca36..69c0e8b20c2c4 100644 --- a/x-pack/qa/multi-cluster-search-security/legacy-with-full-license/build.gradle +++ b/x-pack/qa/multi-cluster-search-security/legacy-with-full-license/build.gradle @@ -1,5 +1,6 @@ import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.test.RestIntegTestTask +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-resources' @@ -40,11 +41,11 @@ def queryingCluster = testClusters.register('querying-cluster') { setting 'cluster.remote.my_remote_cluster.mode', 'proxy' setting 'cluster.remote.my_remote_cluster.proxy_address', { "\"${fulfillingCluster.get().getAllTransportPortURI().get(0)}\"" - } + }, IGNORE_VALUE } else { setting 'cluster.remote.my_remote_cluster.seeds', { fulfillingCluster.get().getAllTransportPortURI().collect { "\"$it\"" }.toString() - } + }, IGNORE_VALUE } } diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/build.gradle b/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/build.gradle index 810895abe1e86..1164aa240ee22 100644 --- a/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/build.gradle +++ b/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/build.gradle @@ -1,5 +1,6 @@ import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.test.RestIntegTestTask +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-resources' @@ -63,11 +64,11 @@ def queryingCluster = testClusters.register('querying-cluster') { setting 'cluster.remote.my_remote_cluster.mode', 'proxy' setting 'cluster.remote.my_remote_cluster.proxy_address', { "\"${fulfillingCluster.get().getAllTransportPortURI().get(0)}\"" - } + }, IGNORE_VALUE } else { setting 'cluster.remote.my_remote_cluster.seeds', { fulfillingCluster.get().getAllTransportPortURI().collect { "\"$it\"" }.toString() - } + }, IGNORE_VALUE } } diff --git a/x-pack/qa/repository-old-versions/build.gradle b/x-pack/qa/repository-old-versions/build.gradle index da8802e452c43..3c55aa8aa4663 100644 --- a/x-pack/qa/repository-old-versions/build.gradle +++ b/x-pack/qa/repository-old-versions/build.gradle @@ -13,6 +13,7 @@ import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.test.AntFixture import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask import org.elasticsearch.gradle.transform.UnzipTransform +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'elasticsearch.jdk-download' apply plugin: 'elasticsearch.internal-testclusters' @@ -86,7 +87,7 @@ if (OS.current() == OS.WINDOWS) { numberOfNodes = 2 versions = [project.version, project.version] // to test full cluster restart - setting 'path.repo', repoLocation + setting 'path.repo', repoLocation, IGNORE_VALUE setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' @@ -151,7 +152,7 @@ if (OS.current() == OS.WINDOWS) { } tasks.matching { it.name.startsWith("javaRestTest") && it.name.endsWith(versionNoDots) }.configureEach { - it.systemProperty "tests.repo.location", repoLocation + it.nonInputProperties.systemProperty "tests.repo.location", repoLocation it.systemProperty "tests.es.version", version.toString() /* Use a closure on the string to delay evaluation until right before we diff --git a/x-pack/qa/saml-idp-tests/build.gradle b/x-pack/qa/saml-idp-tests/build.gradle index c2dffe2470b84..6027a421b62f2 100644 --- a/x-pack/qa/saml-idp-tests/build.gradle +++ b/x-pack/qa/saml-idp-tests/build.gradle @@ -31,6 +31,12 @@ tasks.register("copyIdpFiles", Sync) { } } +normalization { + runtimeClasspath { + ignore 'idp-metadata.xml' + } +} + tasks.named("javaRestTest").configure { usesDefaultDistribution() classpath += files(tasks.named("copyIdpFiles"))