diff --git a/.buildkite/pipelines/dra-workflow.yml b/.buildkite/pipelines/dra-workflow.yml index bcc6c9c57d756..25477c8541fa9 100644 --- a/.buildkite/pipelines/dra-workflow.yml +++ b/.buildkite/pipelines/dra-workflow.yml @@ -6,7 +6,8 @@ steps: provider: gcp image: family/elasticsearch-ubuntu-2204 machineType: custom-32-98304 - buildDirectory: /dev/shm/bk + localSsds: 1 + localSsdInterface: nvme diskSizeGb: 350 - wait # The hadoop build depends on the ES artifact diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index e44a1e67e9d59..1bb13c4c10966 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -76,6 +76,7 @@ steps: - trigger: elasticsearch-dra-workflow label: Trigger DRA snapshot workflow async: true + branches: "main 8.* 7.17" build: branch: "$BUILDKITE_BRANCH" commit: "$BUILDKITE_COMMIT" diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index ac75a3a968ed1..169c187ef115a 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -30,7 +30,7 @@ httpcore = 4.4.13 httpasyncclient = 4.1.5 commonslogging = 1.2 commonscodec = 1.15 -protobuf = 3.21.9 +protobuf = 3.25.5 # test dependencies randomizedrunner = 2.8.0 diff --git a/docs/changelog/111336.yaml b/docs/changelog/111336.yaml new file mode 100644 index 0000000000000..d5bf602cb7a88 --- /dev/null +++ b/docs/changelog/111336.yaml @@ -0,0 +1,5 @@ +pr: 111336 +summary: Use the same chunking configurations for models in the Elasticsearch service +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/112933.yaml b/docs/changelog/112933.yaml new file mode 100644 index 0000000000000..222cd5aadf739 --- /dev/null +++ b/docs/changelog/112933.yaml @@ -0,0 +1,5 @@ +pr: 112933 +summary: "Allow incubating Panama Vector in simdvec, and add vectorized `ipByteBin`" +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/113251.yaml b/docs/changelog/113251.yaml new file mode 100644 index 0000000000000..49167e6e4c915 --- /dev/null +++ b/docs/changelog/113251.yaml @@ -0,0 +1,5 @@ +pr: 113251 +summary: Span term query to convert to match no docs when unmapped field is targeted +area: Search +type: bug +issues: [] diff --git a/docs/changelog/113297.yaml b/docs/changelog/113297.yaml new file mode 100644 index 0000000000000..476619f432639 --- /dev/null +++ b/docs/changelog/113297.yaml @@ -0,0 +1,5 @@ +pr: 113297 +summary: "[ES|QL] add reverse function" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/113812.yaml b/docs/changelog/113812.yaml new file mode 100644 index 0000000000000..04498b4ae5f7e --- /dev/null +++ b/docs/changelog/113812.yaml @@ -0,0 +1,5 @@ +pr: 113812 +summary: Add Streaming Inference spec +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/113846.yaml b/docs/changelog/113846.yaml new file mode 100644 index 0000000000000..5fdd56e98d706 --- /dev/null +++ b/docs/changelog/113846.yaml @@ -0,0 +1,6 @@ +pr: 113846 +summary: Don't validate internal stats if they are empty +area: Aggregations +type: bug +issues: + - 113811 diff --git a/docs/changelog/113869.yaml b/docs/changelog/113869.yaml new file mode 100644 index 0000000000000..f1cd1ec423966 --- /dev/null +++ b/docs/changelog/113869.yaml @@ -0,0 +1,5 @@ +pr: 113869 +summary: Upgrade protobufer to 3.25.5 +area: Snapshot/Restore +type: upgrade +issues: [] diff --git a/docs/changelog/113900.yaml b/docs/changelog/113900.yaml new file mode 100644 index 0000000000000..25f833d251784 --- /dev/null +++ b/docs/changelog/113900.yaml @@ -0,0 +1,5 @@ +pr: 113900 +summary: Fix BWC for file-settings based role mappings +area: Authentication +type: bug +issues: [] diff --git a/docs/changelog/113961.yaml b/docs/changelog/113961.yaml new file mode 100644 index 0000000000000..24cb1f45f029e --- /dev/null +++ b/docs/changelog/113961.yaml @@ -0,0 +1,5 @@ +pr: 113961 +summary: "[ESQL] Support datetime data type in Least and Greatest functions" +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/114080.yaml b/docs/changelog/114080.yaml new file mode 100644 index 0000000000000..395768c46369a --- /dev/null +++ b/docs/changelog/114080.yaml @@ -0,0 +1,5 @@ +pr: 114080 +summary: Stream Cohere Completion +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/114116.yaml b/docs/changelog/114116.yaml new file mode 100644 index 0000000000000..8d1c9e162ae23 --- /dev/null +++ b/docs/changelog/114116.yaml @@ -0,0 +1,5 @@ +pr: 114116 +summary: "ES|QL: Ensure minimum capacity for `PlanStreamInput` caches" +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/114177.yaml b/docs/changelog/114177.yaml new file mode 100644 index 0000000000000..d68486469d797 --- /dev/null +++ b/docs/changelog/114177.yaml @@ -0,0 +1,5 @@ +pr: 114177 +summary: "Make `randomInstantBetween` always return value in range [minInstant, `maxInstant]`" +area: Infra/Metrics +type: bug +issues: [] diff --git a/docs/reference/connector/apis/create-connector-api.asciidoc b/docs/reference/connector/apis/create-connector-api.asciidoc index a115eab8853c0..3ecef6d302732 100644 --- a/docs/reference/connector/apis/create-connector-api.asciidoc +++ b/docs/reference/connector/apis/create-connector-api.asciidoc @@ -116,7 +116,7 @@ PUT _connector/my-connector "name": "My Connector", "description": "My Connector to sync data to Elastic index from Google Drive", "service_type": "google_drive", - "language": "english" + "language": "en" } ---- diff --git a/docs/reference/connector/docs/connectors-zoom.asciidoc b/docs/reference/connector/docs/connectors-zoom.asciidoc index d01b9c2be0368..d945a0aec3da1 100644 --- a/docs/reference/connector/docs/connectors-zoom.asciidoc +++ b/docs/reference/connector/docs/connectors-zoom.asciidoc @@ -63,18 +63,22 @@ To connect to Zoom you need to https://developers.zoom.us/docs/internal-apps/s2s 6. Click on the "Create" button to create the app registration. 7. After the registration is complete, you will be redirected to the app's overview page. Take note of the "App Credentials" value, as you'll need it later. 8. Navigate to the "Scopes" section and click on the "Add Scopes" button. -9. The following scopes need to be added to the app. +9. The following granular scopes need to be added to the app. + [source,bash] ---- -user:read:admin -meeting:read:admin -chat_channel:read:admin -recording:read:admin -chat_message:read:admin -report:read:admin +user:read:list_users:admin +meeting:read:list_meetings:admin +meeting:read:list_past_participants:admin +cloud_recording:read:list_user_recordings:admin +team_chat:read:list_user_channels:admin +team_chat:read:list_user_messages:admin ---- - +[NOTE] +==== +The connector requires a minimum scope of `user:read:list_users:admin` to ingest data into Elasticsearch. +==== ++ 10. Click on the "Done" button to add the selected scopes to your app. 11. Navigate to the "Activation" section and input the necessary information to activate the app. @@ -220,18 +224,22 @@ To connect to Zoom you need to https://developers.zoom.us/docs/internal-apps/s2s 6. Click on the "Create" button to create the app registration. 7. After the registration is complete, you will be redirected to the app's overview page. Take note of the "App Credentials" value, as you'll need it later. 8. Navigate to the "Scopes" section and click on the "Add Scopes" button. -9. The following scopes need to be added to the app. +9. The following granular scopes need to be added to the app. + [source,bash] ---- -user:read:admin -meeting:read:admin -chat_channel:read:admin -recording:read:admin -chat_message:read:admin -report:read:admin +user:read:list_users:admin +meeting:read:list_meetings:admin +meeting:read:list_past_participants:admin +cloud_recording:read:list_user_recordings:admin +team_chat:read:list_user_channels:admin +team_chat:read:list_user_messages:admin ---- - +[NOTE] +==== +The connector requires a minimum scope of `user:read:list_users:admin` to ingest data into Elasticsearch. +==== ++ 10. Click on the "Done" button to add the selected scopes to your app. 11. Navigate to the "Activation" section and input the necessary information to activate the app. diff --git a/docs/reference/esql/functions/description/reverse.asciidoc b/docs/reference/esql/functions/description/reverse.asciidoc new file mode 100644 index 0000000000000..fbb3f3f6b4d54 --- /dev/null +++ b/docs/reference/esql/functions/description/reverse.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Returns a new string representing the input string in reverse order. diff --git a/docs/reference/esql/functions/examples/reverse.asciidoc b/docs/reference/esql/functions/examples/reverse.asciidoc new file mode 100644 index 0000000000000..67c8af077b174 --- /dev/null +++ b/docs/reference/esql/functions/examples/reverse.asciidoc @@ -0,0 +1,22 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Examples* + +[source.merge.styled,esql] +---- +include::{esql-specs}/string.csv-spec[tag=reverse] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/string.csv-spec[tag=reverse-result] +|=== +`REVERSE` works with unicode, too! It keeps unicode grapheme clusters together during reversal. +[source.merge.styled,esql] +---- +include::{esql-specs}/string.csv-spec[tag=reverseEmoji] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/string.csv-spec[tag=reverseEmoji-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/greatest.json b/docs/reference/esql/functions/kibana/definition/greatest.json index 0e32fca5b4ca1..2818a5ac56339 100644 --- a/docs/reference/esql/functions/kibana/definition/greatest.json +++ b/docs/reference/esql/functions/kibana/definition/greatest.json @@ -35,6 +35,24 @@ "variadic" : true, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "first", + "type" : "date", + "optional" : false, + "description" : "First of the columns to evaluate." + }, + { + "name" : "rest", + "type" : "date", + "optional" : true, + "description" : "The rest of the columns to evaluate." + } + ], + "variadic" : true, + "returnType" : "date" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/kibana/definition/least.json b/docs/reference/esql/functions/kibana/definition/least.json index 0ba34cf3cc9a2..7b545896f4ddc 100644 --- a/docs/reference/esql/functions/kibana/definition/least.json +++ b/docs/reference/esql/functions/kibana/definition/least.json @@ -34,6 +34,24 @@ "variadic" : true, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "first", + "type" : "date", + "optional" : false, + "description" : "First of the columns to evaluate." + }, + { + "name" : "rest", + "type" : "date", + "optional" : true, + "description" : "The rest of the columns to evaluate." + } + ], + "variadic" : true, + "returnType" : "date" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/kibana/definition/reverse.json b/docs/reference/esql/functions/kibana/definition/reverse.json new file mode 100644 index 0000000000000..1b222691530f2 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/reverse.json @@ -0,0 +1,38 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "reverse", + "description" : "Returns a new string representing the input string in reverse order.", + "signatures" : [ + { + "params" : [ + { + "name" : "str", + "type" : "keyword", + "optional" : false, + "description" : "String expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "str", + "type" : "text", + "optional" : false, + "description" : "String expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "text" + } + ], + "examples" : [ + "ROW message = \"Some Text\" | EVAL message_reversed = REVERSE(message);", + "ROW bending_arts = \"💧🪨🔥💨\" | EVAL bending_arts_reversed = REVERSE(bending_arts);" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/docs/reverse.md b/docs/reference/esql/functions/kibana/docs/reverse.md new file mode 100644 index 0000000000000..cbeade9189d80 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/reverse.md @@ -0,0 +1,10 @@ + + +### REVERSE +Returns a new string representing the input string in reverse order. + +``` +ROW message = "Some Text" | EVAL message_reversed = REVERSE(message); +``` diff --git a/docs/reference/esql/functions/layout/reverse.asciidoc b/docs/reference/esql/functions/layout/reverse.asciidoc new file mode 100644 index 0000000000000..99c236d63492e --- /dev/null +++ b/docs/reference/esql/functions/layout/reverse.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-reverse]] +=== `REVERSE` + +*Syntax* + +[.text-center] +image::esql/functions/signature/reverse.svg[Embedded,opts=inline] + +include::../parameters/reverse.asciidoc[] +include::../description/reverse.asciidoc[] +include::../types/reverse.asciidoc[] +include::../examples/reverse.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/reverse.asciidoc b/docs/reference/esql/functions/parameters/reverse.asciidoc new file mode 100644 index 0000000000000..d56d115662491 --- /dev/null +++ b/docs/reference/esql/functions/parameters/reverse.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`str`:: +String expression. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/signature/reverse.svg b/docs/reference/esql/functions/signature/reverse.svg new file mode 100644 index 0000000000000..c23ce5583a8c0 --- /dev/null +++ b/docs/reference/esql/functions/signature/reverse.svg @@ -0,0 +1 @@ +REVERSE(str) \ No newline at end of file diff --git a/docs/reference/esql/functions/string-functions.asciidoc b/docs/reference/esql/functions/string-functions.asciidoc index ed97769b900e7..f5222330d579d 100644 --- a/docs/reference/esql/functions/string-functions.asciidoc +++ b/docs/reference/esql/functions/string-functions.asciidoc @@ -17,6 +17,7 @@ * <> * <> * <> +* <> * <> * <> * <> @@ -38,6 +39,7 @@ include::layout/locate.asciidoc[] include::layout/ltrim.asciidoc[] include::layout/repeat.asciidoc[] include::layout/replace.asciidoc[] +include::layout/reverse.asciidoc[] include::layout/right.asciidoc[] include::layout/rtrim.asciidoc[] include::layout/space.asciidoc[] diff --git a/docs/reference/esql/functions/types/greatest.asciidoc b/docs/reference/esql/functions/types/greatest.asciidoc index 537be55cd17ef..1454bbb6f81c1 100644 --- a/docs/reference/esql/functions/types/greatest.asciidoc +++ b/docs/reference/esql/functions/types/greatest.asciidoc @@ -7,6 +7,7 @@ first | rest | result boolean | boolean | boolean boolean | | boolean +date | date | date double | double | double integer | integer | integer integer | | integer diff --git a/docs/reference/esql/functions/types/least.asciidoc b/docs/reference/esql/functions/types/least.asciidoc index 537be55cd17ef..1454bbb6f81c1 100644 --- a/docs/reference/esql/functions/types/least.asciidoc +++ b/docs/reference/esql/functions/types/least.asciidoc @@ -7,6 +7,7 @@ first | rest | result boolean | boolean | boolean boolean | | boolean +date | date | date double | double | double integer | integer | integer integer | | integer diff --git a/docs/reference/esql/functions/types/reverse.asciidoc b/docs/reference/esql/functions/types/reverse.asciidoc new file mode 100644 index 0000000000000..974066d225bca --- /dev/null +++ b/docs/reference/esql/functions/types/reverse.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +str | result +keyword | keyword +text | text +|=== diff --git a/docs/reference/ingest/processors/inference.asciidoc b/docs/reference/ingest/processors/inference.asciidoc index fa4f246cdd7c8..4699f634afe37 100644 --- a/docs/reference/ingest/processors/inference.asciidoc +++ b/docs/reference/ingest/processors/inference.asciidoc @@ -169,6 +169,18 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -224,6 +236,18 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -304,6 +328,23 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`span`:::: +(Optional, integer) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] + +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -363,6 +404,18 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -424,6 +477,22 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`span`:::: +(Optional, integer) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] + +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -515,6 +584,18 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] diff --git a/docs/reference/mapping/runtime.asciidoc b/docs/reference/mapping/runtime.asciidoc index 190081fa801b4..1ee1194279061 100644 --- a/docs/reference/mapping/runtime.asciidoc +++ b/docs/reference/mapping/runtime.asciidoc @@ -821,8 +821,6 @@ address. [[lookup-runtime-fields]] ==== Retrieve fields from related indices -experimental[] - The <> parameter on the `_search` API can also be used to retrieve fields from the related indices via runtime fields with a type of `lookup`. diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc index 44e1c2949775e..ca2c23f932fc3 100644 --- a/docs/reference/mapping/types/date.asciidoc +++ b/docs/reference/mapping/types/date.asciidoc @@ -125,8 +125,7 @@ The following parameters are accepted by `date` fields: `locale`:: The locale to use when parsing dates since months do not have the same names - and/or abbreviations in all languages. The default is the - https://docs.oracle.com/javase/8/docs/api/java/util/Locale.html#ROOT[`ROOT` locale]. + and/or abbreviations in all languages. The default is ENGLISH. <>:: diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index 97122141d7558..ef19fbf4e267d 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -988,6 +988,7 @@ values are + -- * `bert`: Use for BERT-style models +* `deberta_v2`: Use for DeBERTa v2 and v3-style models * `mpnet`: Use for MPNet-style models * `roberta`: Use for RoBERTa-style and BART-style models * experimental:[] `xlm_roberta`: Use for XLMRoBERTa-style models @@ -1037,6 +1038,19 @@ sequence. Therefore, do not use `second` in this case. end::inference-config-nlp-tokenization-truncate[] +tag::inference-config-nlp-tokenization-truncate-deberta-v2[] +Indicates how tokens are truncated when they exceed `max_sequence_length`. +The default value is `first`. ++ +-- +* `balanced`: One or both of the first and second sequences may be truncated so as to balance the tokens included from both sequences. +* `none`: No truncation occurs; the inference request receives an error. +* `first`: Only the first sequence is truncated. +* `second`: Only the second sequence is truncated. If there is just one sequence, that sequence is truncated. +-- + +end::inference-config-nlp-tokenization-truncate-deberta-v2[] + tag::inference-config-nlp-tokenization-bert-with-special-tokens[] Tokenize with special tokens. The tokens typically included in BERT-style tokenization are: + @@ -1050,10 +1064,23 @@ tag::inference-config-nlp-tokenization-bert-ja-with-special-tokens[] Tokenize with special tokens if `true`. end::inference-config-nlp-tokenization-bert-ja-with-special-tokens[] +tag::inference-config-nlp-tokenization-deberta-v2[] +DeBERTa-style tokenization is to be performed with the enclosed settings. +end::inference-config-nlp-tokenization-deberta-v2[] + tag::inference-config-nlp-tokenization-max-sequence-length[] Specifies the maximum number of tokens allowed to be output by the tokenizer. end::inference-config-nlp-tokenization-max-sequence-length[] +tag::inference-config-nlp-tokenization-deberta-v2-with-special-tokens[] +Tokenize with special tokens. The tokens typically included in DeBERTa-style tokenization are: ++ +-- +* `[CLS]`: The first token of the sequence being classified. +* `[SEP]`: Indicates sequence separation and sequence end. +-- +end::inference-config-nlp-tokenization-deberta-v2-with-special-tokens[] + tag::inference-config-nlp-tokenization-roberta[] RoBERTa-style tokenization is to be performed with the enclosed settings. end::inference-config-nlp-tokenization-roberta[] diff --git a/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc b/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc index 9aac913e7559f..99c3ecad03a9d 100644 --- a/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc +++ b/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc @@ -137,6 +137,18 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, string) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] diff --git a/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc b/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc index e29bc8823ab29..32265af5f795b 100644 --- a/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc +++ b/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc @@ -773,6 +773,37 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, boolean) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] ==== +`deberta_v2`:: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +==== +`do_lower_case`::: +(Optional, boolean) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] ++ +-- +Defaults to `false`. +-- + +`max_sequence_length`::: +(Optional, integer) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] + +`span`::: +(Optional, integer) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] + +`truncate`::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] + +`with_special_tokens`::: +(Optional, boolean) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2-with-special-tokens] +==== `roberta`:: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] diff --git a/docs/reference/release-notes/8.15.2.asciidoc b/docs/reference/release-notes/8.15.2.asciidoc new file mode 100644 index 0000000000000..7dfd8690109b2 --- /dev/null +++ b/docs/reference/release-notes/8.15.2.asciidoc @@ -0,0 +1,42 @@ +[[release-notes-8.15.2]] +== {es} version 8.15.2 + +Also see <>. + +[[bug-8.15.2]] +[float] +=== Bug fixes + +Authorization:: +* Fix remote cluster credential secure settings reload {es-pull}111535[#111535] + +ES|QL:: +* ESQL: Don't mutate the `BoolQueryBuilder` in plan {es-pull}111519[#111519] +* ES|QL: Fix `ResolvedEnrichPolicy` serialization (bwc) in v 8.15 {es-pull}112985[#112985] (issue: {es-issue}112968[#112968]) +* Fix union-types where one index is missing the field {es-pull}111932[#111932] (issue: {es-issue}111912[#111912]) +* Support widening of numeric types in union-types {es-pull}112610[#112610] (issue: {es-issue}111277[#111277]) + +Infra/Core:: +* JSON parse failures should be 4xx codes {es-pull}112703[#112703] +* Json parsing exceptions should not cause 500 errors {es-pull}111548[#111548] (issue: {es-issue}111542[#111542]) +* Make sure file accesses in `DnRoleMapper` are done in stack frames with permissions {es-pull}112400[#112400] + +Ingest Node:: +* Fix missing header in `put_geoip_database` JSON spec {es-pull}112581[#112581] + +Logs:: +* Fix encoding of dynamic arrays in ignored source {es-pull}112713[#112713] + +Mapping:: +* Full coverage of ECS by ecs@mappings when `date_detection` is disabled {es-pull}112444[#112444] (issue: {es-issue}112398[#112398]) + +Search:: +* Fix parsing error in `_terms_enum` API {es-pull}112872[#112872] (issue: {es-issue}94378[#94378]) + +Security:: +* Allowlist `tracestate` header on remote server port {es-pull}112649[#112649] + +Vector Search:: +* Fix NPE in `dense_vector` stats {es-pull}112720[#112720] + + diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index bf5260928797c..1e0018f590ac0 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -72,16 +72,54 @@ version 8.16 `allow_rebalance` setting defaults to `always` unless the legacy al [discrete] [[add_global_retention_in_data_stream_lifecycle]] === Add global retention in data stream lifecycle -Data stream lifecycle now supports configuring retention on a cluster level, namely global retention. Global retention -allows us to configure two different retentions: +Data stream lifecycle now supports configuring retention on a cluster level, +namely global retention. Global retention \nallows us to configure two different +retentions: -- `data_streams.lifecycle.retention.default` is applied to all data streams managed by the data stream lifecycle that do not have retention -defined on the data stream level. -- `data_streams.lifecycle.retention.max` is applied to all data streams managed by the data stream lifecycle and it allows any data stream -data to be deleted after the `max_retention` has passed. +- `data_streams.lifecycle.retention.default` is applied to all data streams managed +by the data stream lifecycle that do not have retention defined on the data stream level. +- `data_streams.lifecycle.retention.max` is applied to all data streams managed by the +data stream lifecycle and it allows any data stream \ndata to be deleted after the `max_retention` has passed. {es-pull}111972[#111972] +[discrete] +[[enable_zstandard_compression_for_indices_with_index_codec_set_to_best_compression]] +=== Enable ZStandard compression for indices with index.codec set to best_compression +Before DEFLATE compression was used to compress stored fields in indices with index.codec index setting set to +best_compression, with this change ZStandard is used as compression algorithm to stored fields for indices with +index.codec index setting set to best_compression. The usage ZStandard results in less storage usage with a +similar indexing throughput depending on what options are used. Experiments with indexing logs have shown that +ZStandard offers ~12% lower storage usage and a ~14% higher indexing throughput compared to DEFLATE. + +{es-pull}112665[#112665] + // end::notable-highlights[] +[discrete] +[[esql_multi_value_fields_supported_in_geospatial_predicates]] +=== ESQL: Multi-value fields supported in Geospatial predicates +Supporting multi-value fields in `WHERE` predicates is a challenge due to not knowing whether `ALL` or `ANY` +of the values in the field should pass the predicate. +For example, should the field `age:[10,30]` pass the predicate `WHERE age>20` or not? +This ambiguity does not exist with the spatial predicates +`ST_INTERSECTS` and `ST_DISJOINT`, because the choice between `ANY` or `ALL` +is implied by the predicate itself. +Consider a predicate checking a field named `location` against a test geometry named `shape`: + +* `ST_INTERSECTS(field, shape)` - true if `ANY` value can intersect the shape +* `ST_DISJOINT(field, shape)` - true only if `ALL` values are disjoint from the shape + +This works even if the shape argument is itself a complex or compound geometry. + +Similar logic exists for `ST_CONTAINS` and `ST_WITHIN` predicates, but these are not as easily solved +with `ANY` or `ALL`, because a collection of geometries contains another collection if each of the contained +geometries is within at least one of the containing geometries. Evaluating this requires that the multi-value +field is first combined into a single geometry before performing the predicate check. + +* `ST_CONTAINS(field, shape)` - true if the combined geometry contains the shape +* `ST_WITHIN(field, shape)` - true if the combined geometry is within the shape + +{es-pull}112063[#112063] + diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index fabd495cdc525..993bb8cb894f9 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -1298,10 +1298,11 @@ tag::wait_for_active_shards[] `wait_for_active_shards`:: + -- -(Optional, string) The number of shard copies that must be active before -proceeding with the operation. Set to `all` or any positive integer up -to the total number of shards in the index (`number_of_replicas+1`). -Default: 1, the primary shard. +(Optional, string) The number of copies of each shard that must be active +before proceeding with the operation. Set to `all` or any non-negative integer +up to the total number of copies of each shard in the index +(`number_of_replicas+1`). Defaults to `1`, meaning to wait just for each +primary shard to be active. See <>. -- diff --git a/docs/reference/setup/install.asciidoc b/docs/reference/setup/install.asciidoc index 89373d0ce8d44..a38fdcfc36fd5 100644 --- a/docs/reference/setup/install.asciidoc +++ b/docs/reference/setup/install.asciidoc @@ -76,27 +76,29 @@ Docker container images may be downloaded from the Elastic Docker Registry. [[jvm-version]] === Java (JVM) Version -{es} is built using Java, and includes a bundled version of -https://openjdk.java.net[OpenJDK] from the JDK maintainers (GPLv2+CE) within -each distribution. The bundled JVM is the recommended JVM. - -To use your own version of Java, set the `ES_JAVA_HOME` environment variable. -If you must use a version of Java that is different from the bundled JVM, it is -best to use the latest release of a link:/support/matrix[supported] -https://www.oracle.com/technetwork/java/eol-135779.html[LTS version of Java]. -{es} is closely coupled to certain OpenJDK-specific features, so it may not -work correctly with other JVMs. {es} will refuse to start if a known-bad -version of Java is used. - -If you use a JVM other than the bundled one, you are responsible for reacting -to announcements related to its security issues and bug fixes, and must -yourself determine whether each update is necessary or not. In contrast, the -bundled JVM is treated as an integral part of {es}, which means that Elastic -takes responsibility for keeping it up to date. Security issues and bugs within -the bundled JVM are treated as if they were within {es} itself. - -The bundled JVM is located within the `jdk` subdirectory of the {es} home -directory. You may remove this directory if using your own JVM. +{es} is built using Java, and includes a bundled version of https://openjdk.java.net[OpenJDK] within each distribution. We strongly +recommend using the bundled JVM in all installations of {es}. + +The bundled JVM is treated the same as any other dependency of {es} in terms of support and maintenance. This means that Elastic takes +responsibility for keeping it up to date, and reacts to security issues and bug reports as needed to address vulnerabilities and other bugs +in {es}. Elastic's support of the bundled JVM is subject to Elastic's https://www.elastic.co/support_policy[support policy] and +https://www.elastic.co/support/eol[end-of-life schedule] and is independent of the support policy and end-of-life schedule offered by the +original supplier of the JVM. Elastic does not support using the bundled JVM for purposes other than running {es}. + +TIP: {es} uses only a subset of the features offered by the JVM. Bugs and security issues in the bundled JVM often relate to features that +{es} does not use. Such issues do not apply to {es}. Elastic analyzes reports of security vulnerabilities in all its dependencies, including +in the bundled JVM, and will issue an https://www.elastic.co/community/security[Elastic Security Advisory] if such an advisory is needed. + +If you decide to run {es} using a version of Java that is different from the bundled one, prefer to use the latest release of a +https://www.oracle.com/technetwork/java/eol-135779.html[LTS version of Java] which is link:/support/matrix[listed in the support matrix]. +Although such a configuration is supported, if you encounter a security issue or other bug in your chosen JVM then Elastic may not be able +to help unless the issue is also present in the bundled JVM. Instead, you must seek assistance directly from the supplier of your chosen +JVM. You must also take responsibility for reacting to security and bug announcements from the supplier of your chosen JVM. {es} may not +perform optimally if using a JVM other than the bundled one. {es} is closely coupled to certain OpenJDK-specific features, so it may not +work correctly with JVMs that are not OpenJDK. {es} will refuse to start if you attempt to use a known-bad JVM version. + +To use your own version of Java, set the `ES_JAVA_HOME` environment variable to the path to your own JVM installation. The bundled JVM is +located within the `jdk` subdirectory of the {es} home directory. You may remove this directory if using your own JVM. [discrete] [[jvm-agents]] diff --git a/docs/reference/snapshot-restore/repository-s3.asciidoc b/docs/reference/snapshot-restore/repository-s3.asciidoc index 1f55296139cd3..71a9fd8b87c96 100644 --- a/docs/reference/snapshot-restore/repository-s3.asciidoc +++ b/docs/reference/snapshot-restore/repository-s3.asciidoc @@ -296,9 +296,8 @@ include::repository-shared-settings.asciidoc[] `max_multipart_parts` :: - (<>) The maximum number of parts that {es} will write during a multipart upload - of a single object. Files which are larger than `buffer_size × max_multipart_parts` will be - chunked into several smaller objects. {es} may also split a file across multiple objects to + (integer) The maximum number of parts that {es} will write during a multipart upload of a single object. Files which are larger than + `buffer_size × max_multipart_parts` will be chunked into several smaller objects. {es} may also split a file across multiple objects to satisfy other constraints such as the `chunk_size` limit. Defaults to `10000` which is the https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html[maximum number of parts in a multipart upload in AWS S3]. @@ -321,20 +320,14 @@ include::repository-shared-settings.asciidoc[] `delete_objects_max_size`:: - (<>) Sets the maxmimum batch size, betewen 1 and 1000, used - for `DeleteObjects` requests. Defaults to 1000 which is the maximum number - supported by the - https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html[AWS - DeleteObjects API]. + (integer) Sets the maxmimum batch size, betewen 1 and 1000, used for `DeleteObjects` requests. Defaults to 1000 which is the maximum + number supported by the https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html[AWS DeleteObjects API]. `max_multipart_upload_cleanup_size`:: - (<>) Sets the maximum number of possibly-dangling multipart - uploads to clean up in each batch of snapshot deletions. Defaults to `1000` - which is the maximum number supported by the - https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html[AWS - ListMultipartUploads API]. If set to `0`, {es} will not attempt to clean up - dangling multipart uploads. + (integer) Sets the maximum number of possibly-dangling multipart uploads to clean up in each batch of snapshot deletions. Defaults to + `1000` which is the maximum number supported by the https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html[AWS + ListMultipartUploads API]. If set to `0`, {es} will not attempt to clean up dangling multipart uploads. NOTE: The option of defining client settings in the repository settings as documented below is considered deprecated, and will be removed in a future diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index f1c4b15ea5702..53a65e217ed18 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -749,9 +749,9 @@ - - - + + + @@ -759,9 +759,9 @@ - - - + + + diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/CellBoundary.java b/libs/h3/src/main/java/org/elasticsearch/h3/CellBoundary.java index 74115d5a002d6..e0f9df174c2b5 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/CellBoundary.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/CellBoundary.java @@ -22,36 +22,52 @@ */ package org.elasticsearch.h3; +import java.util.Arrays; +import java.util.Objects; + /** * cell boundary points as {@link LatLng} */ public final class CellBoundary { - /** Maximum number of cell boundary vertices; worst case is pentagon: * 5 original verts + 5 edge crossings */ - private static final int MAX_CELL_BNDRY_VERTS = 10; + static final int MAX_CELL_BNDRY_VERTS = 10; /** How many points it holds */ - private int numVertext; + private final int numPoints; /** The actual points */ - private final LatLng[] points = new LatLng[MAX_CELL_BNDRY_VERTS]; - - CellBoundary() {} + private final LatLng[] points; - void add(LatLng point) { - points[numVertext++] = point; + CellBoundary(LatLng[] points, int numPoints) { + this.points = points; + this.numPoints = numPoints; } /** Number of points in this boundary */ public int numPoints() { - return numVertext; + return numPoints; } /** Return the point at the given position*/ public LatLng getLatLon(int i) { - if (i >= numVertext) { - throw new IndexOutOfBoundsException(); - } + assert i >= 0 && i < numPoints; return points[i]; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final CellBoundary that = (CellBoundary) o; + return numPoints == that.numPoints && Arrays.equals(points, that.points); + } + + @Override + public int hashCode() { + return Objects.hash(numPoints, Arrays.hashCode(points)); + } } diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java b/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java index 570052700615f..3b3f760c0534f 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java @@ -34,10 +34,6 @@ final class Constants { * 2.0 * PI */ public static final double M_2PI = 2.0 * Math.PI; - /** - * max H3 resolution; H3 version 1 has 16 resolutions, numbered 0 through 15 - */ - public static int MAX_H3_RES = 15; /** * The number of H3 base cells */ diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java b/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java index ae59ff359d1f8..866fdfe8a7f8b 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java @@ -439,7 +439,8 @@ public CellBoundary faceIjkPentToCellBoundary(int res, int start, int length) { // convert each vertex to lat/lng // adjust the face of each vertex as appropriate and introduce // edge-crossing vertices as needed - final CellBoundary boundary = new CellBoundary(); + final LatLng[] points = new LatLng[CellBoundary.MAX_CELL_BNDRY_VERTS]; + int numPoints = 0; final CoordIJK scratch = new CoordIJK(0, 0, 0); final FaceIJK fijk = new FaceIJK(this.face, scratch); final int[][] coord = isResolutionClassIII ? VERTEX_CLASSIII : VERTEX_CLASSII; @@ -501,21 +502,19 @@ public CellBoundary faceIjkPentToCellBoundary(int res, int start, int length) { // find the intersection and add the lat/lng point to the result final Vec2d inter = Vec2d.v2dIntersect(orig2d0, orig2d1, edge0, edge1); - final LatLng point = inter.hex2dToGeo(fijkOrient.face, adjRes, true); - boundary.add(point); + points[numPoints++] = inter.hex2dToGeo(fijkOrient.face, adjRes, true); } // convert vertex to lat/lng and add to the result // vert == start + NUM_PENT_VERTS is only used to test for possible // intersection on last edge if (vert < start + Constants.NUM_PENT_VERTS) { - final LatLng point = fijk.coord.ijkToGeo(fijk.face, adjRes, true); - boundary.add(point); + points[numPoints++] = fijk.coord.ijkToGeo(fijk.face, adjRes, true); } lastFace = fijk.face; lastCoord.reset(fijk.coord.i, fijk.coord.j, fijk.coord.k); } - return boundary; + return new CellBoundary(points, numPoints); } /** @@ -547,7 +546,8 @@ public CellBoundary faceIjkToCellBoundary(final int res, final int start, final // convert each vertex to lat/lng // adjust the face of each vertex as appropriate and introduce // edge-crossing vertices as needed - final CellBoundary boundary = new CellBoundary(); + final LatLng[] points = new LatLng[CellBoundary.MAX_CELL_BNDRY_VERTS]; + int numPoints = 0; final CoordIJK scratch1 = new CoordIJK(0, 0, 0); final FaceIJK fijk = new FaceIJK(this.face, scratch1); final CoordIJK scratch2 = isResolutionClassIII ? new CoordIJK(0, 0, 0) : null; @@ -616,8 +616,7 @@ public CellBoundary faceIjkToCellBoundary(final int res, final int start, final */ final boolean isIntersectionAtVertex = orig2d0.numericallyIdentical(inter) || orig2d1.numericallyIdentical(inter); if (isIntersectionAtVertex == false) { - final LatLng point = inter.hex2dToGeo(this.face, adjRes, true); - boundary.add(point); + points[numPoints++] = inter.hex2dToGeo(this.face, adjRes, true); } } @@ -625,13 +624,12 @@ public CellBoundary faceIjkToCellBoundary(final int res, final int start, final // vert == start + NUM_HEX_VERTS is only used to test for possible // intersection on last edge if (vert < start + Constants.NUM_HEX_VERTS) { - final LatLng point = fijk.coord.ijkToGeo(fijk.face, adjRes, true); - boundary.add(point); + points[numPoints++] = fijk.coord.ijkToGeo(fijk.face, adjRes, true); } lastFace = fijk.face; lastOverage = overage; } - return boundary; + return new CellBoundary(points, numPoints); } /** diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/H3.java b/libs/h3/src/main/java/org/elasticsearch/h3/H3.java index 46bcc3f141dde..8c0bba62cecdb 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/H3.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/H3.java @@ -30,8 +30,10 @@ * Defines the public API of the H3 library. */ public final class H3 { - - public static int MAX_H3_RES = Constants.MAX_H3_RES; + /** + * max H3 resolution; H3 version 1 has 16 resolutions, numbered 0 through 15 + */ + public static int MAX_H3_RES = 15; private static final long[] NORTH = new long[MAX_H3_RES + 1]; private static final long[] SOUTH = new long[MAX_H3_RES + 1]; @@ -97,7 +99,7 @@ public static boolean h3IsValid(long h3) { } int res = H3Index.H3_get_resolution(h3); - if (res < 0 || res > Constants.MAX_H3_RES) { // LCOV_EXCL_BR_LINE + if (res < 0 || res > MAX_H3_RES) { // LCOV_EXCL_BR_LINE // Resolutions less than zero can not be represented in an index return false; } @@ -118,7 +120,7 @@ public static boolean h3IsValid(long h3) { } } - for (int r = res + 1; r <= Constants.MAX_H3_RES; r++) { + for (int r = res + 1; r <= MAX_H3_RES; r++) { int digit = H3Index.H3_get_index_digit(h3, r); if (digit != CoordIJK.Direction.INVALID_DIGIT.digit()) { return false; @@ -601,7 +603,7 @@ private static String[] h3ToStringList(long[] h3s) { * @throws IllegalArgumentException res is not a valid H3 resolution. */ private static void checkResolution(int res) { - if (res < 0 || res > Constants.MAX_H3_RES) { + if (res < 0 || res > MAX_H3_RES) { throw new IllegalArgumentException("resolution [" + res + "] is out of range (must be 0 <= res <= 15)"); } } diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java b/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java index 7babedc55eb0e..2b1b9cade21a4 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java @@ -160,14 +160,14 @@ public static int H3_get_resolution(long h3) { * Gets the resolution res integer digit (0-7) of h3. */ public static int H3_get_index_digit(long h3, int res) { - return ((int) ((((h3) >> ((Constants.MAX_H3_RES - (res)) * H3_PER_DIGIT_OFFSET)) & H3_DIGIT_MASK))); + return ((int) ((((h3) >> ((H3.MAX_H3_RES - (res)) * H3_PER_DIGIT_OFFSET)) & H3_DIGIT_MASK))); } /** * Sets the resolution res digit of h3 to the integer digit (0-7) */ public static long H3_set_index_digit(long h3, int res, long digit) { - int x = (Constants.MAX_H3_RES - res) * H3_PER_DIGIT_OFFSET; + int x = (H3.MAX_H3_RES - res) * H3_PER_DIGIT_OFFSET; return (((h3) & ~((H3_DIGIT_MASK << (x)))) | (((digit)) << x)); } diff --git a/libs/h3/src/test/java/org/elasticsearch/h3/CellBoundaryTests.java b/libs/h3/src/test/java/org/elasticsearch/h3/CellBoundaryTests.java index 903e4ed40ec16..00ca6f7021e3d 100644 --- a/libs/h3/src/test/java/org/elasticsearch/h3/CellBoundaryTests.java +++ b/libs/h3/src/test/java/org/elasticsearch/h3/CellBoundaryTests.java @@ -218,4 +218,22 @@ private boolean isSharedBoundary(int clon1, int clat1, int clon2, int clat2, Cel } return false; } + + public void testEqualsAndHashCode() { + final long h3 = H3.geoToH3(GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude(), randomIntBetween(0, 15)); + final CellBoundary boundary1 = H3.h3ToGeoBoundary(h3); + final CellBoundary boundary2 = H3.h3ToGeoBoundary(h3); + assertEquals(boundary1, boundary2); + assertEquals(boundary1.hashCode(), boundary2.hashCode()); + + final long otherH3 = H3.geoToH3(GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude(), randomIntBetween(0, 15)); + final CellBoundary otherCellBoundary = H3.h3ToGeoBoundary(otherH3); + if (otherH3 != h3) { + assertNotEquals(boundary1, otherCellBoundary); + assertNotEquals(boundary1.hashCode(), otherCellBoundary.hashCode()); + } else { + assertEquals(boundary1, otherCellBoundary); + assertEquals(boundary1.hashCode(), otherCellBoundary.hashCode()); + } + } } diff --git a/libs/h3/src/test/java/org/elasticsearch/h3/GeoToH3Tests.java b/libs/h3/src/test/java/org/elasticsearch/h3/GeoToH3Tests.java index cb7d416a5a9d3..3f2c329d9ff3c 100644 --- a/libs/h3/src/test/java/org/elasticsearch/h3/GeoToH3Tests.java +++ b/libs/h3/src/test/java/org/elasticsearch/h3/GeoToH3Tests.java @@ -38,7 +38,7 @@ public void testRandomPoints() { private void testPoint(double lat, double lon) { GeoPoint point = new GeoPoint(PlanetModel.SPHERE, Math.toRadians(lat), Math.toRadians(lon)); - for (int res = 0; res < Constants.MAX_H3_RES; res++) { + for (int res = 0; res < H3.MAX_H3_RES; res++) { String h3Address = H3.geoToH3Address(lat, lon, res); assertEquals(res, H3.getResolution(h3Address)); GeoPolygon polygon = getGeoPolygon(h3Address); diff --git a/libs/h3/src/test/java/org/elasticsearch/h3/HexRingTests.java b/libs/h3/src/test/java/org/elasticsearch/h3/HexRingTests.java index 8fe5c6206fff8..864c0322cac90 100644 --- a/libs/h3/src/test/java/org/elasticsearch/h3/HexRingTests.java +++ b/libs/h3/src/test/java/org/elasticsearch/h3/HexRingTests.java @@ -38,7 +38,7 @@ public void testHexRing() { for (int i = 0; i < 500; i++) { double lat = GeoTestUtil.nextLatitude(); double lon = GeoTestUtil.nextLongitude(); - for (int res = 0; res <= Constants.MAX_H3_RES; res++) { + for (int res = 0; res <= H3.MAX_H3_RES; res++) { String origin = H3.geoToH3Address(lat, lon, res); assertFalse(H3.areNeighborCells(origin, origin)); String[] ring = H3.hexRing(origin); diff --git a/libs/simdvec/build.gradle b/libs/simdvec/build.gradle index 5a523a19d4b68..dab5c25b34679 100644 --- a/libs/simdvec/build.gradle +++ b/libs/simdvec/build.gradle @@ -23,6 +23,20 @@ dependencies { } } +// compileMain21Java does not exist within idea (see MrJarPlugin) so we cannot reference directly by name +tasks.matching { it.name == "compileMain21Java" }.configureEach { + options.compilerArgs << '--add-modules=jdk.incubator.vector' + // we remove Werror, since incubating suppression (-Xlint:-incubating) + // is only support since JDK 22 + options.compilerArgs -= '-Werror' +} + +tasks.named('test').configure { + if (JavaVersion.current().majorVersion.toInteger() >= 21) { + jvmArgs '--add-modules=jdk.incubator.vector' + } +} + tasks.withType(CheckForbiddenApisTask).configureEach { replaceSignatureFiles 'jdk-signatures' } diff --git a/libs/simdvec/src/main/java/module-info.java b/libs/simdvec/src/main/java/module-info.java index 64e685ba3cbb5..44f6e39d5dbab 100644 --- a/libs/simdvec/src/main/java/module-info.java +++ b/libs/simdvec/src/main/java/module-info.java @@ -10,6 +10,7 @@ module org.elasticsearch.simdvec { requires org.elasticsearch.nativeaccess; requires org.apache.lucene.core; + requires org.elasticsearch.logging; exports org.elasticsearch.simdvec to org.elasticsearch.server; } diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java new file mode 100644 index 0000000000000..91193d5fa6eaf --- /dev/null +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec; + +import org.elasticsearch.simdvec.internal.vectorization.ESVectorUtilSupport; +import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider; + +import static org.elasticsearch.simdvec.internal.vectorization.ESVectorUtilSupport.B_QUERY; + +public class ESVectorUtil { + + private static final ESVectorUtilSupport IMPL = ESVectorizationProvider.getInstance().getVectorUtilSupport(); + + public static long ipByteBinByte(byte[] q, byte[] d) { + if (q.length != d.length * B_QUERY) { + throw new IllegalArgumentException("vector dimensions incompatible: " + q.length + "!= " + B_QUERY + " x " + d.length); + } + return IMPL.ipByteBinByte(q, d); + } +} diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorUtilSupport.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorUtilSupport.java new file mode 100644 index 0000000000000..4a08096119d6a --- /dev/null +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorUtilSupport.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +import org.apache.lucene.util.BitUtil; + +final class DefaultESVectorUtilSupport implements ESVectorUtilSupport { + + DefaultESVectorUtilSupport() {} + + @Override + public long ipByteBinByte(byte[] q, byte[] d) { + return ipByteBinByteImpl(q, d); + } + + public static long ipByteBinByteImpl(byte[] q, byte[] d) { + long ret = 0; + int size = d.length; + for (int i = 0; i < B_QUERY; i++) { + int r = 0; + long subRet = 0; + for (final int upperBound = d.length & -Integer.BYTES; r < upperBound; r += Integer.BYTES) { + subRet += Integer.bitCount((int) BitUtil.VH_NATIVE_INT.get(q, i * size + r) & (int) BitUtil.VH_NATIVE_INT.get(d, r)); + } + for (; r < d.length; r++) { + subRet += Integer.bitCount((q[i * size + r] & d[r]) & 0xFF); + } + ret += subRet << i; + } + return ret; + } +} diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorizationProvider.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorizationProvider.java new file mode 100644 index 0000000000000..6c0f7ed146b86 --- /dev/null +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorizationProvider.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +final class DefaultESVectorizationProvider extends ESVectorizationProvider { + private final ESVectorUtilSupport vectorUtilSupport; + + DefaultESVectorizationProvider() { + vectorUtilSupport = new DefaultESVectorUtilSupport(); + } + + @Override + public ESVectorUtilSupport getVectorUtilSupport() { + return vectorUtilSupport; + } +} diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorUtilSupport.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorUtilSupport.java new file mode 100644 index 0000000000000..d7611173ca693 --- /dev/null +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorUtilSupport.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +public interface ESVectorUtilSupport { + + short B_QUERY = 4; + + long ipByteBinByte(byte[] q, byte[] d); +} diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java new file mode 100644 index 0000000000000..e541c10e145bf --- /dev/null +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +import java.util.Objects; + +public abstract class ESVectorizationProvider { + + public static ESVectorizationProvider getInstance() { + return Objects.requireNonNull( + ESVectorizationProvider.Holder.INSTANCE, + "call to getInstance() from subclass of VectorizationProvider" + ); + } + + ESVectorizationProvider() {} + + public abstract ESVectorUtilSupport getVectorUtilSupport(); + + // visible for tests + static ESVectorizationProvider lookup(boolean testMode) { + return new DefaultESVectorizationProvider(); + } + + /** This static holder class prevents classloading deadlock. */ + private static final class Holder { + private Holder() {} + + static final ESVectorizationProvider INSTANCE = lookup(false); + } +} diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java new file mode 100644 index 0000000000000..5b7aab7ddfa48 --- /dev/null +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +import org.apache.lucene.util.Constants; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; + +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; + +public abstract class ESVectorizationProvider { + + protected static final Logger logger = LogManager.getLogger(ESVectorizationProvider.class); + + public static ESVectorizationProvider getInstance() { + return Objects.requireNonNull( + ESVectorizationProvider.Holder.INSTANCE, + "call to getInstance() from subclass of VectorizationProvider" + ); + } + + ESVectorizationProvider() {} + + public abstract ESVectorUtilSupport getVectorUtilSupport(); + + // visible for tests + static ESVectorizationProvider lookup(boolean testMode) { + final int runtimeVersion = Runtime.version().feature(); + assert runtimeVersion >= 21; + if (runtimeVersion <= 23) { + // only use vector module with Hotspot VM + if (Constants.IS_HOTSPOT_VM == false) { + logger.warn("Java runtime is not using Hotspot VM; Java vector incubator API can't be enabled."); + return new DefaultESVectorizationProvider(); + } + // is the incubator module present and readable (JVM providers may to exclude them or it is + // build with jlink) + final var vectorMod = lookupVectorModule(); + if (vectorMod.isEmpty()) { + logger.warn( + "Java vector incubator module is not readable. " + + "For optimal vector performance, pass '--add-modules jdk.incubator.vector' to enable Vector API." + ); + return new DefaultESVectorizationProvider(); + } + vectorMod.ifPresent(ESVectorizationProvider.class.getModule()::addReads); + var impl = new PanamaESVectorizationProvider(); + logger.info( + String.format( + Locale.ENGLISH, + "Java vector incubator API enabled; uses preferredBitSize=%d", + PanamaESVectorUtilSupport.VECTOR_BITSIZE + ) + ); + return impl; + } else { + logger.warn( + "You are running with unsupported Java " + + runtimeVersion + + ". To make full use of the Vector API, please update Elasticsearch." + ); + } + return new DefaultESVectorizationProvider(); + } + + private static Optional lookupVectorModule() { + return Optional.ofNullable(ESVectorizationProvider.class.getModule().getLayer()) + .orElse(ModuleLayer.boot()) + .findModule("jdk.incubator.vector"); + } + + /** This static holder class prevents classloading deadlock. */ + private static final class Holder { + private Holder() {} + + static final ESVectorizationProvider INSTANCE = lookup(false); + } +} diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorUtilSupport.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorUtilSupport.java new file mode 100644 index 0000000000000..0e5827d046736 --- /dev/null +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorUtilSupport.java @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +import jdk.incubator.vector.ByteVector; +import jdk.incubator.vector.IntVector; +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.VectorOperators; +import jdk.incubator.vector.VectorShape; +import jdk.incubator.vector.VectorSpecies; + +import org.apache.lucene.util.Constants; + +public final class PanamaESVectorUtilSupport implements ESVectorUtilSupport { + + static final int VECTOR_BITSIZE; + + /** Whether integer vectors can be trusted to actually be fast. */ + static final boolean HAS_FAST_INTEGER_VECTORS; + + static { + // default to platform supported bitsize + VECTOR_BITSIZE = VectorShape.preferredShape().vectorBitSize(); + + // hotspot misses some SSE intrinsics, workaround it + // to be fair, they do document this thing only works well with AVX2/AVX3 and Neon + boolean isAMD64withoutAVX2 = Constants.OS_ARCH.equals("amd64") && VECTOR_BITSIZE < 256; + HAS_FAST_INTEGER_VECTORS = isAMD64withoutAVX2 == false; + } + + @Override + public long ipByteBinByte(byte[] q, byte[] d) { + // 128 / 8 == 16 + if (d.length >= 16 && HAS_FAST_INTEGER_VECTORS) { + if (VECTOR_BITSIZE >= 256) { + return ipByteBin256(q, d); + } else if (VECTOR_BITSIZE == 128) { + return ipByteBin128(q, d); + } + } + return DefaultESVectorUtilSupport.ipByteBinByteImpl(q, d); + } + + private static final VectorSpecies BYTE_SPECIES_128 = ByteVector.SPECIES_128; + private static final VectorSpecies BYTE_SPECIES_256 = ByteVector.SPECIES_256; + + static long ipByteBin256(byte[] q, byte[] d) { + long subRet0 = 0; + long subRet1 = 0; + long subRet2 = 0; + long subRet3 = 0; + int i = 0; + + if (d.length >= ByteVector.SPECIES_256.vectorByteSize() * 2) { + int limit = ByteVector.SPECIES_256.loopBound(d.length); + var sum0 = LongVector.zero(LongVector.SPECIES_256); + var sum1 = LongVector.zero(LongVector.SPECIES_256); + var sum2 = LongVector.zero(LongVector.SPECIES_256); + var sum3 = LongVector.zero(LongVector.SPECIES_256); + for (; i < limit; i += ByteVector.SPECIES_256.length()) { + var vq0 = ByteVector.fromArray(BYTE_SPECIES_256, q, i).reinterpretAsLongs(); + var vq1 = ByteVector.fromArray(BYTE_SPECIES_256, q, i + d.length).reinterpretAsLongs(); + var vq2 = ByteVector.fromArray(BYTE_SPECIES_256, q, i + d.length * 2).reinterpretAsLongs(); + var vq3 = ByteVector.fromArray(BYTE_SPECIES_256, q, i + d.length * 3).reinterpretAsLongs(); + var vd = ByteVector.fromArray(BYTE_SPECIES_256, d, i).reinterpretAsLongs(); + sum0 = sum0.add(vq0.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum1 = sum1.add(vq1.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum2 = sum2.add(vq2.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum3 = sum3.add(vq3.and(vd).lanewise(VectorOperators.BIT_COUNT)); + } + subRet0 += sum0.reduceLanes(VectorOperators.ADD); + subRet1 += sum1.reduceLanes(VectorOperators.ADD); + subRet2 += sum2.reduceLanes(VectorOperators.ADD); + subRet3 += sum3.reduceLanes(VectorOperators.ADD); + } + + if (d.length - i >= ByteVector.SPECIES_128.vectorByteSize()) { + var sum0 = LongVector.zero(LongVector.SPECIES_128); + var sum1 = LongVector.zero(LongVector.SPECIES_128); + var sum2 = LongVector.zero(LongVector.SPECIES_128); + var sum3 = LongVector.zero(LongVector.SPECIES_128); + int limit = ByteVector.SPECIES_128.loopBound(d.length); + for (; i < limit; i += ByteVector.SPECIES_128.length()) { + var vq0 = ByteVector.fromArray(BYTE_SPECIES_128, q, i).reinterpretAsLongs(); + var vq1 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + d.length).reinterpretAsLongs(); + var vq2 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + d.length * 2).reinterpretAsLongs(); + var vq3 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + d.length * 3).reinterpretAsLongs(); + var vd = ByteVector.fromArray(BYTE_SPECIES_128, d, i).reinterpretAsLongs(); + sum0 = sum0.add(vq0.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum1 = sum1.add(vq1.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum2 = sum2.add(vq2.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum3 = sum3.add(vq3.and(vd).lanewise(VectorOperators.BIT_COUNT)); + } + subRet0 += sum0.reduceLanes(VectorOperators.ADD); + subRet1 += sum1.reduceLanes(VectorOperators.ADD); + subRet2 += sum2.reduceLanes(VectorOperators.ADD); + subRet3 += sum3.reduceLanes(VectorOperators.ADD); + } + // tail as bytes + for (; i < d.length; i++) { + subRet0 += Integer.bitCount((q[i] & d[i]) & 0xFF); + subRet1 += Integer.bitCount((q[i + d.length] & d[i]) & 0xFF); + subRet2 += Integer.bitCount((q[i + 2 * d.length] & d[i]) & 0xFF); + subRet3 += Integer.bitCount((q[i + 3 * d.length] & d[i]) & 0xFF); + } + return subRet0 + (subRet1 << 1) + (subRet2 << 2) + (subRet3 << 3); + } + + public static long ipByteBin128(byte[] q, byte[] d) { + long subRet0 = 0; + long subRet1 = 0; + long subRet2 = 0; + long subRet3 = 0; + int i = 0; + + var sum0 = IntVector.zero(IntVector.SPECIES_128); + var sum1 = IntVector.zero(IntVector.SPECIES_128); + var sum2 = IntVector.zero(IntVector.SPECIES_128); + var sum3 = IntVector.zero(IntVector.SPECIES_128); + int limit = ByteVector.SPECIES_128.loopBound(d.length); + for (; i < limit; i += ByteVector.SPECIES_128.length()) { + var vd = ByteVector.fromArray(BYTE_SPECIES_128, d, i).reinterpretAsInts(); + var vq0 = ByteVector.fromArray(BYTE_SPECIES_128, q, i).reinterpretAsInts(); + var vq1 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + d.length).reinterpretAsInts(); + var vq2 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + d.length * 2).reinterpretAsInts(); + var vq3 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + d.length * 3).reinterpretAsInts(); + sum0 = sum0.add(vd.and(vq0).lanewise(VectorOperators.BIT_COUNT)); + sum1 = sum1.add(vd.and(vq1).lanewise(VectorOperators.BIT_COUNT)); + sum2 = sum2.add(vd.and(vq2).lanewise(VectorOperators.BIT_COUNT)); + sum3 = sum3.add(vd.and(vq3).lanewise(VectorOperators.BIT_COUNT)); + } + subRet0 += sum0.reduceLanes(VectorOperators.ADD); + subRet1 += sum1.reduceLanes(VectorOperators.ADD); + subRet2 += sum2.reduceLanes(VectorOperators.ADD); + subRet3 += sum3.reduceLanes(VectorOperators.ADD); + // tail as bytes + for (; i < d.length; i++) { + int dValue = d[i]; + subRet0 += Integer.bitCount((dValue & q[i]) & 0xFF); + subRet1 += Integer.bitCount((dValue & q[i + d.length]) & 0xFF); + subRet2 += Integer.bitCount((dValue & q[i + 2 * d.length]) & 0xFF); + subRet3 += Integer.bitCount((dValue & q[i + 3 * d.length]) & 0xFF); + } + return subRet0 + (subRet1 << 1) + (subRet2 << 2) + (subRet3 << 3); + } +} diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java new file mode 100644 index 0000000000000..62d25d79487ed --- /dev/null +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +final class PanamaESVectorizationProvider extends ESVectorizationProvider { + + private final ESVectorUtilSupport vectorUtilSupport; + + PanamaESVectorizationProvider() { + vectorUtilSupport = new PanamaESVectorUtilSupport(); + } + + @Override + public ESVectorUtilSupport getVectorUtilSupport() { + return vectorUtilSupport; + } +} diff --git a/libs/simdvec/src/test/java/org/elasticsearch/simdvec/ESVectorUtilTests.java b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/ESVectorUtilTests.java new file mode 100644 index 0000000000000..0dbc41c0c1055 --- /dev/null +++ b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/ESVectorUtilTests.java @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec; + +import org.elasticsearch.simdvec.internal.vectorization.BaseVectorizationTests; +import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider; + +import java.util.Arrays; + +import static org.elasticsearch.simdvec.internal.vectorization.ESVectorUtilSupport.B_QUERY; + +public class ESVectorUtilTests extends BaseVectorizationTests { + + static final ESVectorizationProvider defaultedProvider = BaseVectorizationTests.defaultProvider(); + static final ESVectorizationProvider defOrPanamaProvider = BaseVectorizationTests.maybePanamaProvider(); + + public void testIpByteBinInvariants() { + int iterations = atLeast(10); + for (int i = 0; i < iterations; i++) { + int size = randomIntBetween(1, 10); + var d = new byte[size]; + var q = new byte[size * B_QUERY - 1]; + expectThrows(IllegalArgumentException.class, () -> ESVectorUtil.ipByteBinByte(q, d)); + } + } + + public void testBasicIpByteBin() { + testBasicIpByteBinImpl(ESVectorUtil::ipByteBinByte); + testBasicIpByteBinImpl(defaultedProvider.getVectorUtilSupport()::ipByteBinByte); + testBasicIpByteBinImpl(defOrPanamaProvider.getVectorUtilSupport()::ipByteBinByte); + } + + interface IpByteBin { + long apply(byte[] q, byte[] d); + } + + void testBasicIpByteBinImpl(IpByteBin ipByteBinFunc) { + assertEquals(15L, ipByteBinFunc.apply(new byte[] { 1, 1, 1, 1 }, new byte[] { 1 })); + assertEquals(30L, ipByteBinFunc.apply(new byte[] { 1, 2, 1, 2, 1, 2, 1, 2 }, new byte[] { 1, 2 })); + + var d = new byte[] { 1, 2, 3 }; + var q = new byte[] { 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3 }; + assert scalarIpByteBin(q, d) == 60L; // 4 + 8 + 16 + 32 + assertEquals(60L, ipByteBinFunc.apply(q, d)); + + d = new byte[] { 1, 2, 3, 4 }; + q = new byte[] { 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4 }; + assert scalarIpByteBin(q, d) == 75L; // 5 + 10 + 20 + 40 + assertEquals(75L, ipByteBinFunc.apply(q, d)); + + d = new byte[] { 1, 2, 3, 4, 5 }; + q = new byte[] { 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5 }; + assert scalarIpByteBin(q, d) == 105L; // 7 + 14 + 28 + 56 + assertEquals(105L, ipByteBinFunc.apply(q, d)); + + d = new byte[] { 1, 2, 3, 4, 5, 6 }; + q = new byte[] { 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6 }; + assert scalarIpByteBin(q, d) == 135L; // 9 + 18 + 36 + 72 + assertEquals(135L, ipByteBinFunc.apply(q, d)); + + d = new byte[] { 1, 2, 3, 4, 5, 6, 7 }; + q = new byte[] { 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7 }; + assert scalarIpByteBin(q, d) == 180L; // 12 + 24 + 48 + 96 + assertEquals(180L, ipByteBinFunc.apply(q, d)); + + d = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }; + q = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 }; + assert scalarIpByteBin(q, d) == 195L; // 13 + 26 + 52 + 104 + assertEquals(195L, ipByteBinFunc.apply(q, d)); + + d = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + q = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + assert scalarIpByteBin(q, d) == 225L; // 15 + 30 + 60 + 120 + assertEquals(225L, ipByteBinFunc.apply(q, d)); + } + + public void testIpByteBin() { + testIpByteBinImpl(ESVectorUtil::ipByteBinByte); + testIpByteBinImpl(defaultedProvider.getVectorUtilSupport()::ipByteBinByte); + testIpByteBinImpl(defOrPanamaProvider.getVectorUtilSupport()::ipByteBinByte); + } + + void testIpByteBinImpl(IpByteBin ipByteBinFunc) { + int iterations = atLeast(50); + for (int i = 0; i < iterations; i++) { + int size = random().nextInt(5000); + var d = new byte[size]; + var q = new byte[size * B_QUERY]; + random().nextBytes(d); + random().nextBytes(q); + assertEquals(scalarIpByteBin(q, d), ipByteBinFunc.apply(q, d)); + + Arrays.fill(d, Byte.MAX_VALUE); + Arrays.fill(q, Byte.MAX_VALUE); + assertEquals(scalarIpByteBin(q, d), ipByteBinFunc.apply(q, d)); + + Arrays.fill(d, Byte.MIN_VALUE); + Arrays.fill(q, Byte.MIN_VALUE); + assertEquals(scalarIpByteBin(q, d), ipByteBinFunc.apply(q, d)); + } + } + + static int scalarIpByteBin(byte[] q, byte[] d) { + int res = 0; + for (int i = 0; i < B_QUERY; i++) { + res += (popcount(q, i * d.length, d, d.length) << i); + } + return res; + } + + public static int popcount(byte[] a, int aOffset, byte[] b, int length) { + int res = 0; + for (int j = 0; j < length; j++) { + int value = (a[aOffset + j] & b[j]) & 0xFF; + for (int k = 0; k < Byte.SIZE; k++) { + if ((value & (1 << k)) != 0) { + ++res; + } + } + } + return res; + } +} diff --git a/libs/simdvec/src/test/java/org/elasticsearch/simdvec/internal/vectorization/BaseVectorizationTests.java b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/internal/vectorization/BaseVectorizationTests.java new file mode 100644 index 0000000000000..f2bc8a11b04aa --- /dev/null +++ b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/internal/vectorization/BaseVectorizationTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +public class BaseVectorizationTests extends ESTestCase { + + @Before + public void sanity() { + assert Runtime.version().feature() < 21 || ModuleLayer.boot().findModule("jdk.incubator.vector").isPresent(); + } + + public static ESVectorizationProvider defaultProvider() { + return new DefaultESVectorizationProvider(); + } + + public static ESVectorizationProvider maybePanamaProvider() { + return ESVectorizationProvider.lookup(true); + } +} diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java index 5455daf0a79ec..227557590731e 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.aggregations.bucket.histogram; -import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.aggregations.bucket.AggregationMultiBucketAggregationTestCase; import org.elasticsearch.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder.RoundingInfo; @@ -28,7 +27,6 @@ import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.test.InternalAggregationTestCase; -import org.elasticsearch.test.TransportVersionUtils; import java.io.IOException; import java.time.Instant; @@ -459,33 +457,6 @@ public void testCreateWithReplacementBuckets() { assertThat(copy.getInterval(), equalTo(orig.getInterval())); } - public void testSerializationPre830() throws IOException { - // we need to test without sub-aggregations, otherwise we need to also update the interval within the inner aggs - InternalAutoDateHistogram instance = createTestInstance( - randomAlphaOfLengthBetween(3, 7), - createTestMetadata(), - InternalAggregations.EMPTY - ); - TransportVersion version = TransportVersionUtils.randomVersionBetween( - random(), - TransportVersions.MINIMUM_COMPATIBLE, - TransportVersionUtils.getPreviousVersion(TransportVersions.V_8_3_0) - ); - InternalAutoDateHistogram deserialized = copyInstance(instance, version); - assertEquals(1, deserialized.getBucketInnerInterval()); - - InternalAutoDateHistogram modified = new InternalAutoDateHistogram( - deserialized.getName(), - deserialized.getBuckets(), - deserialized.getTargetBuckets(), - deserialized.getBucketInfo(), - deserialized.getFormatter(), - deserialized.getMetadata(), - instance.getBucketInnerInterval() - ); - assertEqualInstances(instance, modified); - } - public void testReadFromPre830() throws IOException { byte[] bytes = Base64.getDecoder() .decode( diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml index d9298a832e650..82371c973407c 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml @@ -30,7 +30,7 @@ setup: cluster_features: "gte_v8.15.0" reason: fixed in 8.15.0 - do: - catch: /Cannot format stat \[sum\] with format \[DocValueFormat.DateTime\(format\[date_hour_minute_second_millis\] locale\[\], Z, MILLISECONDS\)\]/ + catch: /Cannot format stat \[sum\] with format \[DocValueFormat.DateTime\(format\[date_hour_minute_second_millis\] locale\[(en)?\], Z, MILLISECONDS\)\]/ search: index: test_date body: diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java index 515d07103bff8..8b29b1609711f 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java @@ -12,25 +12,18 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.logsdb.datageneration.DataGenerator; import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; import org.elasticsearch.logsdb.datageneration.fields.PredefinedField; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.function.Consumer; public class DataGenerationHelper { - private final ObjectMapper.Subobjects subobjects; private final boolean keepArraySource; private final DataGenerator dataGenerator; @@ -40,44 +33,10 @@ public DataGenerationHelper() { } public DataGenerationHelper(Consumer builderConfigurator) { - // TODO enable subobjects: auto - // It is disabled because it currently does not have auto flattening and that results in asserts being triggered when using copy_to. - this.subobjects = ESTestCase.randomValueOtherThan( - ObjectMapper.Subobjects.AUTO, - () -> ESTestCase.randomFrom(ObjectMapper.Subobjects.values()) - ); this.keepArraySource = ESTestCase.randomBoolean(); - var specificationBuilder = DataGeneratorSpecification.builder().withFullyDynamicMapping(ESTestCase.randomBoolean()); - if (subobjects != ObjectMapper.Subobjects.ENABLED) { - specificationBuilder = specificationBuilder.withNestedFieldsLimit(0); - } - - specificationBuilder.withDataSourceHandlers(List.of(new DataSourceHandler() { - @Override - public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) { - if (subobjects == ObjectMapper.Subobjects.ENABLED) { - // Use default behavior - return null; - } - - assert request.isNested() == false; - - // "enabled: false" is not compatible with subobjects: false - // "dynamic: false/strict/runtime" is not compatible with subobjects: false - return new DataSourceResponse.ObjectMappingParametersGenerator(() -> { - var parameters = new HashMap(); - parameters.put("subobjects", subobjects.toString()); - if (ESTestCase.randomBoolean()) { - parameters.put("dynamic", "true"); - } - if (ESTestCase.randomBoolean()) { - parameters.put("enabled", "true"); - } - return parameters; - }); - } - })) + var specificationBuilder = DataGeneratorSpecification.builder() + .withFullyDynamicMapping(ESTestCase.randomBoolean()) .withPredefinedFields( List.of( // Customized because it always needs doc_values for aggregations. @@ -136,11 +95,7 @@ void logsDbMapping(XContentBuilder builder) throws IOException { } void standardMapping(XContentBuilder builder) throws IOException { - if (subobjects != ObjectMapper.Subobjects.ENABLED) { - dataGenerator.writeMapping(builder, Map.of("subobjects", subobjects.toString())); - } else { - dataGenerator.writeMapping(builder); - } + dataGenerator.writeMapping(builder); } void logsDbSettings(Settings.Builder builder) { diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceIT.java index 73d8976c3a4b7..786f091e0c024 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceIT.java @@ -84,7 +84,7 @@ private void assertValidDatabase(DatabaseNodeService databaseNodeService, String IpDatabase database = databaseNodeService.getDatabase(databaseFileName); assertNotNull(database); assertThat(database.getDatabaseType(), equalTo(databaseType)); - CountryResponse countryResponse = database.getCountry("89.160.20.128"); + CountryResponse countryResponse = database.getResponse("89.160.20.128", GeoIpTestUtils::getCountry); assertNotNull(countryResponse); Country country = countryResponse.getCountry(); assertNotNull(country); diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java index 2c7d5fbcc56b7..b28926673069d 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java @@ -205,10 +205,10 @@ private static DatabaseNodeService createRegistry(Path geoIpConfigDir, Path geoI private static void lazyLoadReaders(DatabaseNodeService databaseNodeService) throws IOException { if (databaseNodeService.get("GeoLite2-City.mmdb") != null) { databaseNodeService.get("GeoLite2-City.mmdb").getDatabaseType(); - databaseNodeService.get("GeoLite2-City.mmdb").getCity("2.125.160.216"); + databaseNodeService.get("GeoLite2-City.mmdb").getResponse("2.125.160.216", GeoIpTestUtils::getCity); } databaseNodeService.get("GeoLite2-City-Test.mmdb").getDatabaseType(); - databaseNodeService.get("GeoLite2-City-Test.mmdb").getCity("2.125.160.216"); + databaseNodeService.get("GeoLite2-City-Test.mmdb").getResponse("2.125.160.216", GeoIpTestUtils::getCity); } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java index dccda0d58cfbf..128c16e163764 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java @@ -9,7 +9,6 @@ package org.elasticsearch.ingest.geoip; -import org.elasticsearch.common.Strings; import org.elasticsearch.core.Nullable; import java.util.Arrays; @@ -19,18 +18,21 @@ import java.util.Set; /** - * A high-level representation of a kind of geoip database that is supported by the {@link GeoIpProcessor}. + * A high-level representation of a kind of ip location database that is supported by the {@link GeoIpProcessor}. *

* A database has a set of properties that are valid to use with it (see {@link Database#properties()}), * as well as a list of default properties to use if no properties are specified (see {@link Database#defaultProperties()}). *

- * See especially {@link Database#getDatabase(String, String)} which is used to obtain instances of this class. + * Some database providers have similar concepts but might have slightly different properties associated with those types. + * This can be accommodated, for example, by having a Foo value and a separate FooV2 value where the 'V' should be read as + * 'variant' or 'variation'. A V-less Database type is inherently the first variant/variation (i.e. V1). */ enum Database { City( Set.of( Property.IP, + Property.COUNTRY_IN_EUROPEAN_UNION, Property.COUNTRY_ISO_CODE, Property.CONTINENT_CODE, Property.COUNTRY_NAME, @@ -39,7 +41,9 @@ enum Database { Property.REGION_NAME, Property.CITY_NAME, Property.TIMEZONE, - Property.LOCATION + Property.LOCATION, + Property.POSTAL_CODE, + Property.ACCURACY_RADIUS ), Set.of( Property.COUNTRY_ISO_CODE, @@ -52,7 +56,14 @@ enum Database { ) ), Country( - Set.of(Property.IP, Property.CONTINENT_CODE, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE), + Set.of( + Property.IP, + Property.CONTINENT_CODE, + Property.CONTINENT_NAME, + Property.COUNTRY_NAME, + Property.COUNTRY_IN_EUROPEAN_UNION, + Property.COUNTRY_ISO_CODE + ), Set.of(Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE) ), Asn( @@ -83,12 +94,15 @@ enum Database { Enterprise( Set.of( Property.IP, + Property.COUNTRY_CONFIDENCE, + Property.COUNTRY_IN_EUROPEAN_UNION, Property.COUNTRY_ISO_CODE, Property.COUNTRY_NAME, Property.CONTINENT_CODE, Property.CONTINENT_NAME, Property.REGION_ISO_CODE, Property.REGION_NAME, + Property.CITY_CONFIDENCE, Property.CITY_NAME, Property.TIMEZONE, Property.LOCATION, @@ -107,7 +121,10 @@ enum Database { Property.MOBILE_COUNTRY_CODE, Property.MOBILE_NETWORK_CODE, Property.USER_TYPE, - Property.CONNECTION_TYPE + Property.CONNECTION_TYPE, + Property.POSTAL_CODE, + Property.POSTAL_CONFIDENCE, + Property.ACCURACY_RADIUS ), Set.of( Property.COUNTRY_ISO_CODE, @@ -140,63 +157,20 @@ enum Database { Property.MOBILE_COUNTRY_CODE, Property.MOBILE_NETWORK_CODE ) + ), + AsnV2( + Set.of( + Property.IP, + Property.ASN, + Property.ORGANIZATION_NAME, + Property.NETWORK, + Property.DOMAIN, + Property.COUNTRY_ISO_CODE, + Property.TYPE + ), + Set.of(Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK) ); - private static final String CITY_DB_SUFFIX = "-City"; - private static final String COUNTRY_DB_SUFFIX = "-Country"; - private static final String ASN_DB_SUFFIX = "-ASN"; - private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP"; - private static final String CONNECTION_TYPE_DB_SUFFIX = "-Connection-Type"; - private static final String DOMAIN_DB_SUFFIX = "-Domain"; - private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise"; - private static final String ISP_DB_SUFFIX = "-ISP"; - - @Nullable - private static Database getMaxmindDatabase(final String databaseType) { - if (databaseType.endsWith(Database.CITY_DB_SUFFIX)) { - return Database.City; - } else if (databaseType.endsWith(Database.COUNTRY_DB_SUFFIX)) { - return Database.Country; - } else if (databaseType.endsWith(Database.ASN_DB_SUFFIX)) { - return Database.Asn; - } else if (databaseType.endsWith(Database.ANONYMOUS_IP_DB_SUFFIX)) { - return Database.AnonymousIp; - } else if (databaseType.endsWith(Database.CONNECTION_TYPE_DB_SUFFIX)) { - return Database.ConnectionType; - } else if (databaseType.endsWith(Database.DOMAIN_DB_SUFFIX)) { - return Database.Domain; - } else if (databaseType.endsWith(Database.ENTERPRISE_DB_SUFFIX)) { - return Database.Enterprise; - } else if (databaseType.endsWith(Database.ISP_DB_SUFFIX)) { - return Database.Isp; - } else { - return null; // no match was found - } - } - - /** - * Parses the passed-in databaseType (presumably from the passed-in databaseFile) and return the Database instance that is - * associated with that databaseType. - * - * @param databaseType the database type String from the metadata of the database file - * @param databaseFile the database file from which the database type was obtained - * @throws IllegalArgumentException if the databaseType is not associated with a Database instance - * @return the Database instance that is associated with the databaseType - */ - public static Database getDatabase(final String databaseType, final String databaseFile) { - Database database = null; - - if (Strings.hasText(databaseType)) { - database = getMaxmindDatabase(databaseType); - } - - if (database == null) { - throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]"); - } - - return database; - } - private final Set properties; private final Set defaultProperties; @@ -245,12 +219,15 @@ public Set parseProperties(@Nullable final List propertyNames) enum Property { IP, + COUNTRY_CONFIDENCE, + COUNTRY_IN_EUROPEAN_UNION, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, + CITY_CONFIDENCE, CITY_NAME, TIMEZONE, LOCATION, @@ -269,7 +246,11 @@ enum Property { MOBILE_COUNTRY_CODE, MOBILE_NETWORK_CODE, CONNECTION_TYPE, - USER_TYPE; + USER_TYPE, + TYPE, + POSTAL_CODE, + POSTAL_CONFIDENCE, + ACCURACY_RADIUS; /** * Parses a string representation of a property into an actual Property instance. Not all properties that exist are diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java index e160c8ad1543f..120afe0e9e815 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java @@ -9,18 +9,8 @@ package org.elasticsearch.ingest.geoip; -import com.maxmind.db.DatabaseRecord; -import com.maxmind.db.Network; import com.maxmind.db.NoCache; import com.maxmind.db.Reader; -import com.maxmind.geoip2.model.AnonymousIpResponse; -import com.maxmind.geoip2.model.AsnResponse; -import com.maxmind.geoip2.model.CityResponse; -import com.maxmind.geoip2.model.ConnectionTypeResponse; -import com.maxmind.geoip2.model.CountryResponse; -import com.maxmind.geoip2.model.DomainResponse; -import com.maxmind.geoip2.model.EnterpriseResponse; -import com.maxmind.geoip2.model.IspResponse; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -28,8 +18,6 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.common.CheckedSupplier; -import org.elasticsearch.common.network.InetAddresses; -import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.core.Booleans; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Nullable; @@ -37,19 +25,16 @@ import java.io.File; import java.io.IOException; -import java.net.InetAddress; import java.nio.file.Files; import java.nio.file.Path; -import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; /** * Facilitates lazy loading of the database reader, so that when the geoip plugin is installed, but not used, * no memory is being wasted on the database reader. */ -class DatabaseReaderLazyLoader implements IpDatabase { +public class DatabaseReaderLazyLoader implements IpDatabase { private static final boolean LOAD_DATABASE_ON_HEAP = Booleans.parseBoolean(System.getProperty("es.geoip.load_db_on_heap", "false")); @@ -96,94 +81,6 @@ public final String getDatabaseType() throws IOException { return databaseType.get(); } - @Nullable - @Override - public CityResponse getCity(String ipAddress) { - return getResponse(ipAddress, (reader, ip) -> lookup(reader, ip, CityResponse.class, CityResponse::new)); - } - - @Nullable - @Override - public CountryResponse getCountry(String ipAddress) { - return getResponse(ipAddress, (reader, ip) -> lookup(reader, ip, CountryResponse.class, CountryResponse::new)); - } - - @Nullable - @Override - public AsnResponse getAsn(String ipAddress) { - return getResponse( - ipAddress, - (reader, ip) -> lookup( - reader, - ip, - AsnResponse.class, - (response, responseIp, network, locales) -> new AsnResponse(response, responseIp, network) - ) - ); - } - - @Nullable - @Override - public AnonymousIpResponse getAnonymousIp(String ipAddress) { - return getResponse( - ipAddress, - (reader, ip) -> lookup( - reader, - ip, - AnonymousIpResponse.class, - (response, responseIp, network, locales) -> new AnonymousIpResponse(response, responseIp, network) - ) - ); - } - - @Nullable - @Override - public ConnectionTypeResponse getConnectionType(String ipAddress) { - return getResponse( - ipAddress, - (reader, ip) -> lookup( - reader, - ip, - ConnectionTypeResponse.class, - (response, responseIp, network, locales) -> new ConnectionTypeResponse(response, responseIp, network) - ) - ); - } - - @Nullable - @Override - public DomainResponse getDomain(String ipAddress) { - return getResponse( - ipAddress, - (reader, ip) -> lookup( - reader, - ip, - DomainResponse.class, - (response, responseIp, network, locales) -> new DomainResponse(response, responseIp, network) - ) - ); - } - - @Nullable - @Override - public EnterpriseResponse getEnterprise(String ipAddress) { - return getResponse(ipAddress, (reader, ip) -> lookup(reader, ip, EnterpriseResponse.class, EnterpriseResponse::new)); - } - - @Nullable - @Override - public IspResponse getIsp(String ipAddress) { - return getResponse( - ipAddress, - (reader, ip) -> lookup( - reader, - ip, - IspResponse.class, - (response, responseIp, network, locales) -> new IspResponse(response, responseIp, network) - ) - ); - } - boolean preLookup() { return currentUsages.updateAndGet(current -> current < 0 ? current : current + 1) > 0; } @@ -199,14 +96,12 @@ int current() { return currentUsages.get(); } + @Override @Nullable - private RESPONSE getResponse( - String ipAddress, - CheckedBiFunction, Exception> responseProvider - ) { + public RESPONSE getResponse(String ipAddress, CheckedBiFunction responseProvider) { return cache.putIfAbsent(ipAddress, databasePath.toString(), ip -> { try { - return responseProvider.apply(get(), ipAddress).orElse(null); + return responseProvider.apply(get(), ipAddress); } catch (Exception e) { throw ExceptionsHelper.convertToRuntime(e); } @@ -263,23 +158,6 @@ private static File pathToFile(Path databasePath) { return databasePath.toFile(); } - @FunctionalInterface - private interface ResponseBuilder { - RESPONSE build(RESPONSE response, String responseIp, Network network, List locales); - } - - private Optional lookup(Reader reader, String ip, Class clazz, ResponseBuilder builder) - throws IOException { - InetAddress inetAddress = InetAddresses.forString(ip); - DatabaseRecord record = reader.getRecord(inetAddress, clazz); - RESPONSE result = record.getData(); - if (result == null) { - return Optional.empty(); - } else { - return Optional.of(builder.build(result, NetworkAddress.format(inetAddress), record.getNetwork(), List.of("en"))); - } - } - long getBuildDateMillis() throws IOException { if (buildDate.get() == null) { synchronized (buildDate) { diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java index 335331ac0ab9d..d9c9c3aaf3266 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java @@ -26,7 +26,7 @@ * cost of deserialization for each lookup (cached or not). This comes at slight expense of higher memory usage, but significant * reduction of CPU usage. */ -final class GeoIpCache { +public final class GeoIpCache { /** * Internal-only sentinel object for recording that a result from the geoip database was null (i.e. there was no result). By caching diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index ce160b060ae4c..e2b516bf5b943 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -9,23 +9,6 @@ package org.elasticsearch.ingest.geoip; -import com.maxmind.db.Network; -import com.maxmind.geoip2.model.AnonymousIpResponse; -import com.maxmind.geoip2.model.AsnResponse; -import com.maxmind.geoip2.model.CityResponse; -import com.maxmind.geoip2.model.ConnectionTypeResponse; -import com.maxmind.geoip2.model.ConnectionTypeResponse.ConnectionType; -import com.maxmind.geoip2.model.CountryResponse; -import com.maxmind.geoip2.model.DomainResponse; -import com.maxmind.geoip2.model.EnterpriseResponse; -import com.maxmind.geoip2.model.IspResponse; -import com.maxmind.geoip2.record.City; -import com.maxmind.geoip2.record.Continent; -import com.maxmind.geoip2.record.Country; -import com.maxmind.geoip2.record.Location; -import com.maxmind.geoip2.record.Subdivision; - -import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; @@ -34,10 +17,10 @@ import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; import org.elasticsearch.ingest.geoip.Database.Property; +import org.elasticsearch.ingest.geoip.IpDataLookupFactories.IpDataLookupFactory; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -60,7 +43,7 @@ public final class GeoIpProcessor extends AbstractProcessor { private final Supplier isValid; private final String targetField; private final CheckedSupplier supplier; - private final Set properties; + private final IpDataLookup ipDataLookup; private final boolean ignoreMissing; private final boolean firstOnly; private final String databaseFile; @@ -73,7 +56,7 @@ public final class GeoIpProcessor extends AbstractProcessor { * @param supplier a supplier of a geo-IP database reader; ideally this is lazily-loaded once on first use * @param isValid a supplier that determines if the available database files are up-to-date and license compliant * @param targetField the target field - * @param properties the properties; ideally this is lazily-loaded once on first use + * @param ipDataLookup a lookup capable of retrieving a result from an available geo-IP database reader * @param ignoreMissing true if documents with a missing value for the field should be ignored * @param firstOnly true if only first result should be returned in case of array * @param databaseFile the name of the database file being queried; used only for tagging documents if the database is unavailable @@ -85,7 +68,7 @@ public final class GeoIpProcessor extends AbstractProcessor { final CheckedSupplier supplier, final Supplier isValid, final String targetField, - final Set properties, + final IpDataLookup ipDataLookup, final boolean ignoreMissing, final boolean firstOnly, final String databaseFile @@ -95,7 +78,7 @@ public final class GeoIpProcessor extends AbstractProcessor { this.isValid = isValid; this.targetField = targetField; this.supplier = supplier; - this.properties = properties; + this.ipDataLookup = ipDataLookup; this.ignoreMissing = ignoreMissing; this.firstOnly = firstOnly; this.databaseFile = databaseFile; @@ -127,7 +110,7 @@ public IngestDocument execute(IngestDocument document) throws IOException { } if (ip instanceof String ipString) { - Map data = getGeoData(ipDatabase, ipString); + Map data = ipDataLookup.getData(ipDatabase, ipString); if (data.isEmpty() == false) { document.setFieldValue(targetField, data); } @@ -138,7 +121,7 @@ public IngestDocument execute(IngestDocument document) throws IOException { if (ipAddr instanceof String == false) { throw new IllegalArgumentException("array in field [" + field + "] should only contain strings"); } - Map data = getGeoData(ipDatabase, (String) ipAddr); + Map data = ipDataLookup.getData(ipDatabase, (String) ipAddr); if (data.isEmpty()) { dataList.add(null); continue; @@ -161,26 +144,6 @@ public IngestDocument execute(IngestDocument document) throws IOException { return document; } - private Map getGeoData(IpDatabase ipDatabase, String ipAddress) throws IOException { - final String databaseType = ipDatabase.getDatabaseType(); - final Database database; - try { - database = Database.getDatabase(databaseType, databaseFile); - } catch (IllegalArgumentException e) { - throw new ElasticsearchParseException(e.getMessage(), e); - } - return switch (database) { - case City -> retrieveCityGeoData(ipDatabase, ipAddress); - case Country -> retrieveCountryGeoData(ipDatabase, ipAddress); - case Asn -> retrieveAsnGeoData(ipDatabase, ipAddress); - case AnonymousIp -> retrieveAnonymousIpGeoData(ipDatabase, ipAddress); - case ConnectionType -> retrieveConnectionTypeGeoData(ipDatabase, ipAddress); - case Domain -> retrieveDomainGeoData(ipDatabase, ipAddress); - case Enterprise -> retrieveEnterpriseGeoData(ipDatabase, ipAddress); - case Isp -> retrieveIspGeoData(ipDatabase, ipAddress); - }; - } - @Override public String getType() { return TYPE; @@ -199,478 +162,7 @@ String getDatabaseType() throws IOException { } Set getProperties() { - return properties; - } - - private Map retrieveCityGeoData(IpDatabase ipDatabase, String ipAddress) { - CityResponse response = ipDatabase.getCity(ipAddress); - if (response == null) { - return Map.of(); - } - Country country = response.getCountry(); - City city = response.getCity(); - Location location = response.getLocation(); - Continent continent = response.getContinent(); - Subdivision subdivision = response.getMostSpecificSubdivision(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getTraits().getIpAddress()); - case COUNTRY_ISO_CODE -> { - String countryIsoCode = country.getIsoCode(); - if (countryIsoCode != null) { - geoData.put("country_iso_code", countryIsoCode); - } - } - case COUNTRY_NAME -> { - String countryName = country.getName(); - if (countryName != null) { - geoData.put("country_name", countryName); - } - } - case CONTINENT_CODE -> { - String continentCode = continent.getCode(); - if (continentCode != null) { - geoData.put("continent_code", continentCode); - } - } - case CONTINENT_NAME -> { - String continentName = continent.getName(); - if (continentName != null) { - geoData.put("continent_name", continentName); - } - } - case REGION_ISO_CODE -> { - // ISO 3166-2 code for country subdivisions. - // See iso.org/iso-3166-country-codes.html - String countryIso = country.getIsoCode(); - String subdivisionIso = subdivision.getIsoCode(); - if (countryIso != null && subdivisionIso != null) { - String regionIsoCode = countryIso + "-" + subdivisionIso; - geoData.put("region_iso_code", regionIsoCode); - } - } - case REGION_NAME -> { - String subdivisionName = subdivision.getName(); - if (subdivisionName != null) { - geoData.put("region_name", subdivisionName); - } - } - case CITY_NAME -> { - String cityName = city.getName(); - if (cityName != null) { - geoData.put("city_name", cityName); - } - } - case TIMEZONE -> { - String locationTimeZone = location.getTimeZone(); - if (locationTimeZone != null) { - geoData.put("timezone", locationTimeZone); - } - } - case LOCATION -> { - Double latitude = location.getLatitude(); - Double longitude = location.getLongitude(); - if (latitude != null && longitude != null) { - Map locationObject = new HashMap<>(); - locationObject.put("lat", latitude); - locationObject.put("lon", longitude); - geoData.put("location", locationObject); - } - } - } - } - return geoData; - } - - private Map retrieveCountryGeoData(IpDatabase ipDatabase, String ipAddress) { - CountryResponse response = ipDatabase.getCountry(ipAddress); - if (response == null) { - return Map.of(); - } - Country country = response.getCountry(); - Continent continent = response.getContinent(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getTraits().getIpAddress()); - case COUNTRY_ISO_CODE -> { - String countryIsoCode = country.getIsoCode(); - if (countryIsoCode != null) { - geoData.put("country_iso_code", countryIsoCode); - } - } - case COUNTRY_NAME -> { - String countryName = country.getName(); - if (countryName != null) { - geoData.put("country_name", countryName); - } - } - case CONTINENT_CODE -> { - String continentCode = continent.getCode(); - if (continentCode != null) { - geoData.put("continent_code", continentCode); - } - } - case CONTINENT_NAME -> { - String continentName = continent.getName(); - if (continentName != null) { - geoData.put("continent_name", continentName); - } - } - } - } - return geoData; - } - - private Map retrieveAsnGeoData(IpDatabase ipDatabase, String ipAddress) { - AsnResponse response = ipDatabase.getAsn(ipAddress); - if (response == null) { - return Map.of(); - } - Long asn = response.getAutonomousSystemNumber(); - String organizationName = response.getAutonomousSystemOrganization(); - Network network = response.getNetwork(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getIpAddress()); - case ASN -> { - if (asn != null) { - geoData.put("asn", asn); - } - } - case ORGANIZATION_NAME -> { - if (organizationName != null) { - geoData.put("organization_name", organizationName); - } - } - case NETWORK -> { - if (network != null) { - geoData.put("network", network.toString()); - } - } - } - } - return geoData; - } - - private Map retrieveAnonymousIpGeoData(IpDatabase ipDatabase, String ipAddress) { - AnonymousIpResponse response = ipDatabase.getAnonymousIp(ipAddress); - if (response == null) { - return Map.of(); - } - - boolean isHostingProvider = response.isHostingProvider(); - boolean isTorExitNode = response.isTorExitNode(); - boolean isAnonymousVpn = response.isAnonymousVpn(); - boolean isAnonymous = response.isAnonymous(); - boolean isPublicProxy = response.isPublicProxy(); - boolean isResidentialProxy = response.isResidentialProxy(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getIpAddress()); - case HOSTING_PROVIDER -> { - geoData.put("hosting_provider", isHostingProvider); - } - case TOR_EXIT_NODE -> { - geoData.put("tor_exit_node", isTorExitNode); - } - case ANONYMOUS_VPN -> { - geoData.put("anonymous_vpn", isAnonymousVpn); - } - case ANONYMOUS -> { - geoData.put("anonymous", isAnonymous); - } - case PUBLIC_PROXY -> { - geoData.put("public_proxy", isPublicProxy); - } - case RESIDENTIAL_PROXY -> { - geoData.put("residential_proxy", isResidentialProxy); - } - } - } - return geoData; - } - - private Map retrieveConnectionTypeGeoData(IpDatabase ipDatabase, String ipAddress) { - ConnectionTypeResponse response = ipDatabase.getConnectionType(ipAddress); - if (response == null) { - return Map.of(); - } - - ConnectionType connectionType = response.getConnectionType(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getIpAddress()); - case CONNECTION_TYPE -> { - if (connectionType != null) { - geoData.put("connection_type", connectionType.toString()); - } - } - } - } - return geoData; - } - - private Map retrieveDomainGeoData(IpDatabase ipDatabase, String ipAddress) { - DomainResponse response = ipDatabase.getDomain(ipAddress); - if (response == null) { - return Map.of(); - } - - String domain = response.getDomain(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getIpAddress()); - case DOMAIN -> { - if (domain != null) { - geoData.put("domain", domain); - } - } - } - } - return geoData; - } - - private Map retrieveEnterpriseGeoData(IpDatabase ipDatabase, String ipAddress) { - EnterpriseResponse response = ipDatabase.getEnterprise(ipAddress); - if (response == null) { - return Map.of(); - } - - Country country = response.getCountry(); - City city = response.getCity(); - Location location = response.getLocation(); - Continent continent = response.getContinent(); - Subdivision subdivision = response.getMostSpecificSubdivision(); - - Long asn = response.getTraits().getAutonomousSystemNumber(); - String organizationName = response.getTraits().getAutonomousSystemOrganization(); - Network network = response.getTraits().getNetwork(); - - String isp = response.getTraits().getIsp(); - String ispOrganization = response.getTraits().getOrganization(); - String mobileCountryCode = response.getTraits().getMobileCountryCode(); - String mobileNetworkCode = response.getTraits().getMobileNetworkCode(); - - boolean isHostingProvider = response.getTraits().isHostingProvider(); - boolean isTorExitNode = response.getTraits().isTorExitNode(); - boolean isAnonymousVpn = response.getTraits().isAnonymousVpn(); - boolean isAnonymous = response.getTraits().isAnonymous(); - boolean isPublicProxy = response.getTraits().isPublicProxy(); - boolean isResidentialProxy = response.getTraits().isResidentialProxy(); - - String userType = response.getTraits().getUserType(); - - String domain = response.getTraits().getDomain(); - - ConnectionType connectionType = response.getTraits().getConnectionType(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getTraits().getIpAddress()); - case COUNTRY_ISO_CODE -> { - String countryIsoCode = country.getIsoCode(); - if (countryIsoCode != null) { - geoData.put("country_iso_code", countryIsoCode); - } - } - case COUNTRY_NAME -> { - String countryName = country.getName(); - if (countryName != null) { - geoData.put("country_name", countryName); - } - } - case CONTINENT_CODE -> { - String continentCode = continent.getCode(); - if (continentCode != null) { - geoData.put("continent_code", continentCode); - } - } - case CONTINENT_NAME -> { - String continentName = continent.getName(); - if (continentName != null) { - geoData.put("continent_name", continentName); - } - } - case REGION_ISO_CODE -> { - // ISO 3166-2 code for country subdivisions. - // See iso.org/iso-3166-country-codes.html - String countryIso = country.getIsoCode(); - String subdivisionIso = subdivision.getIsoCode(); - if (countryIso != null && subdivisionIso != null) { - String regionIsoCode = countryIso + "-" + subdivisionIso; - geoData.put("region_iso_code", regionIsoCode); - } - } - case REGION_NAME -> { - String subdivisionName = subdivision.getName(); - if (subdivisionName != null) { - geoData.put("region_name", subdivisionName); - } - } - case CITY_NAME -> { - String cityName = city.getName(); - if (cityName != null) { - geoData.put("city_name", cityName); - } - } - case TIMEZONE -> { - String locationTimeZone = location.getTimeZone(); - if (locationTimeZone != null) { - geoData.put("timezone", locationTimeZone); - } - } - case LOCATION -> { - Double latitude = location.getLatitude(); - Double longitude = location.getLongitude(); - if (latitude != null && longitude != null) { - Map locationObject = new HashMap<>(); - locationObject.put("lat", latitude); - locationObject.put("lon", longitude); - geoData.put("location", locationObject); - } - } - case ASN -> { - if (asn != null) { - geoData.put("asn", asn); - } - } - case ORGANIZATION_NAME -> { - if (organizationName != null) { - geoData.put("organization_name", organizationName); - } - } - case NETWORK -> { - if (network != null) { - geoData.put("network", network.toString()); - } - } - case HOSTING_PROVIDER -> { - geoData.put("hosting_provider", isHostingProvider); - } - case TOR_EXIT_NODE -> { - geoData.put("tor_exit_node", isTorExitNode); - } - case ANONYMOUS_VPN -> { - geoData.put("anonymous_vpn", isAnonymousVpn); - } - case ANONYMOUS -> { - geoData.put("anonymous", isAnonymous); - } - case PUBLIC_PROXY -> { - geoData.put("public_proxy", isPublicProxy); - } - case RESIDENTIAL_PROXY -> { - geoData.put("residential_proxy", isResidentialProxy); - } - case DOMAIN -> { - if (domain != null) { - geoData.put("domain", domain); - } - } - case ISP -> { - if (isp != null) { - geoData.put("isp", isp); - } - } - case ISP_ORGANIZATION_NAME -> { - if (ispOrganization != null) { - geoData.put("isp_organization_name", ispOrganization); - } - } - case MOBILE_COUNTRY_CODE -> { - if (mobileCountryCode != null) { - geoData.put("mobile_country_code", mobileCountryCode); - } - } - case MOBILE_NETWORK_CODE -> { - if (mobileNetworkCode != null) { - geoData.put("mobile_network_code", mobileNetworkCode); - } - } - case USER_TYPE -> { - if (userType != null) { - geoData.put("user_type", userType); - } - } - case CONNECTION_TYPE -> { - if (connectionType != null) { - geoData.put("connection_type", connectionType.toString()); - } - } - } - } - return geoData; - } - - private Map retrieveIspGeoData(IpDatabase ipDatabase, String ipAddress) { - IspResponse response = ipDatabase.getIsp(ipAddress); - if (response == null) { - return Map.of(); - } - - String isp = response.getIsp(); - String ispOrganization = response.getOrganization(); - String mobileNetworkCode = response.getMobileNetworkCode(); - String mobileCountryCode = response.getMobileCountryCode(); - Long asn = response.getAutonomousSystemNumber(); - String organizationName = response.getAutonomousSystemOrganization(); - Network network = response.getNetwork(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getIpAddress()); - case ASN -> { - if (asn != null) { - geoData.put("asn", asn); - } - } - case ORGANIZATION_NAME -> { - if (organizationName != null) { - geoData.put("organization_name", organizationName); - } - } - case NETWORK -> { - if (network != null) { - geoData.put("network", network.toString()); - } - } - case ISP -> { - if (isp != null) { - geoData.put("isp", isp); - } - } - case ISP_ORGANIZATION_NAME -> { - if (ispOrganization != null) { - geoData.put("isp_organization_name", ispOrganization); - } - } - case MOBILE_COUNTRY_CODE -> { - if (mobileCountryCode != null) { - geoData.put("mobile_country_code", mobileCountryCode); - } - } - case MOBILE_NETWORK_CODE -> { - if (mobileNetworkCode != null) { - geoData.put("mobile_network_code", mobileNetworkCode); - } - } - } - } - return geoData; + return ipDataLookup.getProperties(); } /** @@ -752,19 +244,20 @@ public Processor create( databaseType = ipDatabase.getDatabaseType(); } - final Database database; + final IpDataLookupFactory factory; try { - database = Database.getDatabase(databaseType, databaseFile); + factory = IpDataLookupFactories.get(databaseType, databaseFile); } catch (IllegalArgumentException e) { throw newConfigurationException(TYPE, processorTag, "database_file", e.getMessage()); } - final Set properties; + final IpDataLookup ipDataLookup; try { - properties = database.parseProperties(propertyNames); + ipDataLookup = factory.create(propertyNames); } catch (IllegalArgumentException e) { throw newConfigurationException(TYPE, processorTag, "properties", e.getMessage()); } + return new GeoIpProcessor( processorTag, description, @@ -772,7 +265,7 @@ public Processor create( new DatabaseVerifyingSupplier(ipDatabaseProvider, databaseFile, databaseType), () -> ipDatabaseProvider.isValid(databaseFile), targetField, - properties, + ipDataLookup, ignoreMissing, firstOnly, databaseFile diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookup.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookup.java new file mode 100644 index 0000000000000..7442c8e930886 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookup.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +interface IpDataLookup { + /** + * Gets data from the provided {@code ipDatabase} for the provided {@code ip} + * + * @param ipDatabase the database from which to lookup a result + * @param ip the ip address + * @return a map of data corresponding to the configured properties + * @throws IOException if the implementation encounters any problem while retrieving the response + */ + Map getData(IpDatabase ipDatabase, String ip) throws IOException; + + /** + * @return the set of properties this lookup will provide + */ + Set getProperties(); +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java new file mode 100644 index 0000000000000..3379fdff0633a --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.core.Nullable; + +import java.util.List; +import java.util.Set; +import java.util.function.Function; + +final class IpDataLookupFactories { + + private IpDataLookupFactories() { + // utility class + } + + interface IpDataLookupFactory { + IpDataLookup create(List properties); + } + + private static final String CITY_DB_SUFFIX = "-City"; + private static final String COUNTRY_DB_SUFFIX = "-Country"; + private static final String ASN_DB_SUFFIX = "-ASN"; + private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP"; + private static final String CONNECTION_TYPE_DB_SUFFIX = "-Connection-Type"; + private static final String DOMAIN_DB_SUFFIX = "-Domain"; + private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise"; + private static final String ISP_DB_SUFFIX = "-ISP"; + + @Nullable + private static Database getMaxmindDatabase(final String databaseType) { + if (databaseType.endsWith(CITY_DB_SUFFIX)) { + return Database.City; + } else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) { + return Database.Country; + } else if (databaseType.endsWith(ASN_DB_SUFFIX)) { + return Database.Asn; + } else if (databaseType.endsWith(ANONYMOUS_IP_DB_SUFFIX)) { + return Database.AnonymousIp; + } else if (databaseType.endsWith(CONNECTION_TYPE_DB_SUFFIX)) { + return Database.ConnectionType; + } else if (databaseType.endsWith(DOMAIN_DB_SUFFIX)) { + return Database.Domain; + } else if (databaseType.endsWith(ENTERPRISE_DB_SUFFIX)) { + return Database.Enterprise; + } else if (databaseType.endsWith(ISP_DB_SUFFIX)) { + return Database.Isp; + } else { + return null; // no match was found + } + } + + /** + * Parses the passed-in databaseType and return the Database instance that is + * associated with that databaseType. + * + * @param databaseType the database type String from the metadata of the database file + * @return the Database instance that is associated with the databaseType + */ + @Nullable + static Database getDatabase(final String databaseType) { + Database database = null; + + if (Strings.hasText(databaseType)) { + database = getMaxmindDatabase(databaseType); + } + + return database; + } + + @Nullable + static Function, IpDataLookup> getMaxmindLookup(final Database database) { + return switch (database) { + case City -> MaxmindIpDataLookups.City::new; + case Country -> MaxmindIpDataLookups.Country::new; + case Asn -> MaxmindIpDataLookups.Asn::new; + case AnonymousIp -> MaxmindIpDataLookups.AnonymousIp::new; + case ConnectionType -> MaxmindIpDataLookups.ConnectionType::new; + case Domain -> MaxmindIpDataLookups.Domain::new; + case Enterprise -> MaxmindIpDataLookups.Enterprise::new; + case Isp -> MaxmindIpDataLookups.Isp::new; + default -> null; + }; + } + + static IpDataLookupFactory get(final String databaseType, final String databaseFile) { + final Database database = getDatabase(databaseType); + if (database == null) { + throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]"); + } + + final Function, IpDataLookup> factoryMethod = getMaxmindLookup(database); + + if (factoryMethod == null) { + throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]"); + } + + return (properties) -> factoryMethod.apply(database.parseProperties(properties)); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDatabase.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDatabase.java index f416259a87d27..db1ffc1c682b8 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDatabase.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDatabase.java @@ -9,15 +9,9 @@ package org.elasticsearch.ingest.geoip; -import com.maxmind.geoip2.model.AnonymousIpResponse; -import com.maxmind.geoip2.model.AsnResponse; -import com.maxmind.geoip2.model.CityResponse; -import com.maxmind.geoip2.model.ConnectionTypeResponse; -import com.maxmind.geoip2.model.CountryResponse; -import com.maxmind.geoip2.model.DomainResponse; -import com.maxmind.geoip2.model.EnterpriseResponse; -import com.maxmind.geoip2.model.IspResponse; +import com.maxmind.db.Reader; +import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.core.Nullable; import java.io.IOException; @@ -34,44 +28,15 @@ public interface IpDatabase extends AutoCloseable { String getDatabaseType() throws IOException; /** - * @param ipAddress the IP address to look up - * @return a response containing the city data for the given address if it exists, or null if it could not be found - * @throws UnsupportedOperationException may be thrown if the implementation does not support retrieving city data - */ - @Nullable - CityResponse getCity(String ipAddress); - - /** - * @param ipAddress the IP address to look up - * @return a response containing the country data for the given address if it exists, or null if it could not be found - * @throws UnsupportedOperationException may be thrown if the implementation does not support retrieving country data - */ - @Nullable - CountryResponse getCountry(String ipAddress); - - /** - * @param ipAddress the IP address to look up - * @return a response containing the Autonomous System Number for the given address if it exists, or null if it could not - * be found - * @throws UnsupportedOperationException may be thrown if the implementation does not support retrieving ASN data + * Returns a response from this database's reader for the given IP address. + * + * @param ipAddress the address to lookup + * @param responseProvider a method for extracting a response from a {@link Reader}, usually this will be a method reference + * @return a possibly-null response + * @param the type of response that will be returned */ @Nullable - AsnResponse getAsn(String ipAddress); - - @Nullable - AnonymousIpResponse getAnonymousIp(String ipAddress); - - @Nullable - ConnectionTypeResponse getConnectionType(String ipAddress); - - @Nullable - DomainResponse getDomain(String ipAddress); - - @Nullable - EnterpriseResponse getEnterprise(String ipAddress); - - @Nullable - IspResponse getIsp(String ipAddress); + RESPONSE getResponse(String ipAddress, CheckedBiFunction responseProvider); /** * Releases the current database object. Called after processing a single document. Databases should be closed or returned to a diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java new file mode 100644 index 0000000000000..ac7f56468f37e --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java @@ -0,0 +1,235 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import com.maxmind.db.DatabaseRecord; +import com.maxmind.db.MaxMindDbConstructor; +import com.maxmind.db.MaxMindDbParameter; +import com.maxmind.db.Reader; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.core.Nullable; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * A collection of {@link IpDataLookup} implementations for IPinfo databases + */ +final class IpinfoIpDataLookups { + + private IpinfoIpDataLookups() { + // utility class + } + + private static final Logger logger = LogManager.getLogger(IpinfoIpDataLookups.class); + + /** + * Lax-ly parses a string that (ideally) looks like 'AS123' into a Long like 123L (or null, if such parsing isn't possible). + * @param asn a potentially empty (or null) ASN string that is expected to contain 'AS' and then a parsable long + * @return the parsed asn + */ + static Long parseAsn(final String asn) { + if (asn == null || Strings.hasText(asn) == false) { + return null; + } else { + String stripped = asn.toUpperCase(Locale.ROOT).replaceAll("AS", "").trim(); + try { + return Long.parseLong(stripped); + } catch (NumberFormatException e) { + logger.trace("Unable to parse non-compliant ASN string [{}]", asn); + return null; + } + } + } + + public record AsnResult( + Long asn, + @Nullable String country, // not present in the free asn database + String domain, + String name, + @Nullable String type // not present in the free asn database + ) { + @SuppressWarnings("checkstyle:RedundantModifier") + @MaxMindDbConstructor + public AsnResult( + @MaxMindDbParameter(name = "asn") String asn, + @Nullable @MaxMindDbParameter(name = "country") String country, + @MaxMindDbParameter(name = "domain") String domain, + @MaxMindDbParameter(name = "name") String name, + @Nullable @MaxMindDbParameter(name = "type") String type + ) { + this(parseAsn(asn), country, domain, name, type); + } + } + + public record CountryResult( + @MaxMindDbParameter(name = "continent") String continent, + @MaxMindDbParameter(name = "continent_name") String continentName, + @MaxMindDbParameter(name = "country") String country, + @MaxMindDbParameter(name = "country_name") String countryName + ) { + @MaxMindDbConstructor + public CountryResult {} + } + + static class Asn extends AbstractBase { + Asn(Set properties) { + super(properties, AsnResult.class); + } + + @Override + protected Map transform(final Result result) { + AsnResult response = result.result; + Long asn = response.asn; + String organizationName = response.name; + String network = result.network; + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", result.ip); + case ASN -> { + if (asn != null) { + data.put("asn", asn); + } + } + case ORGANIZATION_NAME -> { + if (organizationName != null) { + data.put("organization_name", organizationName); + } + } + case NETWORK -> { + if (network != null) { + data.put("network", network); + } + } + case COUNTRY_ISO_CODE -> { + if (response.country != null) { + data.put("country_iso_code", response.country); + } + } + case DOMAIN -> { + if (response.domain != null) { + data.put("domain", response.domain); + } + } + case TYPE -> { + if (response.type != null) { + data.put("type", response.type); + } + } + } + } + return data; + } + } + + static class Country extends AbstractBase { + Country(Set properties) { + super(properties, CountryResult.class); + } + + @Override + protected Map transform(final Result result) { + CountryResult response = result.result; + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", result.ip); + case COUNTRY_ISO_CODE -> { + String countryIsoCode = response.country; + if (countryIsoCode != null) { + data.put("country_iso_code", countryIsoCode); + } + } + case COUNTRY_NAME -> { + String countryName = response.countryName; + if (countryName != null) { + data.put("country_name", countryName); + } + } + case CONTINENT_CODE -> { + String continentCode = response.continent; + if (continentCode != null) { + data.put("continent_code", continentCode); + } + } + case CONTINENT_NAME -> { + String continentName = response.continentName; + if (continentName != null) { + data.put("continent_name", continentName); + } + } + } + } + return data; + } + } + + /** + * Just a little record holder -- there's the data that we receive via the binding to our record objects from the Reader via the + * getRecord call, but then we also need to capture the passed-in ip address that came from the caller as well as the network for + * the returned DatabaseRecord from the Reader. + */ + public record Result(T result, String ip, String network) {} + + /** + * The {@link IpinfoIpDataLookups.AbstractBase} is an abstract base implementation of {@link IpDataLookup} that + * provides common functionality for getting a {@link IpinfoIpDataLookups.Result} that wraps a record from a {@link IpDatabase}. + * + * @param the record type that will be wrapped and returned + */ + private abstract static class AbstractBase implements IpDataLookup { + + protected final Set properties; + protected final Class clazz; + + AbstractBase(final Set properties, final Class clazz) { + this.properties = Set.copyOf(properties); + this.clazz = clazz; + } + + @Override + public Set getProperties() { + return this.properties; + } + + @Override + public final Map getData(final IpDatabase ipDatabase, final String ipAddress) { + final Result response = ipDatabase.getResponse(ipAddress, this::lookup); + return (response == null || response.result == null) ? Map.of() : transform(response); + } + + @Nullable + private Result lookup(final Reader reader, final String ipAddress) throws IOException { + final InetAddress ip = InetAddresses.forString(ipAddress); + final DatabaseRecord record = reader.getRecord(ip, clazz); + final RESPONSE data = record.getData(); + return (data == null) ? null : new Result<>(data, NetworkAddress.format(ip), record.getNetwork().toString()); + } + + /** + * Extract the configured properties from the retrieved response + * @param response the non-null response that was retrieved + * @return a mapping of properties for the ip from the response + */ + protected abstract Map transform(Result response); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java new file mode 100644 index 0000000000000..e7c3481938033 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java @@ -0,0 +1,667 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import com.maxmind.db.DatabaseRecord; +import com.maxmind.db.Network; +import com.maxmind.db.Reader; +import com.maxmind.geoip2.model.AbstractResponse; +import com.maxmind.geoip2.model.AnonymousIpResponse; +import com.maxmind.geoip2.model.AsnResponse; +import com.maxmind.geoip2.model.CityResponse; +import com.maxmind.geoip2.model.ConnectionTypeResponse; +import com.maxmind.geoip2.model.CountryResponse; +import com.maxmind.geoip2.model.DomainResponse; +import com.maxmind.geoip2.model.EnterpriseResponse; +import com.maxmind.geoip2.model.IspResponse; +import com.maxmind.geoip2.record.Continent; +import com.maxmind.geoip2.record.Location; +import com.maxmind.geoip2.record.Postal; +import com.maxmind.geoip2.record.Subdivision; + +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.core.Nullable; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * A collection of {@link IpDataLookup} implementations for MaxMind databases + */ +final class MaxmindIpDataLookups { + + private MaxmindIpDataLookups() { + // utility class + } + + static class AnonymousIp extends AbstractBase { + AnonymousIp(final Set properties) { + super( + properties, + AnonymousIpResponse.class, + (response, ipAddress, network, locales) -> new AnonymousIpResponse(response, ipAddress, network) + ); + } + + @Override + protected Map transform(final AnonymousIpResponse response) { + boolean isHostingProvider = response.isHostingProvider(); + boolean isTorExitNode = response.isTorExitNode(); + boolean isAnonymousVpn = response.isAnonymousVpn(); + boolean isAnonymous = response.isAnonymous(); + boolean isPublicProxy = response.isPublicProxy(); + boolean isResidentialProxy = response.isResidentialProxy(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case HOSTING_PROVIDER -> { + data.put("hosting_provider", isHostingProvider); + } + case TOR_EXIT_NODE -> { + data.put("tor_exit_node", isTorExitNode); + } + case ANONYMOUS_VPN -> { + data.put("anonymous_vpn", isAnonymousVpn); + } + case ANONYMOUS -> { + data.put("anonymous", isAnonymous); + } + case PUBLIC_PROXY -> { + data.put("public_proxy", isPublicProxy); + } + case RESIDENTIAL_PROXY -> { + data.put("residential_proxy", isResidentialProxy); + } + } + } + return data; + } + } + + static class Asn extends AbstractBase { + Asn(Set properties) { + super(properties, AsnResponse.class, (response, ipAddress, network, locales) -> new AsnResponse(response, ipAddress, network)); + } + + @Override + protected Map transform(final AsnResponse response) { + Long asn = response.getAutonomousSystemNumber(); + String organizationName = response.getAutonomousSystemOrganization(); + Network network = response.getNetwork(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case ASN -> { + if (asn != null) { + data.put("asn", asn); + } + } + case ORGANIZATION_NAME -> { + if (organizationName != null) { + data.put("organization_name", organizationName); + } + } + case NETWORK -> { + if (network != null) { + data.put("network", network.toString()); + } + } + } + } + return data; + } + } + + static class City extends AbstractBase { + City(final Set properties) { + super(properties, CityResponse.class, CityResponse::new); + } + + @Override + protected Map transform(final CityResponse response) { + com.maxmind.geoip2.record.Country country = response.getCountry(); + com.maxmind.geoip2.record.City city = response.getCity(); + Location location = response.getLocation(); + Continent continent = response.getContinent(); + Subdivision subdivision = response.getMostSpecificSubdivision(); + Postal postal = response.getPostal(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getTraits().getIpAddress()); + case COUNTRY_IN_EUROPEAN_UNION -> { + if (country.getIsoCode() != null) { + // isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country + data.put("country_in_european_union", country.isInEuropeanUnion()); + } + } + case COUNTRY_ISO_CODE -> { + String countryIsoCode = country.getIsoCode(); + if (countryIsoCode != null) { + data.put("country_iso_code", countryIsoCode); + } + } + case COUNTRY_NAME -> { + String countryName = country.getName(); + if (countryName != null) { + data.put("country_name", countryName); + } + } + case CONTINENT_CODE -> { + String continentCode = continent.getCode(); + if (continentCode != null) { + data.put("continent_code", continentCode); + } + } + case CONTINENT_NAME -> { + String continentName = continent.getName(); + if (continentName != null) { + data.put("continent_name", continentName); + } + } + case REGION_ISO_CODE -> { + // ISO 3166-2 code for country subdivisions. + // See iso.org/iso-3166-country-codes.html + String countryIso = country.getIsoCode(); + String subdivisionIso = subdivision.getIsoCode(); + if (countryIso != null && subdivisionIso != null) { + String regionIsoCode = countryIso + "-" + subdivisionIso; + data.put("region_iso_code", regionIsoCode); + } + } + case REGION_NAME -> { + String subdivisionName = subdivision.getName(); + if (subdivisionName != null) { + data.put("region_name", subdivisionName); + } + } + case CITY_NAME -> { + String cityName = city.getName(); + if (cityName != null) { + data.put("city_name", cityName); + } + } + case TIMEZONE -> { + String locationTimeZone = location.getTimeZone(); + if (locationTimeZone != null) { + data.put("timezone", locationTimeZone); + } + } + case LOCATION -> { + Double latitude = location.getLatitude(); + Double longitude = location.getLongitude(); + if (latitude != null && longitude != null) { + Map locationObject = new HashMap<>(); + locationObject.put("lat", latitude); + locationObject.put("lon", longitude); + data.put("location", locationObject); + } + } + case ACCURACY_RADIUS -> { + Integer accuracyRadius = location.getAccuracyRadius(); + if (accuracyRadius != null) { + data.put("accuracy_radius", accuracyRadius); + } + } + case POSTAL_CODE -> { + if (postal != null && postal.getCode() != null) { + data.put("postal_code", postal.getCode()); + } + } + } + } + return data; + } + } + + static class ConnectionType extends AbstractBase { + ConnectionType(final Set properties) { + super( + properties, + ConnectionTypeResponse.class, + (response, ipAddress, network, locales) -> new ConnectionTypeResponse(response, ipAddress, network) + ); + } + + @Override + protected Map transform(final ConnectionTypeResponse response) { + ConnectionTypeResponse.ConnectionType connectionType = response.getConnectionType(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case CONNECTION_TYPE -> { + if (connectionType != null) { + data.put("connection_type", connectionType.toString()); + } + } + } + } + return data; + } + } + + static class Country extends AbstractBase { + Country(final Set properties) { + super(properties, CountryResponse.class, CountryResponse::new); + } + + @Override + protected Map transform(final CountryResponse response) { + com.maxmind.geoip2.record.Country country = response.getCountry(); + Continent continent = response.getContinent(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getTraits().getIpAddress()); + case COUNTRY_IN_EUROPEAN_UNION -> { + if (country.getIsoCode() != null) { + // isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country + data.put("country_in_european_union", country.isInEuropeanUnion()); + } + } + case COUNTRY_ISO_CODE -> { + String countryIsoCode = country.getIsoCode(); + if (countryIsoCode != null) { + data.put("country_iso_code", countryIsoCode); + } + } + case COUNTRY_NAME -> { + String countryName = country.getName(); + if (countryName != null) { + data.put("country_name", countryName); + } + } + case CONTINENT_CODE -> { + String continentCode = continent.getCode(); + if (continentCode != null) { + data.put("continent_code", continentCode); + } + } + case CONTINENT_NAME -> { + String continentName = continent.getName(); + if (continentName != null) { + data.put("continent_name", continentName); + } + } + } + } + return data; + } + } + + static class Domain extends AbstractBase { + Domain(final Set properties) { + super( + properties, + DomainResponse.class, + (response, ipAddress, network, locales) -> new DomainResponse(response, ipAddress, network) + ); + } + + @Override + protected Map transform(final DomainResponse response) { + String domain = response.getDomain(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case DOMAIN -> { + if (domain != null) { + data.put("domain", domain); + } + } + } + } + return data; + } + } + + static class Enterprise extends AbstractBase { + Enterprise(final Set properties) { + super(properties, EnterpriseResponse.class, EnterpriseResponse::new); + } + + @Override + protected Map transform(final EnterpriseResponse response) { + com.maxmind.geoip2.record.Country country = response.getCountry(); + com.maxmind.geoip2.record.City city = response.getCity(); + Location location = response.getLocation(); + Continent continent = response.getContinent(); + Subdivision subdivision = response.getMostSpecificSubdivision(); + Postal postal = response.getPostal(); + + Long asn = response.getTraits().getAutonomousSystemNumber(); + String organizationName = response.getTraits().getAutonomousSystemOrganization(); + Network network = response.getTraits().getNetwork(); + + String isp = response.getTraits().getIsp(); + String ispOrganization = response.getTraits().getOrganization(); + String mobileCountryCode = response.getTraits().getMobileCountryCode(); + String mobileNetworkCode = response.getTraits().getMobileNetworkCode(); + + boolean isHostingProvider = response.getTraits().isHostingProvider(); + boolean isTorExitNode = response.getTraits().isTorExitNode(); + boolean isAnonymousVpn = response.getTraits().isAnonymousVpn(); + boolean isAnonymous = response.getTraits().isAnonymous(); + boolean isPublicProxy = response.getTraits().isPublicProxy(); + boolean isResidentialProxy = response.getTraits().isResidentialProxy(); + + String userType = response.getTraits().getUserType(); + + String domain = response.getTraits().getDomain(); + + ConnectionTypeResponse.ConnectionType connectionType = response.getTraits().getConnectionType(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getTraits().getIpAddress()); + case COUNTRY_CONFIDENCE -> { + Integer countryConfidence = country.getConfidence(); + if (countryConfidence != null) { + data.put("country_confidence", countryConfidence); + } + } + case COUNTRY_IN_EUROPEAN_UNION -> { + if (country.getIsoCode() != null) { + // isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country + data.put("country_in_european_union", country.isInEuropeanUnion()); + } + } + case COUNTRY_ISO_CODE -> { + String countryIsoCode = country.getIsoCode(); + if (countryIsoCode != null) { + data.put("country_iso_code", countryIsoCode); + } + } + case COUNTRY_NAME -> { + String countryName = country.getName(); + if (countryName != null) { + data.put("country_name", countryName); + } + } + case CONTINENT_CODE -> { + String continentCode = continent.getCode(); + if (continentCode != null) { + data.put("continent_code", continentCode); + } + } + case CONTINENT_NAME -> { + String continentName = continent.getName(); + if (continentName != null) { + data.put("continent_name", continentName); + } + } + case REGION_ISO_CODE -> { + // ISO 3166-2 code for country subdivisions. + // See iso.org/iso-3166-country-codes.html + String countryIso = country.getIsoCode(); + String subdivisionIso = subdivision.getIsoCode(); + if (countryIso != null && subdivisionIso != null) { + String regionIsoCode = countryIso + "-" + subdivisionIso; + data.put("region_iso_code", regionIsoCode); + } + } + case REGION_NAME -> { + String subdivisionName = subdivision.getName(); + if (subdivisionName != null) { + data.put("region_name", subdivisionName); + } + } + case CITY_CONFIDENCE -> { + Integer cityConfidence = city.getConfidence(); + if (cityConfidence != null) { + data.put("city_confidence", cityConfidence); + } + } + case CITY_NAME -> { + String cityName = city.getName(); + if (cityName != null) { + data.put("city_name", cityName); + } + } + case TIMEZONE -> { + String locationTimeZone = location.getTimeZone(); + if (locationTimeZone != null) { + data.put("timezone", locationTimeZone); + } + } + case LOCATION -> { + Double latitude = location.getLatitude(); + Double longitude = location.getLongitude(); + if (latitude != null && longitude != null) { + Map locationObject = new HashMap<>(); + locationObject.put("lat", latitude); + locationObject.put("lon", longitude); + data.put("location", locationObject); + } + } + case ACCURACY_RADIUS -> { + Integer accuracyRadius = location.getAccuracyRadius(); + if (accuracyRadius != null) { + data.put("accuracy_radius", accuracyRadius); + } + } + case POSTAL_CODE -> { + if (postal != null && postal.getCode() != null) { + data.put("postal_code", postal.getCode()); + } + } + case POSTAL_CONFIDENCE -> { + Integer postalConfidence = postal.getConfidence(); + if (postalConfidence != null) { + data.put("postal_confidence", postalConfidence); + } + } + case ASN -> { + if (asn != null) { + data.put("asn", asn); + } + } + case ORGANIZATION_NAME -> { + if (organizationName != null) { + data.put("organization_name", organizationName); + } + } + case NETWORK -> { + if (network != null) { + data.put("network", network.toString()); + } + } + case HOSTING_PROVIDER -> { + data.put("hosting_provider", isHostingProvider); + } + case TOR_EXIT_NODE -> { + data.put("tor_exit_node", isTorExitNode); + } + case ANONYMOUS_VPN -> { + data.put("anonymous_vpn", isAnonymousVpn); + } + case ANONYMOUS -> { + data.put("anonymous", isAnonymous); + } + case PUBLIC_PROXY -> { + data.put("public_proxy", isPublicProxy); + } + case RESIDENTIAL_PROXY -> { + data.put("residential_proxy", isResidentialProxy); + } + case DOMAIN -> { + if (domain != null) { + data.put("domain", domain); + } + } + case ISP -> { + if (isp != null) { + data.put("isp", isp); + } + } + case ISP_ORGANIZATION_NAME -> { + if (ispOrganization != null) { + data.put("isp_organization_name", ispOrganization); + } + } + case MOBILE_COUNTRY_CODE -> { + if (mobileCountryCode != null) { + data.put("mobile_country_code", mobileCountryCode); + } + } + case MOBILE_NETWORK_CODE -> { + if (mobileNetworkCode != null) { + data.put("mobile_network_code", mobileNetworkCode); + } + } + case USER_TYPE -> { + if (userType != null) { + data.put("user_type", userType); + } + } + case CONNECTION_TYPE -> { + if (connectionType != null) { + data.put("connection_type", connectionType.toString()); + } + } + } + } + return data; + } + } + + static class Isp extends AbstractBase { + Isp(final Set properties) { + super(properties, IspResponse.class, (response, ipAddress, network, locales) -> new IspResponse(response, ipAddress, network)); + } + + @Override + protected Map transform(final IspResponse response) { + String isp = response.getIsp(); + String ispOrganization = response.getOrganization(); + String mobileNetworkCode = response.getMobileNetworkCode(); + String mobileCountryCode = response.getMobileCountryCode(); + Long asn = response.getAutonomousSystemNumber(); + String organizationName = response.getAutonomousSystemOrganization(); + Network network = response.getNetwork(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case ASN -> { + if (asn != null) { + data.put("asn", asn); + } + } + case ORGANIZATION_NAME -> { + if (organizationName != null) { + data.put("organization_name", organizationName); + } + } + case NETWORK -> { + if (network != null) { + data.put("network", network.toString()); + } + } + case ISP -> { + if (isp != null) { + data.put("isp", isp); + } + } + case ISP_ORGANIZATION_NAME -> { + if (ispOrganization != null) { + data.put("isp_organization_name", ispOrganization); + } + } + case MOBILE_COUNTRY_CODE -> { + if (mobileCountryCode != null) { + data.put("mobile_country_code", mobileCountryCode); + } + } + case MOBILE_NETWORK_CODE -> { + if (mobileNetworkCode != null) { + data.put("mobile_network_code", mobileNetworkCode); + } + } + } + } + return data; + } + } + + /** + * As an internal detail, the {@code com.maxmind.geoip2.model } classes that are populated by + * {@link Reader#getRecord(InetAddress, Class)} are kinda half-populated and need to go through a second round of construction + * with context from the querying caller. This method gives us a place do that additional binding. Cleverly, the signature + * here matches the constructor for many of these model classes exactly, so an appropriate implementation can 'just' be a method + * reference in some cases (in other cases it needs to be a lambda). + */ + @FunctionalInterface + private interface ResponseBuilder { + RESPONSE build(RESPONSE resp, String address, Network network, List locales); + } + + /** + * The {@link MaxmindIpDataLookups.AbstractBase} is an abstract base implementation of {@link IpDataLookup} that + * provides common functionality for getting a specific kind of {@link AbstractResponse} from a {@link IpDatabase}. + * + * @param the intermediate type of {@link AbstractResponse} + */ + private abstract static class AbstractBase implements IpDataLookup { + + protected final Set properties; + protected final Class clazz; + protected final ResponseBuilder builder; + + AbstractBase(final Set properties, final Class clazz, final ResponseBuilder builder) { + this.properties = Set.copyOf(properties); + this.clazz = clazz; + this.builder = builder; + } + + @Override + public Set getProperties() { + return this.properties; + } + + @Override + public final Map getData(final IpDatabase ipDatabase, final String ipAddress) { + final RESPONSE response = ipDatabase.getResponse(ipAddress, this::lookup); + return (response == null) ? Map.of() : transform(response); + } + + @Nullable + private RESPONSE lookup(final Reader reader, final String ipAddress) throws IOException { + final InetAddress ip = InetAddresses.forString(ipAddress); + final DatabaseRecord record = reader.getRecord(ip, clazz); + final RESPONSE data = record.getData(); + return (data == null) ? null : builder.build(data, NetworkAddress.format(ip), record.getNetwork(), List.of("en")); + } + + /** + * Extract the configured properties from the retrieved response + * @param response the non-null response that was retrieved + * @return a mapping of properties for the ip from the response + */ + protected abstract Map transform(RESPONSE response); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java index 83b3d2cfbbc27..7f38a37b43edf 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java @@ -126,7 +126,7 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception { DatabaseReaderLazyLoader loader = configDatabases.getDatabase("GeoLite2-City.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); - CityResponse cityResponse = loader.getCity("89.160.20.128"); + CityResponse cityResponse = loader.getResponse("89.160.20.128", GeoIpTestUtils::getCity); assertThat(cityResponse.getCity().getName(), equalTo("Tumba")); assertThat(cache.count(), equalTo(1)); } @@ -138,7 +138,7 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception { DatabaseReaderLazyLoader loader = configDatabases.getDatabase("GeoLite2-City.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); - CityResponse cityResponse = loader.getCity("89.160.20.128"); + CityResponse cityResponse = loader.getResponse("89.160.20.128", GeoIpTestUtils::getCity); assertThat(cityResponse.getCity().getName(), equalTo("Linköping")); assertThat(cache.count(), equalTo(1)); }); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java index 9972db26b3642..cfea54d2520bd 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java @@ -195,7 +195,7 @@ public void testBuildWithCountryDbAndAsnFields() { equalTo( "[properties] illegal property value [" + asnProperty - + "]. valid values are [IP, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME]" + + "]. valid values are [IP, COUNTRY_IN_EUROPEAN_UNION, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME]" ) ); } @@ -273,8 +273,9 @@ public void testBuildIllegalFieldOption() { assertThat( e.getMessage(), equalTo( - "[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_ISO_CODE, " - + "COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, CITY_NAME, TIMEZONE, LOCATION]" + "[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_IN_EUROPEAN_UNION, COUNTRY_ISO_CODE, " + + "COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, CITY_NAME, TIMEZONE, " + + "LOCATION, POSTAL_CODE, ACCURACY_RADIUS]" ) ); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index f5c3c08579855..ffc40324bd886 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.core.IOUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.RandomDocumentPicks; -import org.elasticsearch.ingest.geoip.Database.Property; import org.elasticsearch.test.ESTestCase; import org.junit.After; import org.junit.Before; @@ -40,7 +39,9 @@ public class GeoIpProcessorTests extends ESTestCase { - private static final Set ALL_PROPERTIES = Set.of(Property.values()); + private static IpDataLookup ipDataLookupAll(final Database database) { + return IpDataLookupFactories.getMaxmindLookup(database).apply(database.properties()); + } // a temporary directory that mmdb files can be copied to and read from private Path tmpDir; @@ -64,8 +65,16 @@ public void testDatabasePropertyInvariants() { assertThat(Sets.difference(Database.Asn.properties(), Database.Isp.properties()), is(empty())); assertThat(Sets.difference(Database.Asn.defaultProperties(), Database.Isp.defaultProperties()), is(empty())); - // the enterprise database is like everything joined together - for (Database type : Database.values()) { + // the enterprise database is like these other databases joined together + for (Database type : Set.of( + Database.City, + Database.Country, + Database.Asn, + Database.AnonymousIp, + Database.ConnectionType, + Database.Domain, + Database.Isp + )) { assertThat(Sets.difference(type.properties(), Database.Enterprise.properties()), is(empty())); } // but in terms of the default fields, it's like a drop-in replacement for the city database @@ -82,7 +91,7 @@ public void testCity() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -97,8 +106,9 @@ public void testCity() throws Exception { @SuppressWarnings("unchecked") Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(7)); + assertThat(geoData.size(), equalTo(9)); assertThat(geoData.get("ip"), equalTo(ip)); + assertThat(geoData.get("country_in_european_union"), equalTo(false)); assertThat(geoData.get("country_iso_code"), equalTo("US")); assertThat(geoData.get("country_name"), equalTo("United States")); assertThat(geoData.get("continent_code"), equalTo("NA")); @@ -115,7 +125,7 @@ public void testNullValueWithIgnoreMissing() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), true, false, "filename" @@ -137,7 +147,7 @@ public void testNonExistentWithIgnoreMissing() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), true, false, "filename" @@ -156,7 +166,7 @@ public void testNullWithoutIgnoreMissing() { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -178,7 +188,7 @@ public void testNonExistentWithoutIgnoreMissing() { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -198,7 +208,7 @@ public void testCity_withIpV6() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -213,8 +223,9 @@ public void testCity_withIpV6() throws Exception { @SuppressWarnings("unchecked") Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(10)); + assertThat(geoData.size(), equalTo(13)); assertThat(geoData.get("ip"), equalTo(ip)); + assertThat(geoData.get("country_in_european_union"), equalTo(false)); assertThat(geoData.get("country_iso_code"), equalTo("US")); assertThat(geoData.get("country_name"), equalTo("United States")); assertThat(geoData.get("continent_code"), equalTo("NA")); @@ -224,6 +235,8 @@ public void testCity_withIpV6() throws Exception { assertThat(geoData.get("city_name"), equalTo("Homestead")); assertThat(geoData.get("timezone"), equalTo("America/New_York")); assertThat(geoData.get("location"), equalTo(Map.of("lat", 25.4573d, "lon", -80.4572d))); + assertThat(geoData.get("accuracy_radius"), equalTo(50)); + assertThat(geoData.get("postal_code"), equalTo("33035")); } public void testCityWithMissingLocation() throws Exception { @@ -235,7 +248,7 @@ public void testCityWithMissingLocation() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -263,7 +276,7 @@ public void testCountry() throws Exception { loader("GeoLite2-Country.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.Country), false, false, "filename" @@ -278,8 +291,9 @@ public void testCountry() throws Exception { @SuppressWarnings("unchecked") Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(5)); + assertThat(geoData.size(), equalTo(6)); assertThat(geoData.get("ip"), equalTo(ip)); + assertThat(geoData.get("country_in_european_union"), equalTo(true)); assertThat(geoData.get("country_iso_code"), equalTo("NL")); assertThat(geoData.get("country_name"), equalTo("Netherlands")); assertThat(geoData.get("continent_code"), equalTo("EU")); @@ -295,7 +309,7 @@ public void testCountryWithMissingLocation() throws Exception { loader("GeoLite2-Country.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.Country), false, false, "filename" @@ -323,7 +337,7 @@ public void testAsn() throws Exception { loader("GeoLite2-ASN.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.Asn), false, false, "filename" @@ -354,7 +368,7 @@ public void testAnonymmousIp() throws Exception { loader("GeoIP2-Anonymous-IP-Test.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.AnonymousIp), false, false, "filename" @@ -388,7 +402,7 @@ public void testConnectionType() throws Exception { loader("GeoIP2-Connection-Type-Test.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.ConnectionType), false, false, "filename" @@ -417,7 +431,7 @@ public void testDomain() throws Exception { loader("GeoIP2-Domain-Test.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.Domain), false, false, "filename" @@ -446,7 +460,7 @@ public void testEnterprise() throws Exception { loader("GeoIP2-Enterprise-Test.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.Enterprise), false, false, "filename" @@ -461,17 +475,23 @@ public void testEnterprise() throws Exception { @SuppressWarnings("unchecked") Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(24)); + assertThat(geoData.size(), equalTo(30)); assertThat(geoData.get("ip"), equalTo(ip)); + assertThat(geoData.get("country_confidence"), equalTo(99)); + assertThat(geoData.get("country_in_european_union"), equalTo(false)); assertThat(geoData.get("country_iso_code"), equalTo("US")); assertThat(geoData.get("country_name"), equalTo("United States")); assertThat(geoData.get("continent_code"), equalTo("NA")); assertThat(geoData.get("continent_name"), equalTo("North America")); assertThat(geoData.get("region_iso_code"), equalTo("US-NY")); assertThat(geoData.get("region_name"), equalTo("New York")); + assertThat(geoData.get("city_confidence"), equalTo(11)); assertThat(geoData.get("city_name"), equalTo("Chatham")); assertThat(geoData.get("timezone"), equalTo("America/New_York")); assertThat(geoData.get("location"), equalTo(Map.of("lat", 42.3478, "lon", -73.5549))); + assertThat(geoData.get("accuracy_radius"), equalTo(27)); + assertThat(geoData.get("postal_code"), equalTo("12037")); + assertThat(geoData.get("city_confidence"), equalTo(11)); assertThat(geoData.get("asn"), equalTo(14671L)); assertThat(geoData.get("organization_name"), equalTo("FairPoint Communications")); assertThat(geoData.get("network"), equalTo("74.209.16.0/20")); @@ -497,7 +517,7 @@ public void testIsp() throws Exception { loader("GeoIP2-ISP-Test.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.Isp), false, false, "filename" @@ -531,7 +551,7 @@ public void testAddressIsNotInTheDatabase() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -555,7 +575,7 @@ public void testInvalid() { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -576,7 +596,7 @@ public void testListAllValid() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -603,7 +623,7 @@ public void testListPartiallyValid() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -630,7 +650,7 @@ public void testListNoMatches() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -650,7 +670,7 @@ public void testListDatabaseReferenceCounting() throws Exception { GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", () -> { loader.preLookup(); return loader; - }, () -> true, "target_field", ALL_PROPERTIES, false, false, "filename"); + }, () -> true, "target_field", ipDataLookupAll(Database.City), false, false, "filename"); Map document = new HashMap<>(); document.put("source_field", List.of("8.8.8.8", "82.171.64.0")); @@ -678,7 +698,7 @@ public void testListFirstOnly() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, true, "filename" @@ -703,7 +723,7 @@ public void testListFirstOnlyNoMatches() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, true, "filename" @@ -725,7 +745,7 @@ public void testInvalidDatabase() throws Exception { loader("GeoLite2-City.mmdb"), () -> false, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, true, "filename" @@ -748,7 +768,7 @@ public void testNoDatabase() throws Exception { () -> null, () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "GeoLite2-City" @@ -771,7 +791,7 @@ public void testNoDatabase_ignoreMissing() throws Exception { () -> null, () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), true, false, "GeoLite2-City" diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java index 461983bb24488..160671fd39001 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java @@ -9,6 +9,13 @@ package org.elasticsearch.ingest.geoip; +import com.maxmind.db.DatabaseRecord; +import com.maxmind.db.Reader; +import com.maxmind.geoip2.model.CityResponse; +import com.maxmind.geoip2.model.CountryResponse; + +import org.elasticsearch.common.CheckedBiFunction; +import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.core.SuppressForbidden; import java.io.FileNotFoundException; @@ -17,6 +24,7 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import java.util.Set; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; @@ -58,4 +66,28 @@ public static void copyDefaultDatabases(final Path directory, ConfigDatabases co configDatabases.updateDatabase(directory.resolve(database), true); } } + + /** + * A static city-specific responseProvider for use with {@link IpDatabase#getResponse(String, CheckedBiFunction)} in + * tests. + *

+ * Like this: {@code CityResponse city = loader.getResponse("some.ip.address", GeoIpTestUtils::getCity);} + */ + public static CityResponse getCity(Reader reader, String ip) throws IOException { + DatabaseRecord record = reader.getRecord(InetAddresses.forString(ip), CityResponse.class); + CityResponse data = record.getData(); + return data == null ? null : new CityResponse(data, ip, record.getNetwork(), List.of("en")); + } + + /** + * A static country-specific responseProvider for use with {@link IpDatabase#getResponse(String, CheckedBiFunction)} in + * tests. + *

+ * Like this: {@code CountryResponse country = loader.getResponse("some.ip.address", GeoIpTestUtils::getCountry);} + */ + public static CountryResponse getCountry(Reader reader, String ip) throws IOException { + DatabaseRecord record = reader.getRecord(InetAddresses.forString(ip), CountryResponse.class); + CountryResponse data = record.getData(); + return data == null ? null : new CountryResponse(data, ip, record.getNetwork(), List.of("en")); + } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java new file mode 100644 index 0000000000000..905eb027626a1 --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java @@ -0,0 +1,223 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import com.maxmind.db.DatabaseRecord; +import com.maxmind.db.Networks; +import com.maxmind.db.Reader; + +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.junit.After; +import org.junit.Before; + +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.nio.file.Path; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; + +import static java.util.Map.entry; +import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase; +import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseAsn; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.startsWith; + +public class IpinfoIpDataLookupsTests extends ESTestCase { + + private ThreadPool threadPool; + private ResourceWatcherService resourceWatcherService; + + @Before + public void setup() { + threadPool = new TestThreadPool(ConfigDatabases.class.getSimpleName()); + Settings settings = Settings.builder().put("resource.reload.interval.high", TimeValue.timeValueMillis(100)).build(); + resourceWatcherService = new ResourceWatcherService(settings, threadPool); + } + + @After + public void cleanup() { + resourceWatcherService.close(); + threadPool.shutdownNow(); + } + + public void testDatabasePropertyInvariants() { + // the second ASN variant database is like a specialization of the ASN database + assertThat(Sets.difference(Database.Asn.properties(), Database.AsnV2.properties()), is(empty())); + assertThat(Database.Asn.defaultProperties(), equalTo(Database.AsnV2.defaultProperties())); + } + + public void testParseAsn() { + // expected case: "AS123" is 123 + assertThat(parseAsn("AS123"), equalTo(123L)); + // defensive cases: null and empty becomes null, this is not expected fwiw + assertThat(parseAsn(null), nullValue()); + assertThat(parseAsn(""), nullValue()); + // defensive cases: we strip whitespace and ignore case + assertThat(parseAsn(" as 456 "), equalTo(456L)); + // defensive cases: we ignore the absence of the 'AS' prefix + assertThat(parseAsn("123"), equalTo(123L)); + // bottom case: a non-parsable string is null + assertThat(parseAsn("anythingelse"), nullValue()); + } + + public void testAsn() throws IOException { + Path configDir = createTempDir(); + copyDatabase("ipinfo/ip_asn_sample.mmdb", configDir.resolve("ip_asn_sample.mmdb")); + copyDatabase("ipinfo/asn_sample.mmdb", configDir.resolve("asn_sample.mmdb")); + + GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload + ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache); + configDatabases.initialize(resourceWatcherService); + + // this is the 'free' ASN database (sample) + { + DatabaseReaderLazyLoader loader = configDatabases.getDatabase("ip_asn_sample.mmdb"); + IpDataLookup lookup = new IpinfoIpDataLookups.Asn(Set.of(Database.Property.values())); + Map data = lookup.getData(loader, "5.182.109.0"); + assertThat( + data, + equalTo( + Map.ofEntries( + entry("ip", "5.182.109.0"), + entry("organization_name", "M247 Europe SRL"), + entry("asn", 9009L), + entry("network", "5.182.109.0/24"), + entry("domain", "m247.com") + ) + ) + ); + } + + // this is the non-free or 'standard' ASN database (sample) + { + DatabaseReaderLazyLoader loader = configDatabases.getDatabase("asn_sample.mmdb"); + IpDataLookup lookup = new IpinfoIpDataLookups.Asn(Set.of(Database.Property.values())); + Map data = lookup.getData(loader, "23.53.116.0"); + assertThat( + data, + equalTo( + Map.ofEntries( + entry("ip", "23.53.116.0"), + entry("organization_name", "Akamai Technologies, Inc."), + entry("asn", 32787L), + entry("network", "23.53.116.0/24"), + entry("domain", "akamai.com"), + entry("type", "hosting"), + entry("country_iso_code", "US") + ) + ) + ); + } + } + + public void testAsnInvariants() { + Path configDir = createTempDir(); + copyDatabase("ipinfo/ip_asn_sample.mmdb", configDir.resolve("ip_asn_sample.mmdb")); + copyDatabase("ipinfo/asn_sample.mmdb", configDir.resolve("asn_sample.mmdb")); + + { + final Set expectedColumns = Set.of("network", "asn", "name", "domain"); + + Path databasePath = configDir.resolve("ip_asn_sample.mmdb"); + assertDatabaseInvariants(databasePath, (ip, row) -> { + assertThat(row.keySet(), equalTo(expectedColumns)); + String asn = (String) row.get("asn"); + assertThat(asn, startsWith("AS")); + assertThat(asn, equalTo(asn.trim())); + Long parsed = parseAsn(asn); + assertThat(parsed, notNullValue()); + assertThat(asn, equalTo("AS" + parsed)); // reverse it + }); + } + + { + final Set expectedColumns = Set.of("network", "asn", "name", "domain", "country", "type"); + + Path databasePath = configDir.resolve("asn_sample.mmdb"); + assertDatabaseInvariants(databasePath, (ip, row) -> { + assertThat(row.keySet(), equalTo(expectedColumns)); + String asn = (String) row.get("asn"); + assertThat(asn, startsWith("AS")); + assertThat(asn, equalTo(asn.trim())); + Long parsed = parseAsn(asn); + assertThat(parsed, notNullValue()); + assertThat(asn, equalTo("AS" + parsed)); // reverse it + }); + } + } + + public void testCountry() throws IOException { + Path configDir = createTempDir(); + copyDatabase("ipinfo/ip_country_sample.mmdb", configDir.resolve("ip_country_sample.mmdb")); + + GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload + ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache); + configDatabases.initialize(resourceWatcherService); + + // this is the 'free' Country database (sample) + { + DatabaseReaderLazyLoader loader = configDatabases.getDatabase("ip_country_sample.mmdb"); + IpDataLookup lookup = new IpinfoIpDataLookups.Country(Set.of(Database.Property.values())); + Map data = lookup.getData(loader, "4.221.143.168"); + assertThat( + data, + equalTo( + Map.ofEntries( + entry("ip", "4.221.143.168"), + entry("country_name", "South Africa"), + entry("country_iso_code", "ZA"), + entry("continent_name", "Africa"), + entry("continent_code", "AF") + ) + ) + ); + } + } + + private static void assertDatabaseInvariants(final Path databasePath, final BiConsumer> rowConsumer) { + try (Reader reader = new Reader(pathToFile(databasePath))) { + Networks networks = reader.networks(Map.class); + while (networks.hasNext()) { + DatabaseRecord dbr = networks.next(); + InetAddress address = dbr.getNetwork().getNetworkAddress(); + @SuppressWarnings("unchecked") + Map result = reader.get(address, Map.class); + try { + rowConsumer.accept(address, result); + } catch (AssertionError e) { + fail(e, "Assert failed for address [%s]", NetworkAddress.format(address)); + } catch (Exception e) { + fail(e, "Exception handling address [%s]", NetworkAddress.format(address)); + } + } + } catch (Exception e) { + fail(e); + } + } + + @SuppressForbidden(reason = "Maxmind API requires java.io.File") + private static File pathToFile(Path databasePath) { + return databasePath.toFile(); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java index f1c7d809b98fe..46a34c2cdad56 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java @@ -116,6 +116,6 @@ public void testDatabaseTypeParsing() throws IOException { } private Database parseDatabaseFromType(String databaseFile) throws IOException { - return Database.getDatabase(MMDBUtil.getDatabaseType(tmpDir.resolve(databaseFile)), null); + return IpDataLookupFactories.getDatabase(MMDBUtil.getDatabaseType(tmpDir.resolve(databaseFile))); } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java index ec05054615bd8..1e05cf2b3ba33 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java @@ -78,13 +78,16 @@ public class MaxMindSupportTests extends ESTestCase { "city.name", "continent.code", "continent.name", + "country.inEuropeanUnion", "country.isoCode", "country.name", + "location.accuracyRadius", "location.latitude", "location.longitude", "location.timeZone", "mostSpecificSubdivision.isoCode", - "mostSpecificSubdivision.name" + "mostSpecificSubdivision.name", + "postal.code" ); private static final Set CITY_UNSUPPORTED_FIELDS = Set.of( "city.confidence", @@ -94,14 +97,12 @@ public class MaxMindSupportTests extends ESTestCase { "continent.names", "country.confidence", "country.geoNameId", - "country.inEuropeanUnion", "country.names", "leastSpecificSubdivision.confidence", "leastSpecificSubdivision.geoNameId", "leastSpecificSubdivision.isoCode", "leastSpecificSubdivision.name", "leastSpecificSubdivision.names", - "location.accuracyRadius", "location.averageIncome", "location.metroCode", "location.populationDensity", @@ -109,7 +110,6 @@ public class MaxMindSupportTests extends ESTestCase { "mostSpecificSubdivision.confidence", "mostSpecificSubdivision.geoNameId", "mostSpecificSubdivision.names", - "postal.code", "postal.confidence", "registeredCountry.confidence", "registeredCountry.geoNameId", @@ -159,6 +159,7 @@ public class MaxMindSupportTests extends ESTestCase { private static final Set COUNTRY_SUPPORTED_FIELDS = Set.of( "continent.name", + "country.inEuropeanUnion", "country.isoCode", "continent.code", "country.name" @@ -168,7 +169,6 @@ public class MaxMindSupportTests extends ESTestCase { "continent.names", "country.confidence", "country.geoNameId", - "country.inEuropeanUnion", "country.names", "maxMind", "registeredCountry.confidence", @@ -213,16 +213,22 @@ public class MaxMindSupportTests extends ESTestCase { private static final Set DOMAIN_UNSUPPORTED_FIELDS = Set.of("ipAddress", "network"); private static final Set ENTERPRISE_SUPPORTED_FIELDS = Set.of( + "city.confidence", "city.name", "continent.code", "continent.name", + "country.confidence", + "country.inEuropeanUnion", "country.isoCode", "country.name", + "location.accuracyRadius", "location.latitude", "location.longitude", "location.timeZone", "mostSpecificSubdivision.isoCode", "mostSpecificSubdivision.name", + "postal.code", + "postal.confidence", "traits.anonymous", "traits.anonymousVpn", "traits.autonomousSystemNumber", @@ -241,21 +247,17 @@ public class MaxMindSupportTests extends ESTestCase { "traits.userType" ); private static final Set ENTERPRISE_UNSUPPORTED_FIELDS = Set.of( - "city.confidence", "city.geoNameId", "city.names", "continent.geoNameId", "continent.names", - "country.confidence", "country.geoNameId", - "country.inEuropeanUnion", "country.names", "leastSpecificSubdivision.confidence", "leastSpecificSubdivision.geoNameId", "leastSpecificSubdivision.isoCode", "leastSpecificSubdivision.name", "leastSpecificSubdivision.names", - "location.accuracyRadius", "location.averageIncome", "location.metroCode", "location.populationDensity", @@ -263,8 +265,6 @@ public class MaxMindSupportTests extends ESTestCase { "mostSpecificSubdivision.confidence", "mostSpecificSubdivision.geoNameId", "mostSpecificSubdivision.names", - "postal.code", - "postal.confidence", "registeredCountry.confidence", "registeredCountry.geoNameId", "registeredCountry.inEuropeanUnion", @@ -361,8 +361,14 @@ public class MaxMindSupportTests extends ESTestCase { private static final Set> KNOWN_UNSUPPORTED_RESPONSE_CLASSES = Set.of(IpRiskResponse.class); + private static final Set KNOWN_UNSUPPORTED_DATABASE_VARIANTS = Set.of(Database.AsnV2); + public void testMaxMindSupport() { for (Database databaseType : Database.values()) { + if (KNOWN_UNSUPPORTED_DATABASE_VARIANTS.contains(databaseType)) { + continue; + } + Class maxMindClass = TYPE_TO_MAX_MIND_CLASS.get(databaseType); Set supportedFields = TYPE_TO_SUPPORTED_FIELDS_MAP.get(databaseType); Set unsupportedFields = TYPE_TO_UNSUPPORTED_FIELDS_MAP.get(databaseType); @@ -468,36 +474,6 @@ public void testUnknownMaxMindResponseClassess() { ); } - /* - * This tests that this test has a mapping in TYPE_TO_MAX_MIND_CLASS for all MaxMind classes exposed through IpDatabase. - */ - public void testUsedMaxMindResponseClassesAreAccountedFor() { - Set> usedMaxMindResponseClasses = getUsedMaxMindResponseClasses(); - Set> supportedMaxMindClasses = new HashSet<>(TYPE_TO_MAX_MIND_CLASS.values()); - Set> usedButNotSupportedMaxMindResponseClasses = Sets.difference( - usedMaxMindResponseClasses, - supportedMaxMindClasses - ); - assertThat( - "IpDatabase exposes MaxMind response classes that this test does not know what to do with. Add mappings to " - + "TYPE_TO_MAX_MIND_CLASS for the following: " - + usedButNotSupportedMaxMindResponseClasses, - usedButNotSupportedMaxMindResponseClasses, - empty() - ); - Set> supportedButNotUsedMaxMindClasses = Sets.difference( - supportedMaxMindClasses, - usedMaxMindResponseClasses - ); - assertThat( - "This test claims to support MaxMind response classes that are not exposed in IpDatabase. Remove the following from " - + "TYPE_TO_MAX_MIND_CLASS: " - + supportedButNotUsedMaxMindClasses, - supportedButNotUsedMaxMindClasses, - empty() - ); - } - /* * This is the list of field types that causes us to stop recursing. That is, fields of these types are the lowest-level fields that * we care about. @@ -616,23 +592,4 @@ private static String getFormattedList(Set fields) { } return result.toString(); } - - /* - * This returns all AbstractResponse classes that are returned from getter methods on IpDatabase. - */ - private static Set> getUsedMaxMindResponseClasses() { - Set> result = new HashSet<>(); - Method[] methods = IpDatabase.class.getMethods(); - for (Method method : methods) { - if (method.getName().startsWith("get")) { - Class returnType = method.getReturnType(); - try { - result.add(returnType.asSubclass(AbstractResponse.class)); - } catch (ClassCastException ignore) { - // This is not what we were looking for, move on - } - } - } - return result; - } } diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/asn_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/asn_sample.mmdb new file mode 100644 index 0000000000000..916a8252a5df1 Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/ipinfo/asn_sample.mmdb differ diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/ip_asn_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/ip_asn_sample.mmdb new file mode 100644 index 0000000000000..3e1fc49ba48a5 Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/ipinfo/ip_asn_sample.mmdb differ diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/ip_country_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/ip_country_sample.mmdb new file mode 100644 index 0000000000000..88428315ee8d6 Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/ipinfo/ip_country_sample.mmdb differ diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryMetricsTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryMetricsTests.java new file mode 100644 index 0000000000000..a9bf0afa37e18 --- /dev/null +++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryMetricsTests.java @@ -0,0 +1,468 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.repositories.azure; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; + +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.blobstore.OperationPurpose; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.repositories.RepositoriesMetrics; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.repositories.blobstore.RequestedRangeNotSatisfiedException; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.telemetry.Measurement; +import org.elasticsearch.telemetry.TestTelemetryPlugin; +import org.junit.After; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.elasticsearch.repositories.azure.AbstractAzureServerTestCase.randomBlobContent; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +@SuppressForbidden(reason = "we use a HttpServer to emulate Azure") +public class AzureBlobStoreRepositoryMetricsTests extends AzureBlobStoreRepositoryTests { + + private static final Predicate GET_BLOB_REQUEST_PREDICATE = request -> GET_BLOB_PATTERN.test( + request.getRequestMethod() + " " + request.getRequestURI() + ); + private static final int MAX_RETRIES = 3; + + private final Queue requestHandlers = new ConcurrentLinkedQueue<>(); + + @Override + protected Map createHttpHandlers() { + Map httpHandlers = super.createHttpHandlers(); + assert httpHandlers.size() == 1 : "This assumes there's a single handler"; + return httpHandlers.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> new ResponseInjectingAzureHttpHandler(requestHandlers, e.getValue()))); + } + + /** + * We want to control the errors in this test + */ + @Override + protected HttpHandler createErroneousHttpHandler(HttpHandler delegate) { + return delegate; + } + + @After + public void checkRequestHandlerQueue() { + if (requestHandlers.isEmpty() == false) { + fail("There were unused request handlers left in the queue, this is probably a broken test"); + } + } + + private static BlobContainer getBlobContainer(String dataNodeName, String repository) { + final var blobStoreRepository = (BlobStoreRepository) internalCluster().getInstance(RepositoriesService.class, dataNodeName) + .repository(repository); + return blobStoreRepository.blobStore().blobContainer(BlobPath.EMPTY.add(randomIdentifier())); + } + + public void testThrottleResponsesAreCountedInMetrics() throws IOException { + final String repository = createRepository(randomRepositoryName()); + final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData); + final BlobContainer blobContainer = getBlobContainer(dataNodeName, repository); + + // Create a blob + final String blobName = "index-" + randomIdentifier(); + final OperationPurpose purpose = randomFrom(OperationPurpose.values()); + blobContainer.writeBlob(purpose, blobName, BytesReference.fromByteBuffer(ByteBuffer.wrap(randomBlobContent())), false); + clearMetrics(dataNodeName); + + // Queue up some throttle responses + final int numThrottles = randomIntBetween(1, MAX_RETRIES); + IntStream.range(0, numThrottles).forEach(i -> requestHandlers.offer(new FixedRequestHandler(RestStatus.TOO_MANY_REQUESTS))); + + // Check that the blob exists + blobContainer.blobExists(purpose, blobName); + + // Correct metrics are recorded + metricsAsserter(dataNodeName, purpose, AzureBlobStore.Operation.GET_BLOB_PROPERTIES, repository).expectMetrics() + .withRequests(numThrottles + 1) + .withThrottles(numThrottles) + .withExceptions(numThrottles) + .forResult(MetricsAsserter.Result.Success); + } + + public void testRangeNotSatisfiedAreCountedInMetrics() throws IOException { + final String repository = createRepository(randomRepositoryName()); + final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData); + final BlobContainer blobContainer = getBlobContainer(dataNodeName, repository); + + // Create a blob + final String blobName = "index-" + randomIdentifier(); + final OperationPurpose purpose = randomFrom(OperationPurpose.values()); + blobContainer.writeBlob(purpose, blobName, BytesReference.fromByteBuffer(ByteBuffer.wrap(randomBlobContent())), false); + clearMetrics(dataNodeName); + + // Queue up a range-not-satisfied error + requestHandlers.offer(new FixedRequestHandler(RestStatus.REQUESTED_RANGE_NOT_SATISFIED, null, GET_BLOB_REQUEST_PREDICATE)); + + // Attempt to read the blob + assertThrows(RequestedRangeNotSatisfiedException.class, () -> blobContainer.readBlob(purpose, blobName)); + + // Correct metrics are recorded + metricsAsserter(dataNodeName, purpose, AzureBlobStore.Operation.GET_BLOB, repository).expectMetrics() + .withRequests(1) + .withThrottles(0) + .withExceptions(1) + .forResult(MetricsAsserter.Result.RangeNotSatisfied); + } + + public void testErrorResponsesAreCountedInMetrics() throws IOException { + final String repository = createRepository(randomRepositoryName()); + final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData); + final BlobContainer blobContainer = getBlobContainer(dataNodeName, repository); + + // Create a blob + final String blobName = "index-" + randomIdentifier(); + final OperationPurpose purpose = randomFrom(OperationPurpose.values()); + blobContainer.writeBlob(purpose, blobName, BytesReference.fromByteBuffer(ByteBuffer.wrap(randomBlobContent())), false); + clearMetrics(dataNodeName); + + // Queue some retry-able error responses + final int numErrors = randomIntBetween(1, MAX_RETRIES); + final AtomicInteger throttles = new AtomicInteger(); + IntStream.range(0, numErrors).forEach(i -> { + RestStatus status = randomFrom(RestStatus.INTERNAL_SERVER_ERROR, RestStatus.TOO_MANY_REQUESTS, RestStatus.SERVICE_UNAVAILABLE); + if (status == RestStatus.TOO_MANY_REQUESTS) { + throttles.incrementAndGet(); + } + requestHandlers.offer(new FixedRequestHandler(status)); + }); + + // Check that the blob exists + blobContainer.blobExists(purpose, blobName); + + // Correct metrics are recorded + metricsAsserter(dataNodeName, purpose, AzureBlobStore.Operation.GET_BLOB_PROPERTIES, repository).expectMetrics() + .withRequests(numErrors + 1) + .withThrottles(throttles.get()) + .withExceptions(numErrors) + .forResult(MetricsAsserter.Result.Success); + } + + public void testRequestFailuresAreCountedInMetrics() { + final String repository = createRepository(randomRepositoryName()); + final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData); + final BlobContainer blobContainer = getBlobContainer(dataNodeName, repository); + clearMetrics(dataNodeName); + + // Repeatedly cause a connection error to exhaust retries + IntStream.range(0, MAX_RETRIES + 1).forEach(i -> requestHandlers.offer((exchange, delegate) -> exchange.close())); + + // Hit the API + OperationPurpose purpose = randomFrom(OperationPurpose.values()); + assertThrows(IOException.class, () -> blobContainer.listBlobs(purpose)); + + // Correct metrics are recorded + metricsAsserter(dataNodeName, purpose, AzureBlobStore.Operation.LIST_BLOBS, repository).expectMetrics() + .withRequests(4) + .withThrottles(0) + .withExceptions(4) + .forResult(MetricsAsserter.Result.Exception); + } + + public void testRequestTimeIsAccurate() throws IOException { + final String repository = createRepository(randomRepositoryName()); + final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData); + final BlobContainer blobContainer = getBlobContainer(dataNodeName, repository); + clearMetrics(dataNodeName); + + AtomicLong totalDelayMillis = new AtomicLong(0); + // Add some artificial delays + IntStream.range(0, randomIntBetween(1, MAX_RETRIES)).forEach(i -> { + long thisDelay = randomLongBetween(10, 100); + totalDelayMillis.addAndGet(thisDelay); + requestHandlers.offer((exchange, delegate) -> { + safeSleep(thisDelay); + // return a retry-able error + exchange.sendResponseHeaders(RestStatus.INTERNAL_SERVER_ERROR.getStatus(), -1); + }); + }); + + // Hit the API + final long startTimeMillis = System.currentTimeMillis(); + blobContainer.listBlobs(randomFrom(OperationPurpose.values())); + final long elapsedTimeMillis = System.currentTimeMillis() - startTimeMillis; + + List longHistogramMeasurement = getTelemetryPlugin(dataNodeName).getLongHistogramMeasurement( + RepositoriesMetrics.HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM + ); + long recordedRequestTime = longHistogramMeasurement.get(0).getLong(); + // Request time should be >= the delays we simulated + assertThat(recordedRequestTime, greaterThanOrEqualTo(totalDelayMillis.get())); + // And <= the elapsed time for the request + assertThat(recordedRequestTime, lessThanOrEqualTo(elapsedTimeMillis)); + } + + private void clearMetrics(String discoveryNode) { + internalCluster().getInstance(PluginsService.class, discoveryNode) + .filterPlugins(TestTelemetryPlugin.class) + .forEach(TestTelemetryPlugin::resetMeter); + } + + private MetricsAsserter metricsAsserter( + String dataNodeName, + OperationPurpose operationPurpose, + AzureBlobStore.Operation operation, + String repository + ) { + return new MetricsAsserter(dataNodeName, operationPurpose, operation, repository); + } + + private class MetricsAsserter { + private final String dataNodeName; + private final OperationPurpose purpose; + private final AzureBlobStore.Operation operation; + private final String repository; + + enum Result { + Success, + Failure, + RangeNotSatisfied, + Exception + } + + enum MetricType { + LongHistogram { + @Override + List getMeasurements(TestTelemetryPlugin testTelemetryPlugin, String name) { + return testTelemetryPlugin.getLongHistogramMeasurement(name); + } + }, + LongCounter { + @Override + List getMeasurements(TestTelemetryPlugin testTelemetryPlugin, String name) { + return testTelemetryPlugin.getLongCounterMeasurement(name); + } + }; + + abstract List getMeasurements(TestTelemetryPlugin testTelemetryPlugin, String name); + } + + private MetricsAsserter(String dataNodeName, OperationPurpose purpose, AzureBlobStore.Operation operation, String repository) { + this.dataNodeName = dataNodeName; + this.purpose = purpose; + this.operation = operation; + this.repository = repository; + } + + private class Expectations { + private int expectedRequests; + private int expectedThrottles; + private int expectedExceptions; + + public Expectations withRequests(int expectedRequests) { + this.expectedRequests = expectedRequests; + return this; + } + + public Expectations withThrottles(int expectedThrottles) { + this.expectedThrottles = expectedThrottles; + return this; + } + + public Expectations withExceptions(int expectedExceptions) { + this.expectedExceptions = expectedExceptions; + return this; + } + + public void forResult(Result result) { + assertMetricsRecorded(expectedRequests, expectedThrottles, expectedExceptions, result); + } + } + + Expectations expectMetrics() { + return new Expectations(); + } + + private void assertMetricsRecorded(int expectedRequests, int expectedThrottles, int expectedExceptions, Result result) { + assertIntMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_OPERATIONS_TOTAL, 1); + assertIntMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_REQUESTS_TOTAL, expectedRequests); + + if (expectedThrottles > 0) { + assertIntMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_THROTTLES_TOTAL, expectedThrottles); + assertIntMetricRecorded(MetricType.LongHistogram, RepositoriesMetrics.METRIC_THROTTLES_HISTOGRAM, expectedThrottles); + } else { + assertNoMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_THROTTLES_TOTAL); + assertNoMetricRecorded(MetricType.LongHistogram, RepositoriesMetrics.METRIC_THROTTLES_HISTOGRAM); + } + + if (expectedExceptions > 0) { + assertIntMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_EXCEPTIONS_TOTAL, expectedExceptions); + assertIntMetricRecorded(MetricType.LongHistogram, RepositoriesMetrics.METRIC_EXCEPTIONS_HISTOGRAM, expectedExceptions); + } else { + assertNoMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_EXCEPTIONS_TOTAL); + assertNoMetricRecorded(MetricType.LongHistogram, RepositoriesMetrics.METRIC_EXCEPTIONS_HISTOGRAM); + } + + if (result == Result.RangeNotSatisfied || result == Result.Failure || result == Result.Exception) { + assertIntMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_UNSUCCESSFUL_OPERATIONS_TOTAL, 1); + } else { + assertNoMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_UNSUCCESSFUL_OPERATIONS_TOTAL); + } + + if (result == Result.RangeNotSatisfied) { + assertIntMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_EXCEPTIONS_REQUEST_RANGE_NOT_SATISFIED_TOTAL, 1); + } else { + assertNoMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_EXCEPTIONS_REQUEST_RANGE_NOT_SATISFIED_TOTAL); + } + + assertMatchingMetricRecorded( + MetricType.LongHistogram, + RepositoriesMetrics.HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM, + m -> assertThat("No request time metric found", m.getLong(), greaterThanOrEqualTo(0L)) + ); + } + + private void assertIntMetricRecorded(MetricType metricType, String metricName, int expectedValue) { + assertMatchingMetricRecorded( + metricType, + metricName, + measurement -> assertEquals("Unexpected value for " + metricType + " " + metricName, expectedValue, measurement.getLong()) + ); + } + + private void assertNoMetricRecorded(MetricType metricType, String metricName) { + assertThat( + "Expected no values for " + metricType + " " + metricName, + metricType.getMeasurements(getTelemetryPlugin(dataNodeName), metricName), + hasSize(0) + ); + } + + private void assertMatchingMetricRecorded(MetricType metricType, String metricName, Consumer assertion) { + List measurements = metricType.getMeasurements(getTelemetryPlugin(dataNodeName), metricName); + Measurement measurement = measurements.stream() + .filter( + m -> m.attributes().get("operation").equals(operation.getKey()) + && m.attributes().get("purpose").equals(purpose.getKey()) + && m.attributes().get("repo_name").equals(repository) + && m.attributes().get("repo_type").equals("azure") + ) + .findFirst() + .orElseThrow( + () -> new IllegalStateException( + "No metric found with name=" + + metricName + + " and operation=" + + operation.getKey() + + " and purpose=" + + purpose.getKey() + + " and repo_name=" + + repository + + " in " + + measurements + ) + ); + + assertion.accept(measurement); + } + } + + @SuppressForbidden(reason = "we use a HttpServer to emulate Azure") + private static class ResponseInjectingAzureHttpHandler implements DelegatingHttpHandler { + + private final HttpHandler delegate; + private final Queue requestHandlerQueue; + + ResponseInjectingAzureHttpHandler(Queue requestHandlerQueue, HttpHandler delegate) { + this.delegate = delegate; + this.requestHandlerQueue = requestHandlerQueue; + } + + @Override + public void handle(HttpExchange exchange) throws IOException { + RequestHandler nextHandler = requestHandlerQueue.peek(); + if (nextHandler != null && nextHandler.matchesRequest(exchange)) { + requestHandlerQueue.poll().writeResponse(exchange, delegate); + } else { + delegate.handle(exchange); + } + } + + @Override + public HttpHandler getDelegate() { + return delegate; + } + } + + @SuppressForbidden(reason = "we use a HttpServer to emulate Azure") + @FunctionalInterface + private interface RequestHandler { + void writeResponse(HttpExchange exchange, HttpHandler delegate) throws IOException; + + default boolean matchesRequest(HttpExchange exchange) { + return true; + } + } + + @SuppressForbidden(reason = "we use a HttpServer to emulate Azure") + private static class FixedRequestHandler implements RequestHandler { + + private final RestStatus status; + private final String responseBody; + private final Predicate requestMatcher; + + FixedRequestHandler(RestStatus status) { + this(status, null, req -> true); + } + + /** + * Create a handler that only gets executed for requests that match the supplied predicate. Note + * that because the errors are stored in a queue this will prevent any subsequently queued errors from + * being returned until after it returns. + */ + FixedRequestHandler(RestStatus status, String responseBody, Predicate requestMatcher) { + this.status = status; + this.responseBody = responseBody; + this.requestMatcher = requestMatcher; + } + + @Override + public boolean matchesRequest(HttpExchange exchange) { + return requestMatcher.test(exchange); + } + + @Override + public void writeResponse(HttpExchange exchange, HttpHandler delegateHandler) throws IOException { + if (responseBody != null) { + byte[] responseBytes = responseBody.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(status.getStatus(), responseBytes.length); + exchange.getResponseBody().write(responseBytes); + } else { + exchange.sendResponseHeaders(status.getStatus(), -1); + } + } + } +} diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java index 1b7628cc0ad8e..473d91da6e34c 100644 --- a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java +++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java @@ -16,11 +16,13 @@ import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; +import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.MockSecureSettings; @@ -30,8 +32,15 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.repositories.RepositoryMissingException; +import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.blobstore.ESMockAPIBasedRepositoryIntegTestCase; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.telemetry.Measurement; +import org.elasticsearch.telemetry.TestTelemetryPlugin; +import org.elasticsearch.test.BackgroundIndexer; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -41,22 +50,33 @@ import java.util.Base64; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.LongAdder; import java.util.function.Predicate; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_OPERATIONS_TOTAL; import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.is; @SuppressForbidden(reason = "this test uses a HttpServer to emulate an Azure endpoint") public class AzureBlobStoreRepositoryTests extends ESMockAPIBasedRepositoryIntegTestCase { - private static final String DEFAULT_ACCOUNT_NAME = "account"; + protected static final String DEFAULT_ACCOUNT_NAME = "account"; + protected static final Predicate LIST_PATTERN = Pattern.compile("GET /[a-zA-Z0-9]+/[a-zA-Z0-9]+\\?.+").asMatchPredicate(); + protected static final Predicate GET_BLOB_PATTERN = Pattern.compile("GET /[a-zA-Z0-9]+/[a-zA-Z0-9]+/.+").asMatchPredicate(); @Override protected String repositoryType() { @@ -78,7 +98,7 @@ protected Settings repositorySettings(String repoName) { @Override protected Collection> nodePlugins() { - return Collections.singletonList(TestAzureRepositoryPlugin.class); + return List.of(TestAzureRepositoryPlugin.class, TestTelemetryPlugin.class); } @Override @@ -91,7 +111,7 @@ protected Map createHttpHandlers() { @Override protected HttpHandler createErroneousHttpHandler(final HttpHandler delegate) { - return new AzureErroneousHttpHandler(delegate, AzureStorageSettings.DEFAULT_MAX_RETRIES); + return new AzureHTTPStatsCollectorHandler(new AzureErroneousHttpHandler(delegate, AzureStorageSettings.DEFAULT_MAX_RETRIES)); } @Override @@ -119,6 +139,13 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .build(); } + protected TestTelemetryPlugin getTelemetryPlugin(String dataNodeName) { + return internalCluster().getInstance(PluginsService.class, dataNodeName) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + } + /** * AzureRepositoryPlugin that allows to set low values for the Azure's client retry policy * and for BlobRequestOptions#getSingleBlobPutThresholdInBytes(). @@ -195,9 +222,6 @@ protected String requestUniqueId(final HttpExchange exchange) { */ @SuppressForbidden(reason = "this test uses a HttpServer to emulate an Azure endpoint") private static class AzureHTTPStatsCollectorHandler extends HttpStatsCollectorHandler { - private static final Predicate LIST_PATTERN = Pattern.compile("GET /[a-zA-Z0-9]+/[a-zA-Z0-9]+\\?.+").asMatchPredicate(); - private static final Predicate GET_BLOB_PATTERN = Pattern.compile("GET /[a-zA-Z0-9]+/[a-zA-Z0-9]+/.+").asMatchPredicate(); - private final Set seenRequestIds = ConcurrentCollections.newConcurrentSet(); private AzureHTTPStatsCollectorHandler(HttpHandler delegate) { @@ -303,4 +327,87 @@ public void testReadByteByByte() throws Exception { container.delete(randomPurpose()); } } + + public void testMetrics() throws Exception { + // Reset all the metrics so there's none lingering from previous tests + internalCluster().getInstances(PluginsService.class) + .forEach(ps -> ps.filterPlugins(TestTelemetryPlugin.class).forEach(TestTelemetryPlugin::resetMeter)); + + // Create the repository and perform some activities + final String repository = createRepository(randomRepositoryName(), false); + final String index = "index-no-merges"; + createIndex(index, 1, 0); + + final long nbDocs = randomLongBetween(10_000L, 20_000L); + try (BackgroundIndexer indexer = new BackgroundIndexer(index, client(), (int) nbDocs)) { + waitForDocs(nbDocs, indexer); + } + flushAndRefresh(index); + BroadcastResponse forceMerge = client().admin().indices().prepareForceMerge(index).setFlush(true).setMaxNumSegments(1).get(); + assertThat(forceMerge.getSuccessfulShards(), equalTo(1)); + assertHitCount(prepareSearch(index).setSize(0).setTrackTotalHits(true), nbDocs); + + final String snapshot = "snapshot"; + assertSuccessfulSnapshot( + clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repository, snapshot).setWaitForCompletion(true).setIndices(index) + ); + assertAcked(client().admin().indices().prepareDelete(index)); + assertSuccessfulRestore( + clusterAdmin().prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, repository, snapshot).setWaitForCompletion(true) + ); + ensureGreen(index); + assertHitCount(prepareSearch(index).setSize(0).setTrackTotalHits(true), nbDocs); + assertAcked(clusterAdmin().prepareDeleteSnapshot(TEST_REQUEST_TIMEOUT, repository, snapshot).get()); + + final Map aggregatedMetrics = new HashMap<>(); + // Compare collected stats and metrics for each node and they should be the same + for (var nodeName : internalCluster().getNodeNames()) { + final BlobStoreRepository blobStoreRepository; + try { + blobStoreRepository = (BlobStoreRepository) internalCluster().getInstance(RepositoriesService.class, nodeName) + .repository(repository); + } catch (RepositoryMissingException e) { + continue; + } + + final AzureBlobStore blobStore = (AzureBlobStore) blobStoreRepository.blobStore(); + final Map statsCollectors = blobStore.getMetricsRecorder().opsCounters; + + final List metrics = Measurement.combine( + getTelemetryPlugin(nodeName).getLongCounterMeasurement(METRIC_OPERATIONS_TOTAL) + ); + + assertThat( + statsCollectors.keySet().stream().map(AzureBlobStore.StatsKey::operation).collect(Collectors.toSet()), + equalTo( + metrics.stream() + .map(m -> AzureBlobStore.Operation.fromKey((String) m.attributes().get("operation"))) + .collect(Collectors.toSet()) + ) + ); + metrics.forEach(metric -> { + assertThat( + metric.attributes(), + allOf(hasEntry("repo_type", AzureRepository.TYPE), hasKey("repo_name"), hasKey("operation"), hasKey("purpose")) + ); + final AzureBlobStore.Operation operation = AzureBlobStore.Operation.fromKey((String) metric.attributes().get("operation")); + final AzureBlobStore.StatsKey statsKey = new AzureBlobStore.StatsKey( + operation, + OperationPurpose.parse((String) metric.attributes().get("purpose")) + ); + assertThat(nodeName + "/" + statsKey + " exists", statsCollectors, hasKey(statsKey)); + assertThat(nodeName + "/" + statsKey + " has correct sum", metric.getLong(), equalTo(statsCollectors.get(statsKey).sum())); + aggregatedMetrics.compute(statsKey.operation(), (k, v) -> v == null ? metric.getLong() : v + metric.getLong()); + }); + } + + // Metrics number should be consistent with server side request count as well. + assertThat(aggregatedMetrics, equalTo(getServerMetrics())); + } + + private Map getServerMetrics() { + return getMockRequestCounts().entrySet() + .stream() + .collect(Collectors.toMap(e -> AzureBlobStore.Operation.fromKey(e.getKey()), Map.Entry::getValue)); + } } diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java index 5466989082129..d520d30f2bac6 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java @@ -60,6 +60,7 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; +import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.azure.AzureRepository.Repository; import org.elasticsearch.repositories.blobstore.ChunkedBlobOutputStream; import org.elasticsearch.rest.RestStatus; @@ -86,11 +87,11 @@ import java.util.Spliterator; import java.util.Spliterators; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.LongAdder; import java.util.function.BiPredicate; -import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -102,59 +103,54 @@ public class AzureBlobStore implements BlobStore { private static final int DEFAULT_UPLOAD_BUFFERS_SIZE = (int) new ByteSizeValue(64, ByteSizeUnit.KB).getBytes(); private final AzureStorageService service; - private final BigArrays bigArrays; + private final RepositoryMetadata repositoryMetadata; private final String clientName; private final String container; private final LocationMode locationMode; private final ByteSizeValue maxSinglePartUploadSize; - private final StatsCollectors statsCollectors = new StatsCollectors(); - private final AzureClientProvider.SuccessfulRequestHandler statsConsumer; + private final RequestMetricsRecorder requestMetricsRecorder; + private final AzureClientProvider.RequestMetricsHandler requestMetricsHandler; - public AzureBlobStore(RepositoryMetadata metadata, AzureStorageService service, BigArrays bigArrays) { + public AzureBlobStore( + RepositoryMetadata metadata, + AzureStorageService service, + BigArrays bigArrays, + RepositoriesMetrics repositoriesMetrics + ) { this.container = Repository.CONTAINER_SETTING.get(metadata.settings()); this.clientName = Repository.CLIENT_NAME.get(metadata.settings()); this.service = service; this.bigArrays = bigArrays; + this.requestMetricsRecorder = new RequestMetricsRecorder(repositoriesMetrics); + this.repositoryMetadata = metadata; // locationMode is set per repository, not per client this.locationMode = Repository.LOCATION_MODE_SETTING.get(metadata.settings()); this.maxSinglePartUploadSize = Repository.MAX_SINGLE_PART_UPLOAD_SIZE_SETTING.get(metadata.settings()); - List requestStatsCollectors = List.of( - RequestStatsCollector.create( - (httpMethod, url) -> httpMethod == HttpMethod.HEAD, - purpose -> statsCollectors.onSuccessfulRequest(Operation.GET_BLOB_PROPERTIES, purpose) - ), - RequestStatsCollector.create( + List requestMatchers = List.of( + new RequestMatcher((httpMethod, url) -> httpMethod == HttpMethod.HEAD, Operation.GET_BLOB_PROPERTIES), + new RequestMatcher( (httpMethod, url) -> httpMethod == HttpMethod.GET && isListRequest(httpMethod, url) == false, - purpose -> statsCollectors.onSuccessfulRequest(Operation.GET_BLOB, purpose) - ), - RequestStatsCollector.create( - AzureBlobStore::isListRequest, - purpose -> statsCollectors.onSuccessfulRequest(Operation.LIST_BLOBS, purpose) - ), - RequestStatsCollector.create( - AzureBlobStore::isPutBlockRequest, - purpose -> statsCollectors.onSuccessfulRequest(Operation.PUT_BLOCK, purpose) + Operation.GET_BLOB ), - RequestStatsCollector.create( - AzureBlobStore::isPutBlockListRequest, - purpose -> statsCollectors.onSuccessfulRequest(Operation.PUT_BLOCK_LIST, purpose) - ), - RequestStatsCollector.create( + new RequestMatcher(AzureBlobStore::isListRequest, Operation.LIST_BLOBS), + new RequestMatcher(AzureBlobStore::isPutBlockRequest, Operation.PUT_BLOCK), + new RequestMatcher(AzureBlobStore::isPutBlockListRequest, Operation.PUT_BLOCK_LIST), + new RequestMatcher( // https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob#uri-parameters // The only URI parameter allowed for put-blob operation is "timeout", but if a sas token is used, // it's possible that the URI parameters contain additional parameters unrelated to the upload type. (httpMethod, url) -> httpMethod == HttpMethod.PUT && isPutBlockRequest(httpMethod, url) == false && isPutBlockListRequest(httpMethod, url) == false, - purpose -> statsCollectors.onSuccessfulRequest(Operation.PUT_BLOB, purpose) + Operation.PUT_BLOB ) ); - this.statsConsumer = (purpose, httpMethod, url) -> { + this.requestMetricsHandler = (purpose, method, url, metrics) -> { try { URI uri = url.toURI(); String path = uri.getPath() == null ? "" : uri.getPath(); @@ -167,9 +163,9 @@ && isPutBlockListRequest(httpMethod, url) == false, return; } - for (RequestStatsCollector requestStatsCollector : requestStatsCollectors) { - if (requestStatsCollector.shouldConsumeRequestInfo(httpMethod, url)) { - requestStatsCollector.consumeHttpRequestInfo(purpose); + for (RequestMatcher requestMatcher : requestMatchers) { + if (requestMatcher.matches(method, url)) { + requestMetricsRecorder.onRequestComplete(requestMatcher.operation, purpose, metrics); return; } } @@ -665,12 +661,12 @@ private BlobServiceAsyncClient asyncClient(OperationPurpose purpose) { } private AzureBlobServiceClient getAzureBlobServiceClientClient(OperationPurpose purpose) { - return service.client(clientName, locationMode, purpose, statsConsumer); + return service.client(clientName, locationMode, purpose, requestMetricsHandler); } @Override public Map stats() { - return statsCollectors.statsMap(service.isStateless()); + return requestMetricsRecorder.statsMap(service.isStateless()); } // visible for testing @@ -691,26 +687,43 @@ public String getKey() { Operation(String key) { this.key = key; } + + public static Operation fromKey(String key) { + for (Operation operation : Operation.values()) { + if (operation.key.equals(key)) { + return operation; + } + } + throw new IllegalArgumentException("No matching key: " + key); + } } - private record StatsKey(Operation operation, OperationPurpose purpose) { + // visible for testing + record StatsKey(Operation operation, OperationPurpose purpose) { @Override public String toString() { return purpose.getKey() + "_" + operation.getKey(); } } - private static class StatsCollectors { - final Map collectors = new ConcurrentHashMap<>(); + // visible for testing + class RequestMetricsRecorder { + private final RepositoriesMetrics repositoriesMetrics; + final Map opsCounters = new ConcurrentHashMap<>(); + final Map> opsAttributes = new ConcurrentHashMap<>(); + + RequestMetricsRecorder(RepositoriesMetrics repositoriesMetrics) { + this.repositoriesMetrics = repositoriesMetrics; + } Map statsMap(boolean stateless) { if (stateless) { - return collectors.entrySet() + return opsCounters.entrySet() .stream() .collect(Collectors.toUnmodifiableMap(e -> e.getKey().toString(), e -> e.getValue().sum())); } else { Map normalisedStats = Arrays.stream(Operation.values()).collect(Collectors.toMap(Operation::getKey, o -> 0L)); - collectors.forEach( + opsCounters.forEach( (key, value) -> normalisedStats.compute( key.operation.getKey(), (k, current) -> Objects.requireNonNull(current) + value.sum() @@ -720,11 +733,50 @@ Map statsMap(boolean stateless) { } } - public void onSuccessfulRequest(Operation operation, OperationPurpose purpose) { - collectors.computeIfAbsent(new StatsKey(operation, purpose), k -> new LongAdder()).increment(); + public void onRequestComplete(Operation operation, OperationPurpose purpose, AzureClientProvider.RequestMetrics requestMetrics) { + final StatsKey statsKey = new StatsKey(operation, purpose); + final LongAdder counter = opsCounters.computeIfAbsent(statsKey, k -> new LongAdder()); + final Map attributes = opsAttributes.computeIfAbsent( + statsKey, + k -> RepositoriesMetrics.createAttributesMap(repositoryMetadata, purpose, operation.getKey()) + ); + + counter.add(1); + + // range not satisfied is not retried, so we count them by checking the final response + if (requestMetrics.getStatusCode() == RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus()) { + repositoriesMetrics.requestRangeNotSatisfiedExceptionCounter().incrementBy(1, attributes); + } + + repositoriesMetrics.operationCounter().incrementBy(1, attributes); + if (RestStatus.isSuccessful(requestMetrics.getStatusCode()) == false) { + repositoriesMetrics.unsuccessfulOperationCounter().incrementBy(1, attributes); + } + + repositoriesMetrics.requestCounter().incrementBy(requestMetrics.getRequestCount(), attributes); + if (requestMetrics.getErrorCount() > 0) { + repositoriesMetrics.exceptionCounter().incrementBy(requestMetrics.getErrorCount(), attributes); + repositoriesMetrics.exceptionHistogram().record(requestMetrics.getErrorCount(), attributes); + } + + if (requestMetrics.getThrottleCount() > 0) { + repositoriesMetrics.throttleCounter().incrementBy(requestMetrics.getThrottleCount(), attributes); + repositoriesMetrics.throttleHistogram().record(requestMetrics.getThrottleCount(), attributes); + } + + // We use nanosecond precision, so a zero value indicates that no requests were executed + if (requestMetrics.getTotalRequestTimeNanos() > 0) { + repositoriesMetrics.httpRequestTimeInMillisHistogram() + .record(TimeUnit.NANOSECONDS.toMillis(requestMetrics.getTotalRequestTimeNanos()), attributes); + } } } + // visible for testing + RequestMetricsRecorder getMetricsRecorder() { + return requestMetricsRecorder; + } + private static class AzureInputStream extends InputStream { private final CancellableRateLimitedFluxIterator cancellableRateLimitedFluxIterator; private ByteBuf byteBuf; @@ -846,26 +898,11 @@ private ByteBuf getNextByteBuf() throws IOException { } } - private static class RequestStatsCollector { - private final BiPredicate filter; - private final Consumer onHttpRequest; - - private RequestStatsCollector(BiPredicate filter, Consumer onHttpRequest) { - this.filter = filter; - this.onHttpRequest = onHttpRequest; - } - - static RequestStatsCollector create(BiPredicate filter, Consumer consumer) { - return new RequestStatsCollector(filter, consumer); - } + private record RequestMatcher(BiPredicate filter, Operation operation) { - private boolean shouldConsumeRequestInfo(HttpMethod httpMethod, URL url) { + private boolean matches(HttpMethod httpMethod, URL url) { return filter.test(httpMethod, url); } - - private void consumeHttpRequestInfo(OperationPurpose operationPurpose) { - onHttpRequest.accept(operationPurpose); - } } OptionalBytesReference getRegister(OperationPurpose purpose, String blobPath, String containerPath, String blobKey) { diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java index ae497ff159576..654742c980268 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java @@ -24,6 +24,7 @@ import com.azure.core.http.HttpMethod; import com.azure.core.http.HttpPipelineCallContext; import com.azure.core.http.HttpPipelineNextPolicy; +import com.azure.core.http.HttpPipelinePosition; import com.azure.core.http.HttpRequest; import com.azure.core.http.HttpResponse; import com.azure.core.http.ProxyOptions; @@ -44,11 +45,13 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.repositories.azure.executors.PrivilegedExecutor; import org.elasticsearch.repositories.azure.executors.ReactorScheduledExecutorService; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.netty4.NettyAllocator; import java.net.URL; import java.time.Duration; +import java.util.Optional; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadFactory; @@ -57,6 +60,8 @@ import static org.elasticsearch.repositories.azure.AzureRepositoryPlugin.REPOSITORY_THREAD_POOL_NAME; class AzureClientProvider extends AbstractLifecycleComponent { + private static final Logger logger = LogManager.getLogger(AzureClientProvider.class); + private static final TimeValue DEFAULT_CONNECTION_TIMEOUT = TimeValue.timeValueSeconds(30); private static final TimeValue DEFAULT_MAX_CONNECTION_IDLE_TIME = TimeValue.timeValueSeconds(60); private static final int DEFAULT_MAX_CONNECTIONS = 50; @@ -160,7 +165,7 @@ AzureBlobServiceClient createClient( LocationMode locationMode, RequestRetryOptions retryOptions, ProxyOptions proxyOptions, - SuccessfulRequestHandler successfulRequestHandler, + RequestMetricsHandler requestMetricsHandler, OperationPurpose purpose ) { if (closed) { @@ -189,8 +194,9 @@ AzureBlobServiceClient createClient( builder.credential(credentialBuilder.build()); } - if (successfulRequestHandler != null) { - builder.addPolicy(new SuccessfulRequestTracker(purpose, successfulRequestHandler)); + if (requestMetricsHandler != null) { + builder.addPolicy(new RequestMetricsTracker(purpose, requestMetricsHandler)); + builder.addPolicy(RetryMetricsTracker.INSTANCE); } if (locationMode.isSecondary()) { @@ -259,38 +265,135 @@ protected void doStop() { @Override protected void doClose() {} - private static final class SuccessfulRequestTracker implements HttpPipelinePolicy { - private static final Logger logger = LogManager.getLogger(SuccessfulRequestTracker.class); + static class RequestMetrics { + private volatile long totalRequestTimeNanos = 0; + private volatile int requestCount; + private volatile int errorCount; + private volatile int throttleCount; + private volatile int statusCode; + + int getRequestCount() { + return requestCount; + } + + int getErrorCount() { + return errorCount; + } + + int getStatusCode() { + return statusCode; + } + + int getThrottleCount() { + return throttleCount; + } + + /** + * Total time spent executing requests to complete operation in nanoseconds + */ + long getTotalRequestTimeNanos() { + return totalRequestTimeNanos; + } + + @Override + public String toString() { + return "RequestMetrics{" + + "totalRequestTimeNanos=" + + totalRequestTimeNanos + + ", requestCount=" + + requestCount + + ", errorCount=" + + errorCount + + ", throttleCount=" + + throttleCount + + ", statusCode=" + + statusCode + + '}'; + } + } + + private enum RetryMetricsTracker implements HttpPipelinePolicy { + INSTANCE; + + @Override + public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { + Optional metricsData = context.getData(RequestMetricsTracker.ES_REQUEST_METRICS_CONTEXT_KEY); + if (metricsData.isPresent() == false) { + assert false : "No metrics object associated with request " + context.getHttpRequest(); + return next.process(); + } + RequestMetrics metrics = (RequestMetrics) metricsData.get(); + metrics.requestCount++; + long requestStartTimeNanos = System.nanoTime(); + return next.process().doOnError(throwable -> { + metrics.totalRequestTimeNanos += System.nanoTime() - requestStartTimeNanos; + logger.debug("Detected error in RetryMetricsTracker", throwable); + metrics.errorCount++; + }).doOnSuccess(response -> { + metrics.totalRequestTimeNanos += System.nanoTime() - requestStartTimeNanos; + if (RestStatus.isSuccessful(response.getStatusCode()) == false) { + metrics.errorCount++; + // Azure always throttles with a 429 response, see + // https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/request-limits-and-throttling#error-code + if (response.getStatusCode() == RestStatus.TOO_MANY_REQUESTS.getStatus()) { + metrics.throttleCount++; + } + } + }); + } + + @Override + public HttpPipelinePosition getPipelinePosition() { + return HttpPipelinePosition.PER_RETRY; + } + } + + private static final class RequestMetricsTracker implements HttpPipelinePolicy { + private static final String ES_REQUEST_METRICS_CONTEXT_KEY = "_es_azure_repo_request_stats"; + private static final Logger logger = LogManager.getLogger(RequestMetricsTracker.class); private final OperationPurpose purpose; - private final SuccessfulRequestHandler onSuccessfulRequest; + private final RequestMetricsHandler requestMetricsHandler; - private SuccessfulRequestTracker(OperationPurpose purpose, SuccessfulRequestHandler onSuccessfulRequest) { + private RequestMetricsTracker(OperationPurpose purpose, RequestMetricsHandler requestMetricsHandler) { this.purpose = purpose; - this.onSuccessfulRequest = onSuccessfulRequest; + this.requestMetricsHandler = requestMetricsHandler; } @Override public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { - return next.process().doOnSuccess(httpResponse -> trackSuccessfulRequest(context.getHttpRequest(), httpResponse)); + final RequestMetrics requestMetrics = new RequestMetrics(); + context.setData(ES_REQUEST_METRICS_CONTEXT_KEY, requestMetrics); + return next.process().doOnSuccess((httpResponse) -> { + requestMetrics.statusCode = httpResponse.getStatusCode(); + trackCompletedRequest(context.getHttpRequest(), requestMetrics); + }).doOnError(throwable -> { + logger.debug("Detected error in RequestMetricsTracker", throwable); + trackCompletedRequest(context.getHttpRequest(), requestMetrics); + }); } - private void trackSuccessfulRequest(HttpRequest httpRequest, HttpResponse httpResponse) { + private void trackCompletedRequest(HttpRequest httpRequest, RequestMetrics requestMetrics) { HttpMethod method = httpRequest.getHttpMethod(); - if (httpResponse != null && method != null && httpResponse.getStatusCode() > 199 && httpResponse.getStatusCode() <= 299) { + if (method != null) { try { - onSuccessfulRequest.onSuccessfulRequest(purpose, method, httpRequest.getUrl()); + requestMetricsHandler.requestCompleted(purpose, method, httpRequest.getUrl(), requestMetrics); } catch (Exception e) { logger.warn("Unable to notify a successful request", e); } } } + + @Override + public HttpPipelinePosition getPipelinePosition() { + return HttpPipelinePosition.PER_CALL; + } } /** - * The {@link SuccessfulRequestTracker} calls this when a request completes successfully + * The {@link RequestMetricsTracker} calls this when a request completes */ - interface SuccessfulRequestHandler { + interface RequestMetricsHandler { - void onSuccessfulRequest(OperationPurpose purpose, HttpMethod method, URL url); + void requestCompleted(OperationPurpose purpose, HttpMethod method, URL url, RequestMetrics metrics); } } diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index aec148adf9aa8..80e662343baee 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.indices.recovery.RecoverySettings; +import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -91,6 +92,7 @@ public static final class Repository { private final ByteSizeValue chunkSize; private final AzureStorageService storageService; private final boolean readonly; + private final RepositoriesMetrics repositoriesMetrics; public AzureRepository( final RepositoryMetadata metadata, @@ -98,7 +100,8 @@ public AzureRepository( final AzureStorageService storageService, final ClusterService clusterService, final BigArrays bigArrays, - final RecoverySettings recoverySettings + final RecoverySettings recoverySettings, + final RepositoriesMetrics repositoriesMetrics ) { super( metadata, @@ -111,6 +114,7 @@ public AzureRepository( ); this.chunkSize = Repository.CHUNK_SIZE_SETTING.get(metadata.settings()); this.storageService = storageService; + this.repositoriesMetrics = repositoriesMetrics; // If the user explicitly did not define a readonly value, we set it by ourselves depending on the location mode setting. // For secondary_only setting, the repository should be read only @@ -152,7 +156,7 @@ protected BlobStore getBlobStore() { @Override protected AzureBlobStore createBlobStore() { - final AzureBlobStore blobStore = new AzureBlobStore(metadata, storageService, bigArrays); + final AzureBlobStore blobStore = new AzureBlobStore(metadata, storageService, bigArrays, repositoriesMetrics); logger.debug( () -> format( diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java index c3cd5e78c5dbe..4556e63378fea 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java @@ -71,7 +71,15 @@ public Map getRepositories( return Collections.singletonMap(AzureRepository.TYPE, metadata -> { AzureStorageService storageService = azureStoreService.get(); assert storageService != null; - return new AzureRepository(metadata, namedXContentRegistry, storageService, clusterService, bigArrays, recoverySettings); + return new AzureRepository( + metadata, + namedXContentRegistry, + storageService, + clusterService, + bigArrays, + recoverySettings, + repositoriesMetrics + ); }); } diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java index c6e85e44d24dd..7373ed9485784 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java @@ -91,7 +91,7 @@ public AzureBlobServiceClient client( String clientName, LocationMode locationMode, OperationPurpose purpose, - AzureClientProvider.SuccessfulRequestHandler successfulRequestHandler + AzureClientProvider.RequestMetricsHandler requestMetricsHandler ) { final AzureStorageSettings azureStorageSettings = getClientSettings(clientName); @@ -102,7 +102,7 @@ public AzureBlobServiceClient client( locationMode, retryOptions, proxyOptions, - successfulRequestHandler, + requestMetricsHandler, purpose ); } diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java index 1962bddd8fdb3..cb9facc061a28 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java @@ -29,6 +29,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.mocksocket.MockHttpServer; +import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -168,7 +169,10 @@ int getMaxReadRetries(String clientName) { .build() ); - return new AzureBlobContainer(BlobPath.EMPTY, new AzureBlobStore(repositoryMetadata, service, BigArrays.NON_RECYCLING_INSTANCE)); + return new AzureBlobContainer( + BlobPath.EMPTY, + new AzureBlobStore(repositoryMetadata, service, BigArrays.NON_RECYCLING_INSTANCE, RepositoriesMetrics.NOOP) + ); } protected static byte[] randomBlobContent() { diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java index 7d82f2d5029f6..2699438de8ac6 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java @@ -26,7 +26,7 @@ import java.util.concurrent.TimeUnit; public class AzureClientProviderTests extends ESTestCase { - private static final AzureClientProvider.SuccessfulRequestHandler EMPTY_CONSUMER = (purpose, method, url) -> {}; + private static final AzureClientProvider.RequestMetricsHandler NOOP_HANDLER = (purpose, method, url, metrics) -> {}; private ThreadPool threadPool; private AzureClientProvider azureClientProvider; @@ -76,7 +76,7 @@ public void testCanCreateAClientWithSecondaryLocation() { locationMode, requestRetryOptions, null, - EMPTY_CONSUMER, + NOOP_HANDLER, randomFrom(OperationPurpose.values()) ); } @@ -106,7 +106,7 @@ public void testCanNotCreateAClientWithSecondaryLocationWithoutAProperEndpoint() locationMode, requestRetryOptions, null, - EMPTY_CONSUMER, + NOOP_HANDLER, randomFrom(OperationPurpose.values()) ) ); diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java index 7037dd4eaf111..3afacb5b7426e 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.util.MockBigArrays; import org.elasticsearch.env.Environment; import org.elasticsearch.indices.recovery.RecoverySettings; +import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -40,7 +41,8 @@ private AzureRepository azureRepository(Settings settings) { mock(AzureStorageService.class), BlobStoreTestUtil.mockClusterService(), MockBigArrays.NON_RECYCLING_INSTANCE, - new RecoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + new RecoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), + RepositoriesMetrics.NOOP ); assertThat(azureRepository.getBlobStore(), is(nullValue())); return azureRepository; diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index 429a81b02bd5e..6b4dd5ed86e2d 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -235,7 +235,6 @@ public void testAbortRequestStats() throws Exception { } @TestIssueLogging(issueUrl = "https://github.com/elastic/elasticsearch/issues/101608", value = "com.amazonaws.request:DEBUG") - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/101608") public void testMetrics() throws Exception { // Create the repository and perform some activities final String repository = createRepository(randomRepositoryName(), false); diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index bd5723b4dbcc4..3e6b7c356cb11 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; @@ -144,16 +145,7 @@ class IgnoreNoResponseMetricsCollector extends RequestMetricCollector { private IgnoreNoResponseMetricsCollector(Operation operation, OperationPurpose purpose) { this.operation = operation; - this.attributes = Map.of( - "repo_type", - S3Repository.TYPE, - "repo_name", - repositoryMetadata.name(), - "operation", - operation.getKey(), - "purpose", - purpose.getKey() - ); + this.attributes = RepositoriesMetrics.createAttributesMap(repositoryMetadata, purpose, operation.getKey()); } @Override diff --git a/muted-tests.yml b/muted-tests.yml index e83f15d445c93..93893d7103afb 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -109,9 +109,6 @@ tests: - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testDeleteJobAsync issue: https://github.com/elastic/elasticsearch/issues/112212 -- class: org.elasticsearch.search.retriever.RankDocRetrieverBuilderIT - method: testRankDocsRetrieverWithCollapse - issue: https://github.com/elastic/elasticsearch/issues/112254 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/rest-api/watcher/put-watch/line_120} issue: https://github.com/elastic/elasticsearch/issues/99517 @@ -336,21 +333,43 @@ tests: - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT method: testPutE5Small_withPlatformAgnosticVariant issue: https://github.com/elastic/elasticsearch/issues/113983 -- class: org.elasticsearch.xpack.rank.rrf.RRFRankClientYamlTestSuiteIT - method: test {yaml=rrf/700_rrf_retriever_search_api_compatibility/rrf retriever with top-level collapse} - issue: https://github.com/elastic/elasticsearch/issues/114019 - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT method: testPutE5WithTrainedModelAndInference issue: https://github.com/elastic/elasticsearch/issues/114023 -- class: org.elasticsearch.xpack.rank.rrf.RRFRetrieverBuilderIT - method: testRRFWithCollapse - issue: https://github.com/elastic/elasticsearch/issues/114074 - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT method: testPutE5Small_withPlatformSpecificVariant issue: https://github.com/elastic/elasticsearch/issues/113950 - class: org.elasticsearch.xpack.inference.services.openai.OpenAiServiceTests method: testInfer_StreamRequest_ErrorResponse issue: https://github.com/elastic/elasticsearch/issues/114105 +- class: org.elasticsearch.xpack.inference.InferenceCrudIT + method: testGet + issue: https://github.com/elastic/elasticsearch/issues/114135 +- class: org.elasticsearch.xpack.esql.expression.function.aggregate.AvgTests + method: "testFold {TestCase= #7}" + issue: https://github.com/elastic/elasticsearch/issues/114175 +- class: org.elasticsearch.action.bulk.IncrementalBulkIT + method: testMultipleBulkPartsWithBackoff + issue: https://github.com/elastic/elasticsearch/issues/114181 +- class: org.elasticsearch.action.bulk.IncrementalBulkIT + method: testIncrementalBulkLowWatermarkBackOff + issue: https://github.com/elastic/elasticsearch/issues/114182 +- class: org.elasticsearch.aggregations.AggregationsClientYamlTestSuiteIT + method: test {yaml=aggregations/stats_metric_fail_formatting/fail formatting} + issue: https://github.com/elastic/elasticsearch/issues/114187 +- class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT + issue: https://github.com/elastic/elasticsearch/issues/114194 +- class: org.elasticsearch.xpack.ilm.ExplainLifecycleIT + method: testStepInfoPreservedOnAutoRetry + issue: https://github.com/elastic/elasticsearch/issues/114220 +- class: org.elasticsearch.xpack.inference.services.openai.OpenAiServiceTests + method: testInfer_StreamRequest + issue: https://github.com/elastic/elasticsearch/issues/114232 +- class: org.elasticsearch.logsdb.datageneration.DataGeneratorTests + method: testDataGeneratorProducesValidMappingAndDocument + issue: https://github.com/elastic/elasticsearch/issues/114188 +- class: org.elasticsearch.ingest.geoip.IpinfoIpDataLookupsTests + issue: https://github.com/elastic/elasticsearch/issues/114266 # Examples: # diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java index aa669a45bc0c7..78ea619d81f84 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java @@ -101,6 +101,7 @@ public void testEC2DiscoveryRetriesOnRateLimiting() throws IOException { exchange.getResponseHeaders().set("Content-Type", "text/xml; charset=UTF-8"); exchange.sendResponseHeaders(HttpStatus.SC_OK, responseBody.length); exchange.getResponseBody().write(responseBody); + exchange.getResponseBody().flush(); return; } } diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java index 8a6f6b84fec0d..135ddcee8da44 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java @@ -59,7 +59,7 @@ public class Ec2DiscoveryTests extends AbstractEC2MockAPITestCase { private static final String PREFIX_PUBLIC_IP = "8.8.8."; private static final String PREFIX_PRIVATE_IP = "10.0.0."; - private Map poorMansDNS = new ConcurrentHashMap<>(); + private final Map poorMansDNS = new ConcurrentHashMap<>(); protected MockTransportService createTransportService() { final Transport transport = new Netty4Transport( @@ -133,7 +133,7 @@ protected List buildDynamicHosts(Settings nodeSettings, int no .stream() .filter(t -> t.getKey().equals(entry.getKey())) .map(Tag::getValue) - .collect(Collectors.toList()) + .toList() .containsAll(entry.getValue()) ) ) @@ -144,6 +144,7 @@ protected List buildDynamicHosts(Settings nodeSettings, int no exchange.getResponseHeaders().set("Content-Type", "text/xml; charset=UTF-8"); exchange.sendResponseHeaders(HttpStatus.SC_OK, responseBody.length); exchange.getResponseBody().write(responseBody); + exchange.getResponseBody().flush(); return; } } @@ -160,14 +161,14 @@ protected List buildDynamicHosts(Settings nodeSettings, int no } } - public void testDefaultSettings() throws InterruptedException { + public void testDefaultSettings() { int nodes = randomInt(10); Settings nodeSettings = Settings.builder().build(); List discoveryNodes = buildDynamicHosts(nodeSettings, nodes); assertThat(discoveryNodes, hasSize(nodes)); } - public void testPrivateIp() throws InterruptedException { + public void testPrivateIp() { int nodes = randomInt(10); for (int i = 0; i < nodes; i++) { poorMansDNS.put(PREFIX_PRIVATE_IP + (i + 1), buildNewFakeTransportAddress()); @@ -183,7 +184,7 @@ public void testPrivateIp() throws InterruptedException { } } - public void testPublicIp() throws InterruptedException { + public void testPublicIp() { int nodes = randomInt(10); for (int i = 0; i < nodes; i++) { poorMansDNS.put(PREFIX_PUBLIC_IP + (i + 1), buildNewFakeTransportAddress()); @@ -199,7 +200,7 @@ public void testPublicIp() throws InterruptedException { } } - public void testPrivateDns() throws InterruptedException { + public void testPrivateDns() { int nodes = randomInt(10); for (int i = 0; i < nodes; i++) { String instanceId = "node" + (i + 1); @@ -217,7 +218,7 @@ public void testPrivateDns() throws InterruptedException { } } - public void testPublicDns() throws InterruptedException { + public void testPublicDns() { int nodes = randomInt(10); for (int i = 0; i < nodes; i++) { String instanceId = "node" + (i + 1); @@ -235,14 +236,14 @@ public void testPublicDns() throws InterruptedException { } } - public void testInvalidHostType() throws InterruptedException { + public void testInvalidHostType() { Settings nodeSettings = Settings.builder().put(AwsEc2Service.HOST_TYPE_SETTING.getKey(), "does_not_exist").build(); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { buildDynamicHosts(nodeSettings, 1); }); assertThat(exception.getMessage(), containsString("does_not_exist is unknown for discovery.ec2.host_type")); } - public void testFilterByTags() throws InterruptedException { + public void testFilterByTags() { int nodes = randomIntBetween(5, 10); Settings nodeSettings = Settings.builder().put(AwsEc2Service.TAG_SETTING.getKey() + "stage", "prod").build(); @@ -265,7 +266,7 @@ public void testFilterByTags() throws InterruptedException { assertThat(dynamicHosts, hasSize(prodInstances)); } - public void testFilterByMultipleTags() throws InterruptedException { + public void testFilterByMultipleTags() { int nodes = randomIntBetween(5, 10); Settings nodeSettings = Settings.builder().putList(AwsEc2Service.TAG_SETTING.getKey() + "stage", "prod", "preprod").build(); diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 9255281e7e5da..8570662f7b523 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -1602,10 +1602,6 @@ public void testResize() throws Exception { @SuppressWarnings("unchecked") public void testSystemIndexMetadataIsUpgraded() throws Exception { - - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // assumeTrue can be removed (condition always true) - var originalClusterTaskIndexIsSystemIndex = oldClusterHasFeature(RestTestLegacyFeatures.TASK_INDEX_SYSTEM_INDEX); - assumeTrue(".tasks became a system index in 7.10.0", originalClusterTaskIndexIsSystemIndex); final String systemIndexWarning = "this request accesses system indices: [.tasks], but in a future major version, direct " + "access to system indices will be prevented by default"; if (isRunningAgainstOldCluster()) { @@ -1665,29 +1661,6 @@ public void testSystemIndexMetadataIsUpgraded() throws Exception { throw new AssertionError(".tasks index does not exist yet"); } }); - - // If we are on 7.x create an alias that includes both a system index and a non-system index so we can be sure it gets - // upgraded properly. If we're already on 8.x, skip this part of the test. - if (clusterHasFeature(RestTestLegacyFeatures.SYSTEM_INDICES_REST_ACCESS_ENFORCED) == false) { - // Create an alias to make sure it gets upgraded properly - Request putAliasRequest = newXContentRequest(HttpMethod.POST, "/_aliases", (builder, params) -> { - builder.startArray("actions"); - for (var index : List.of(".tasks", "test_index_reindex")) { - builder.startObject() - .startObject("add") - .field("index", index) - .field("alias", "test-system-alias") - .endObject() - .endObject(); - } - return builder.endArray(); - }); - putAliasRequest.setOptions(expectVersionSpecificWarnings(v -> { - v.current(systemIndexWarning); - v.compatible(systemIndexWarning); - })); - assertThat(client().performRequest(putAliasRequest).getStatusLine().getStatusCode(), is(200)); - } } else { assertBusy(() -> { Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata"); diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java index dd8ecdd82ca7b..6a526a6dbfded 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java @@ -15,7 +15,6 @@ import org.elasticsearch.client.ResponseException; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.XContentTestUtils.JsonMapView; -import org.elasticsearch.test.rest.RestTestLegacyFeatures; import java.util.Map; @@ -87,25 +86,6 @@ public void testSystemIndicesUpgrades() throws Exception { throw new AssertionError(".tasks index does not exist yet"); } }); - - // If we are on 7.x create an alias that includes both a system index and a non-system index so we can be sure it gets - // upgraded properly. If we're already on 8.x, skip this part of the test. - if (clusterHasFeature(RestTestLegacyFeatures.SYSTEM_INDICES_REST_ACCESS_ENFORCED) == false) { - // Create an alias to make sure it gets upgraded properly - Request putAliasRequest = new Request("POST", "/_aliases"); - putAliasRequest.setJsonEntity(""" - { - "actions": [ - {"add": {"index": ".tasks", "alias": "test-system-alias"}}, - {"add": {"index": "test_index_reindex", "alias": "test-system-alias"}} - ] - }"""); - putAliasRequest.setOptions(expectVersionSpecificWarnings(v -> { - v.current(systemIndexWarning); - v.compatible(systemIndexWarning); - })); - assertThat(client().performRequest(putAliasRequest).getStatusLine().getStatusCode(), is(200)); - } } else if (isUpgradedCluster()) { assertBusy(() -> { Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata"); diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index 9c6a1ca2e96d2..b4672b1d8924d 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -606,3 +606,204 @@ setup: - match: { docs.0.doc._source.foo: "FOO" } - match: { docs.0.doc.executed_pipelines: [] } - not_exists: docs.0.doc.error + +--- +"Test ingest simulate with component template substitutions for data streams": + # In this test, we make sure that when the index template is a data stream template, simulte ingest works the same whether the data stream + # has been created or not -- either way, we expect it to use the template rather than the data stream / index mappings and settings. + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.component.template.substitutions"] + reason: "ingest simulate component template substitutions added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: mappings_template + body: + template: + mappings: + dynamic: strict + properties: + foo: + type: keyword + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo-pipeline" + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - mappings_template + - settings_template + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + composed_of: + - mappings_template + - settings_template + data_stream: {} + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": false + } + } + ], + "pipeline_substitutions": { + "foo-pipeline-2": { + "processors": [ + { + "set": { + "field": "foo", + "value": "FOO" + } + } + ] + } + }, + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": "foo-pipeline-2" + } + } + } + }, + "mappings_template": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } + - not_exists: docs.0.doc.error + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + cluster.health: + wait_for_status: yellow + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": false + } + } + ], + "pipeline_substitutions": { + "foo-pipeline-2": { + "processors": [ + { + "set": { + "field": "foo", + "value": "FOO" + } + } + ] + } + }, + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": "foo-pipeline-2" + } + } + } + }, + "mappings_template": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } + - not_exists: docs.0.doc.error diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_inference.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_inference.json new file mode 100644 index 0000000000000..32b4b2f311837 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_inference.json @@ -0,0 +1,49 @@ +{ + "inference.stream_inference":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/post-stream-inference-api.html", + "description":"Perform streaming inference" + }, + "stability":"experimental", + "visibility":"public", + "headers":{ + "accept": [ "text/event-stream"], + "content_type": ["application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_inference/{inference_id}/_stream", + "methods":[ + "POST" + ], + "parts":{ + "inference_id":{ + "type":"string", + "description":"The inference Id" + } + } + }, + { + "path":"/_inference/{task_type}/{inference_id}/_stream", + "methods":[ + "POST" + ], + "parts":{ + "task_type":{ + "type":"string", + "description":"The task type" + }, + "inference_id":{ + "type":"string", + "description":"The inference Id" + } + } + } + ] + }, + "body":{ + "description":"The inference payload" + } + } +} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml index e2c3006232c53..a4a9b1aaecb22 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml @@ -9,6 +9,8 @@ flattened ignore_above single-value field: body: mappings: properties: + name: + type: keyword keyword: type: keyword ignore_above: 5 @@ -22,6 +24,7 @@ flattened ignore_above single-value field: id: "1" refresh: true body: + name: "A" keyword: "foo" flat: { "value": "foo", "key": "foo key" } @@ -31,12 +34,14 @@ flattened ignore_above single-value field: id: "2" refresh: true body: + name: "B" keyword: "foo bar" flat: { "value": "foo bar", "key": "foo bar key"} - do: search: index: test + sort: name body: fields: - keyword @@ -69,6 +74,8 @@ flattened ignore_above multi-value field: body: mappings: properties: + name: + type: keyword keyword: type: keyword ignore_above: 5 @@ -82,6 +89,7 @@ flattened ignore_above multi-value field: id: "1" refresh: true body: + name: "A" keyword: ["foo","bar"] flat: { "value": ["foo", "bar"], "key": "foo bar array key" } @@ -91,12 +99,14 @@ flattened ignore_above multi-value field: id: "2" refresh: true body: + name: "B" keyword: ["foobar", "foo", "bar"] flat: { "value": ["foobar", "foo"], "key": ["foo key", "bar key"]} - do: search: index: test + sort: name body: fields: - keyword diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java index 60ea4138e923d..cde8d41b292b7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java @@ -233,6 +233,9 @@ public void testIncrementalBulkHighWatermarkBackOff() throws Exception { handlers.add(handlerThrottled); + // Wait until we are ready for the next page + assertBusy(() -> assertTrue(nextPage.get())); + for (IncrementalBulkService.Handler h : handlers) { refCounted.incRef(); PlainActionFuture future = new PlainActionFuture<>(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java index b72257b884f08..4a060eadc735b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java @@ -9,23 +9,38 @@ package org.elasticsearch.monitor.metrics; +import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.mapper.OnScriptError; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.LongFieldScript; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.telemetry.Measurement; import org.elasticsearch.telemetry.TestTelemetryPlugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.json.JsonXContent; import org.hamcrest.Matcher; +import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; import static org.elasticsearch.index.mapper.DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0) @@ -42,7 +57,7 @@ public List> getSettings() { @Override protected Collection> nodePlugins() { - return List.of(TestTelemetryPlugin.class, TestAPMInternalSettings.class); + return List.of(TestTelemetryPlugin.class, TestAPMInternalSettings.class, FailingFieldPlugin.class); } @Override @@ -54,27 +69,57 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { } static final String STANDARD_INDEX_COUNT = "es.indices.standard.total"; + static final String STANDARD_BYTES_SIZE = "es.indices.standard.size"; static final String STANDARD_DOCS_COUNT = "es.indices.standard.docs.total"; - static final String STANDARD_BYTES_SIZE = "es.indices.standard.bytes.total"; + static final String STANDARD_QUERY_COUNT = "es.indices.standard.query.total"; + static final String STANDARD_QUERY_TIME = "es.indices.standard.query.time"; + static final String STANDARD_QUERY_FAILURE = "es.indices.standard.query.failure.total"; + static final String STANDARD_FETCH_COUNT = "es.indices.standard.fetch.total"; + static final String STANDARD_FETCH_TIME = "es.indices.standard.fetch.time"; + static final String STANDARD_FETCH_FAILURE = "es.indices.standard.fetch.failure.total"; + static final String STANDARD_INDEXING_COUNT = "es.indices.standard.indexing.total"; + static final String STANDARD_INDEXING_TIME = "es.indices.standard.indexing.time"; + static final String STANDARD_INDEXING_FAILURE = "es.indices.standard.indexing.failure.total"; static final String TIME_SERIES_INDEX_COUNT = "es.indices.time_series.total"; + static final String TIME_SERIES_BYTES_SIZE = "es.indices.time_series.size"; static final String TIME_SERIES_DOCS_COUNT = "es.indices.time_series.docs.total"; - static final String TIME_SERIES_BYTES_SIZE = "es.indices.time_series.bytes.total"; + static final String TIME_SERIES_QUERY_COUNT = "es.indices.time_series.query.total"; + static final String TIME_SERIES_QUERY_TIME = "es.indices.time_series.query.time"; + static final String TIME_SERIES_QUERY_FAILURE = "es.indices.time_series.query.failure.total"; + static final String TIME_SERIES_FETCH_COUNT = "es.indices.time_series.fetch.total"; + static final String TIME_SERIES_FETCH_TIME = "es.indices.time_series.fetch.time"; + static final String TIME_SERIES_FETCH_FAILURE = "es.indices.time_series.fetch.failure.total"; + static final String TIME_SERIES_INDEXING_COUNT = "es.indices.time_series.indexing.total"; + static final String TIME_SERIES_INDEXING_TIME = "es.indices.time_series.indexing.time"; + static final String TIME_SERIES_INDEXING_FAILURE = "es.indices.time_series.indexing.failure.total"; static final String LOGSDB_INDEX_COUNT = "es.indices.logsdb.total"; + static final String LOGSDB_BYTES_SIZE = "es.indices.logsdb.size"; static final String LOGSDB_DOCS_COUNT = "es.indices.logsdb.docs.total"; - static final String LOGSDB_BYTES_SIZE = "es.indices.logsdb.bytes.total"; + static final String LOGSDB_QUERY_COUNT = "es.indices.logsdb.query.total"; + static final String LOGSDB_QUERY_TIME = "es.indices.logsdb.query.time"; + static final String LOGSDB_QUERY_FAILURE = "es.indices.logsdb.query.failure.total"; + static final String LOGSDB_FETCH_COUNT = "es.indices.logsdb.fetch.total"; + static final String LOGSDB_FETCH_TIME = "es.indices.logsdb.fetch.time"; + static final String LOGSDB_FETCH_FAILURE = "es.indices.logsdb.fetch.failure.total"; + static final String LOGSDB_INDEXING_COUNT = "es.indices.logsdb.indexing.total"; + static final String LOGSDB_INDEXING_TIME = "es.indices.logsdb.indexing.time"; + static final String LOGSDB_INDEXING_FAILURE = "es.indices.logsdb.indexing.failure.total"; - public void testIndicesMetrics() { + public void testIndicesMetrics() throws Exception { String node = internalCluster().startNode(); ensureStableCluster(1); final TestTelemetryPlugin telemetry = internalCluster().getInstance(PluginsService.class, node) .filterPlugins(TestTelemetryPlugin.class) .findFirst() .orElseThrow(); + final IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); + var indexing0 = indicesService.stats(CommonStatsFlags.ALL, false).getIndexing().getTotal(); telemetry.resetMeter(); long numStandardIndices = randomIntBetween(1, 5); long numStandardDocs = populateStandardIndices(numStandardIndices); + var indexing1 = indicesService.stats(CommonStatsFlags.ALL, false).getIndexing().getTotal(); collectThenAssertMetrics( telemetry, 1, @@ -104,6 +149,7 @@ public void testIndicesMetrics() { long numTimeSeriesIndices = randomIntBetween(1, 5); long numTimeSeriesDocs = populateTimeSeriesIndices(numTimeSeriesIndices); + var indexing2 = indicesService.stats(CommonStatsFlags.ALL, false).getIndexing().getTotal(); collectThenAssertMetrics( telemetry, 2, @@ -133,6 +179,7 @@ public void testIndicesMetrics() { long numLogsdbIndices = randomIntBetween(1, 5); long numLogsdbDocs = populateLogsdbIndices(numLogsdbIndices); + var indexing3 = indicesService.stats(CommonStatsFlags.ALL, false).getIndexing().getTotal(); collectThenAssertMetrics( telemetry, 3, @@ -159,6 +206,142 @@ public void testIndicesMetrics() { greaterThan(0L) ) ); + // indexing stats + collectThenAssertMetrics( + telemetry, + 4, + Map.of( + STANDARD_INDEXING_COUNT, + equalTo(numStandardDocs), + STANDARD_INDEXING_TIME, + greaterThanOrEqualTo(0L), + STANDARD_INDEXING_FAILURE, + equalTo(indexing1.getIndexFailedCount() - indexing0.getIndexCount()), + + TIME_SERIES_INDEXING_COUNT, + equalTo(numTimeSeriesDocs), + TIME_SERIES_INDEXING_TIME, + greaterThanOrEqualTo(0L), + TIME_SERIES_INDEXING_FAILURE, + equalTo(indexing2.getIndexFailedCount() - indexing1.getIndexFailedCount()), + + LOGSDB_INDEXING_COUNT, + equalTo(numLogsdbDocs), + LOGSDB_INDEXING_TIME, + greaterThanOrEqualTo(0L), + LOGSDB_INDEXING_FAILURE, + equalTo(indexing3.getIndexFailedCount() - indexing2.getIndexFailedCount()) + ) + ); + telemetry.resetMeter(); + + // search and fetch + client().prepareSearch("standard*").setSize(100).get().decRef(); + var nodeStats1 = indicesService.stats(CommonStatsFlags.ALL, false).getSearch().getTotal(); + collectThenAssertMetrics( + telemetry, + 1, + Map.of( + STANDARD_QUERY_COUNT, + equalTo(numStandardIndices), + STANDARD_QUERY_TIME, + equalTo(nodeStats1.getQueryTimeInMillis()), + STANDARD_FETCH_COUNT, + equalTo(nodeStats1.getFetchCount()), + STANDARD_FETCH_TIME, + equalTo(nodeStats1.getFetchTimeInMillis()), + + TIME_SERIES_QUERY_COUNT, + equalTo(0L), + TIME_SERIES_QUERY_TIME, + equalTo(0L), + + LOGSDB_QUERY_COUNT, + equalTo(0L), + LOGSDB_QUERY_TIME, + equalTo(0L) + ) + ); + + client().prepareSearch("time*").setSize(100).get().decRef(); + var nodeStats2 = indicesService.stats(CommonStatsFlags.ALL, false).getSearch().getTotal(); + collectThenAssertMetrics( + telemetry, + 2, + Map.of( + STANDARD_QUERY_COUNT, + equalTo(numStandardIndices), + STANDARD_QUERY_TIME, + equalTo(nodeStats1.getQueryTimeInMillis()), + + TIME_SERIES_QUERY_COUNT, + equalTo(numTimeSeriesIndices), + TIME_SERIES_QUERY_TIME, + equalTo(nodeStats2.getQueryTimeInMillis() - nodeStats1.getQueryTimeInMillis()), + TIME_SERIES_FETCH_COUNT, + equalTo(nodeStats2.getFetchCount() - nodeStats1.getFetchCount()), + TIME_SERIES_FETCH_TIME, + equalTo(nodeStats2.getFetchTimeInMillis() - nodeStats1.getFetchTimeInMillis()), + + LOGSDB_QUERY_COUNT, + equalTo(0L), + LOGSDB_QUERY_TIME, + equalTo(0L) + ) + ); + client().prepareSearch("logs*").setSize(100).get().decRef(); + var nodeStats3 = indicesService.stats(CommonStatsFlags.ALL, false).getSearch().getTotal(); + collectThenAssertMetrics( + telemetry, + 3, + Map.of( + STANDARD_QUERY_COUNT, + equalTo(numStandardIndices), + STANDARD_QUERY_TIME, + equalTo(nodeStats1.getQueryTimeInMillis()), + + TIME_SERIES_QUERY_COUNT, + equalTo(numTimeSeriesIndices), + TIME_SERIES_QUERY_TIME, + equalTo(nodeStats2.getQueryTimeInMillis() - nodeStats1.getQueryTimeInMillis()), + + LOGSDB_QUERY_COUNT, + equalTo(numLogsdbIndices), + LOGSDB_QUERY_TIME, + equalTo(nodeStats3.getQueryTimeInMillis() - nodeStats2.getQueryTimeInMillis()), + LOGSDB_FETCH_COUNT, + equalTo(nodeStats3.getFetchCount() - nodeStats2.getFetchCount()), + LOGSDB_FETCH_TIME, + equalTo(nodeStats3.getFetchTimeInMillis() - nodeStats2.getFetchTimeInMillis()) + ) + ); + // search failures + expectThrows(Exception.class, () -> { client().prepareSearch("logs*").setRuntimeMappings(parseMapping(""" + { + "fail_me": { + "type": "long", + "script": {"source": "<>", "lang": "failing_field"} + } + } + """)).setQuery(new RangeQueryBuilder("fail_me").gte(0)).setAllowPartialSearchResults(true).get(); }); + collectThenAssertMetrics( + telemetry, + 4, + Map.of( + STANDARD_QUERY_FAILURE, + equalTo(0L), + STANDARD_FETCH_FAILURE, + equalTo(0L), + TIME_SERIES_QUERY_FAILURE, + equalTo(0L), + TIME_SERIES_FETCH_FAILURE, + equalTo(0L), + LOGSDB_QUERY_FAILURE, + equalTo(numLogsdbIndices), + LOGSDB_FETCH_FAILURE, + equalTo(0L) + ) + ); } void collectThenAssertMetrics(TestTelemetryPlugin telemetry, int times, Map> matchers) { @@ -175,7 +358,7 @@ int populateStandardIndices(long numIndices) { int totalDocs = 0; for (int i = 0; i < numIndices; i++) { String indexName = "standard-" + i; - createIndex(indexName); + createIndex(indexName, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build()); int numDocs = between(1, 5); for (int d = 0; d < numDocs; d++) { indexDoc(indexName, Integer.toString(d), "f", Integer.toString(d)); @@ -190,7 +373,11 @@ int populateTimeSeriesIndices(long numIndices) { int totalDocs = 0; for (int i = 0; i < numIndices; i++) { String indexName = "time_series-" + i; - Settings settings = Settings.builder().put("mode", "time_series").putList("routing_path", List.of("host")).build(); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("mode", "time_series") + .putList("routing_path", List.of("host")) + .build(); client().admin() .indices() .prepareCreate(indexName) @@ -214,6 +401,7 @@ int populateTimeSeriesIndices(long numIndices) { } totalDocs += numDocs; flush(indexName); + refresh(indexName); } return totalDocs; } @@ -222,7 +410,7 @@ int populateLogsdbIndices(long numIndices) { int totalDocs = 0; for (int i = 0; i < numIndices; i++) { String indexName = "logsdb-" + i; - Settings settings = Settings.builder().put("mode", "logsdb").build(); + Settings settings = Settings.builder().put("mode", "logsdb").put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build(); client().admin() .indices() .prepareCreate(indexName) @@ -237,9 +425,75 @@ int populateLogsdbIndices(long numIndices) { .setSource("@timestamp", timestamp, "host.name", randomFrom("prod", "qa"), "cpu", randomIntBetween(1, 100)) .get(); } + int numFailures = between(0, 2); + for (int d = 0; d < numFailures; d++) { + expectThrows(Exception.class, () -> { + client().prepareIndex(indexName) + .setSource( + "@timestamp", + "malformed-timestamp", + "host.name", + randomFrom("prod", "qa"), + "cpu", + randomIntBetween(1, 100) + ) + .get(); + }); + } totalDocs += numDocs; flush(indexName); + refresh(indexName); } return totalDocs; } + + private Map parseMapping(String mapping) throws IOException { + try (XContentParser parser = createParser(JsonXContent.jsonXContent, mapping)) { + return parser.map(); + } + } + + public static class FailingFieldPlugin extends Plugin implements ScriptPlugin { + + @Override + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { + return new ScriptEngine() { + @Override + public String getType() { + return "failing_field"; + } + + @Override + @SuppressWarnings("unchecked") + public FactoryType compile( + String name, + String code, + ScriptContext context, + Map params + ) { + return (FactoryType) new LongFieldScript.Factory() { + @Override + public LongFieldScript.LeafFactory newFactory( + String fieldName, + Map params, + SearchLookup searchLookup, + OnScriptError onScriptError + ) { + return ctx -> new LongFieldScript(fieldName, params, searchLookup, onScriptError, ctx) { + @Override + public void execute() { + throw new IllegalStateException("Accessing failing field"); + } + }; + } + }; + } + + @Override + public Set> getSupportedContexts() { + return Set.of(LongFieldScript.CONTEXT); + } + }; + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java deleted file mode 100644 index b78448bfd873f..0000000000000 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java +++ /dev/null @@ -1,756 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.search.retriever; - -import org.apache.lucene.search.TotalHits; -import org.apache.lucene.search.join.ScoreMode; -import org.apache.lucene.util.SetOnce; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.search.MultiSearchRequest; -import org.elasticsearch.action.search.MultiSearchResponse; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.TransportMultiSearchAction; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.Maps; -import org.elasticsearch.index.query.InnerHitBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.QueryRewriteContext; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.MockSearchService; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.builder.PointInTimeBuilder; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.collapse.CollapseBuilder; -import org.elasticsearch.search.rank.RankDoc; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.NestedSortBuilder; -import org.elasticsearch.search.sort.ScoreSortBuilder; -import org.elasticsearch.search.sort.ShardDocSortField; -import org.elasticsearch.search.sort.SortBuilder; -import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentType; -import org.junit.Before; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; -import static org.hamcrest.Matchers.equalTo; - -public class RankDocRetrieverBuilderIT extends ESIntegTestCase { - - @Override - protected Collection> nodePlugins() { - return List.of(MockSearchService.TestPlugin.class); - } - - public record RetrieverSource(RetrieverBuilder retriever, SearchSourceBuilder source) {} - - private static String INDEX = "test_index"; - private static final String ID_FIELD = "_id"; - private static final String DOC_FIELD = "doc"; - private static final String TEXT_FIELD = "text"; - private static final String VECTOR_FIELD = "vector"; - private static final String TOPIC_FIELD = "topic"; - private static final String LAST_30D_FIELD = "views.last30d"; - private static final String ALL_TIME_FIELD = "views.all"; - - @Before - public void setup() throws Exception { - String mapping = """ - { - "properties": { - "vector": { - "type": "dense_vector", - "dims": 3, - "element_type": "float", - "index": true, - "similarity": "l2_norm", - "index_options": { - "type": "hnsw" - } - }, - "text": { - "type": "text" - }, - "doc": { - "type": "keyword" - }, - "topic": { - "type": "keyword" - }, - "views": { - "type": "nested", - "properties": { - "last30d": { - "type": "integer" - }, - "all": { - "type": "integer" - } - } - } - } - } - """; - createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).build()); - admin().indices().preparePutMapping(INDEX).setSource(mapping, XContentType.JSON).get(); - indexDoc( - INDEX, - "doc_1", - DOC_FIELD, - "doc_1", - TOPIC_FIELD, - "technology", - TEXT_FIELD, - "the quick brown fox jumps over the lazy dog", - LAST_30D_FIELD, - 100 - ); - indexDoc( - INDEX, - "doc_2", - DOC_FIELD, - "doc_2", - TOPIC_FIELD, - "astronomy", - TEXT_FIELD, - "you know, for Search!", - VECTOR_FIELD, - new float[] { 1.0f, 2.0f, 3.0f }, - LAST_30D_FIELD, - 3 - ); - indexDoc(INDEX, "doc_3", DOC_FIELD, "doc_3", TOPIC_FIELD, "technology", VECTOR_FIELD, new float[] { 6.0f, 6.0f, 6.0f }); - indexDoc( - INDEX, - "doc_4", - DOC_FIELD, - "doc_4", - TOPIC_FIELD, - "technology", - TEXT_FIELD, - "aardvark is a really awesome animal, but not very quick", - ALL_TIME_FIELD, - 100, - LAST_30D_FIELD, - 40 - ); - indexDoc(INDEX, "doc_5", DOC_FIELD, "doc_5", TOPIC_FIELD, "science", TEXT_FIELD, "irrelevant stuff"); - indexDoc( - INDEX, - "doc_6", - DOC_FIELD, - "doc_6", - TEXT_FIELD, - "quick quick quick quick search", - VECTOR_FIELD, - new float[] { 10.0f, 30.0f, 100.0f }, - LAST_30D_FIELD, - 15 - ); - indexDoc( - INDEX, - "doc_7", - DOC_FIELD, - "doc_7", - TOPIC_FIELD, - "biology", - TEXT_FIELD, - "dog", - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - ALL_TIME_FIELD, - 1000 - ); - refresh(INDEX); - } - - public void testRankDocsRetrieverBasicWithPagination() { - final int rankWindowSize = 100; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.boolQuery() - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(9L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(8L)); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 2 and 6 due to prefilter - standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); - standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 7, 2, 3, and 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - null, - 10, - 100, - null - ); - // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 3, 4, 7 and with pagination, we'd just omit the first result - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList( - new RetrieverSource(standard0, null), - new RetrieverSource(standard1, null), - new RetrieverSource(knnRetrieverBuilder, null) - ) - ) - ); - // include some pagination as well - source.from(1); - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2")); - assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_3")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_4")); - assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_7")); - }); - } - - public void testRankDocsRetrieverWithAggs() { - // same as above, but we only want to bring back the top result from each subsearch - // so that would be 1, 2, and 7 - // and final rank would be (based on score): 2, 1, 7 - // aggs should still account for the same docs as the testRankDocsRetriever test, i.e. all but doc_5 - final int rankWindowSize = 1; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.boolQuery() - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(9L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(8L)); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 2 and 6 due to prefilter - standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); - standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 7, 2, 3, and 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - null, - 10, - 100, - null - ); - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList( - new RetrieverSource(standard0, null), - new RetrieverSource(standard1, null), - new RetrieverSource(knnRetrieverBuilder, null) - ) - ) - ); - source.size(1); - source.aggregation(new TermsAggregationBuilder("topic").field(TOPIC_FIELD)); - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(5L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getHits().length, equalTo(1)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2")); - assertNotNull(resp.getAggregations()); - assertNotNull(resp.getAggregations().get("topic")); - Terms terms = resp.getAggregations().get("topic"); - // doc_3 is not part of the final aggs computation as it is only retrieved through the knn retriever - // and is outside of the rank window - assertThat(terms.getBucketByKey("technology").getDocCount(), equalTo(2L)); - assertThat(terms.getBucketByKey("astronomy").getDocCount(), equalTo(1L)); - assertThat(terms.getBucketByKey("biology").getDocCount(), equalTo(1L)); - }); - } - - public void testRankDocsRetrieverWithCollapse() { - final int rankWindowSize = 100; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.boolQuery() - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(9L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(8L)); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 2 and 6 due to prefilter - standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); - standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 7, 2, 3, and 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - null, - 10, - 100, - null - ); - // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 3, 4, 7 - // with collapsing on topic field we would have 6, 2, 1, 7 - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList( - new RetrieverSource(standard0, null), - new RetrieverSource(standard1, null), - new RetrieverSource(knnRetrieverBuilder, null) - ) - ) - ); - source.collapse( - new CollapseBuilder(TOPIC_FIELD).setInnerHits( - new InnerHitBuilder("a").addSort(new FieldSortBuilder(DOC_FIELD).order(SortOrder.DESC)).setSize(10) - ) - ); - source.fetchField(TOPIC_FIELD); - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getHits().length, equalTo(4)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6")); - assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_2")); - assertThat(resp.getHits().getAt(1).field(TOPIC_FIELD).getValue().toString(), equalTo("astronomy")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(2).field(TOPIC_FIELD).getValue().toString(), equalTo("technology")); - assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(0).getId(), equalTo("doc_4")); - assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(1).getId(), equalTo("doc_3")); - assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(2).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_7")); - assertThat(resp.getHits().getAt(3).field(TOPIC_FIELD).getValue().toString(), equalTo("biology")); - }); - } - - public void testRankDocsRetrieverWithNestedCollapseAndAggs() { - final int rankWindowSize = 10; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1 and 6 as doc_4 is collapsed to doc_1 - standard0.queryBuilder = QueryBuilders.boolQuery() - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(9L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(8L)); - standard0.collapseBuilder = new CollapseBuilder(TOPIC_FIELD).setInnerHits( - new InnerHitBuilder("a").addSort(new FieldSortBuilder(DOC_FIELD).order(SortOrder.DESC)).setSize(10) - ); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 2 and 6 due to prefilter - standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); - standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 7, 2, 3, and 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - null, - 10, - 100, - null - ); - // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 3, 4, 7 - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList( - new RetrieverSource(standard0, null), - new RetrieverSource(standard1, null), - new RetrieverSource(knnRetrieverBuilder, null) - ) - ) - ); - source.aggregation(new TermsAggregationBuilder("topic").field(TOPIC_FIELD)); - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6")); - assertNotNull(resp.getAggregations()); - assertNotNull(resp.getAggregations().get("topic")); - Terms terms = resp.getAggregations().get("topic"); - // doc_3 is not part of the final aggs computation as it is only retrieved through the knn retriever - // and is outside of the rank window - assertThat(terms.getBucketByKey("technology").getDocCount(), equalTo(3L)); - assertThat(terms.getBucketByKey("astronomy").getDocCount(), equalTo(1L)); - assertThat(terms.getBucketByKey("biology").getDocCount(), equalTo(1L)); - }); - } - - public void testRankDocsRetrieverWithNestedQuery() { - final int rankWindowSize = 100; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(LAST_30D_FIELD).gt(10L), ScoreMode.Avg); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 2 and 6 due to prefilter - standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); - standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 7, 2, 3, and 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - null, - 10, - 100, - null - ); - // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 3, 4, 7 - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList( - new RetrieverSource(standard0, null), - new RetrieverSource(standard1, null), - new RetrieverSource(knnRetrieverBuilder, null) - ) - ) - ); - source.fetchField(TOPIC_FIELD); - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6")); - assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_2")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_3")); - assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_4")); - assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_7")); - }); - } - - public void testRankDocsRetrieverMultipleCompoundRetrievers() { - final int rankWindowSize = 100; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.boolQuery() - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(9L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(8L)); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 2 and 6 due to prefilter - standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); - standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 7, 2, 3, and 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - null, - 10, - 100, - null - ); - // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 4, 7, 3 - CompoundRetrieverWithRankDocs compoundRetriever1 = new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList( - new RetrieverSource(standard0, null), - new RetrieverSource(standard1, null), - new RetrieverSource(knnRetrieverBuilder, null) - ) - ); - // simple standard retriever that would have the doc_4 as its first (and only) result - StandardRetrieverBuilder standard2 = new StandardRetrieverBuilder(); - standard2.queryBuilder = QueryBuilders.queryStringQuery("aardvark").defaultField(TEXT_FIELD); - - // combining the two retrievers would bring doc_4 at the top as it would be the only one present in both doc sets - // the rest of the docs would be sorted based on their ranks as they have the same score (1/2) - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList(new RetrieverSource(compoundRetriever1, null), new RetrieverSource(standard2, null)) - ) - ); - - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_4")); - assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_2")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_3")); - assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_6")); - assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_7")); - }); - } - - public void testRankDocsRetrieverDifferentNestedSorting() { - final int rankWindowSize = 100; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1, 4, 6, 2 - standard0.queryBuilder = QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(LAST_30D_FIELD).gt(0), ScoreMode.Avg); - standard0.sortBuilders = List.of( - new FieldSortBuilder(LAST_30D_FIELD).setNestedSort(new NestedSortBuilder("views")).order(SortOrder.DESC) - ); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 4, 7 - standard1.queryBuilder = QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(ALL_TIME_FIELD).gt(0), ScoreMode.Avg); - standard1.sortBuilders = List.of( - new FieldSortBuilder(ALL_TIME_FIELD).setNestedSort(new NestedSortBuilder("views")).order(SortOrder.ASC) - ); - - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList(new RetrieverSource(standard0, null), new RetrieverSource(standard1, null)) - ) - ); - - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(5L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_4")); - assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_2")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_6")); - assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_7")); - }); - } - - class CompoundRetrieverWithRankDocs extends RetrieverBuilder { - - private final List sources; - private final int rankWindowSize; - - private CompoundRetrieverWithRankDocs(int rankWindowSize, List sources) { - this.rankWindowSize = rankWindowSize; - this.sources = Collections.unmodifiableList(sources); - } - - @Override - public boolean isCompound() { - return true; - } - - @Override - public QueryBuilder topDocsQuery() { - throw new UnsupportedOperationException("should not be called"); - } - - @Override - public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException { - if (ctx.getPointInTimeBuilder() == null) { - throw new IllegalStateException("PIT is required"); - } - - // Rewrite prefilters - boolean hasChanged = false; - var newPreFilters = rewritePreFilters(ctx); - hasChanged |= newPreFilters != preFilterQueryBuilders; - - // Rewrite retriever sources - List newRetrievers = new ArrayList<>(); - for (var entry : sources) { - RetrieverBuilder newRetriever = entry.retriever.rewrite(ctx); - if (newRetriever != entry.retriever) { - newRetrievers.add(new RetrieverSource(newRetriever, null)); - hasChanged |= newRetriever != entry.retriever; - } else if (newRetriever == entry.retriever) { - var sourceBuilder = entry.source != null - ? entry.source - : createSearchSourceBuilder(ctx.getPointInTimeBuilder(), newRetriever); - var rewrittenSource = sourceBuilder.rewrite(ctx); - newRetrievers.add(new RetrieverSource(newRetriever, rewrittenSource)); - hasChanged |= rewrittenSource != entry.source; - } - } - if (hasChanged) { - return new CompoundRetrieverWithRankDocs(rankWindowSize, newRetrievers); - } - - // execute searches - final SetOnce results = new SetOnce<>(); - final MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); - for (var entry : sources) { - SearchRequest searchRequest = new SearchRequest().source(entry.source); - // The can match phase can reorder shards, so we disable it to ensure the stable ordering - searchRequest.setPreFilterShardSize(Integer.MAX_VALUE); - multiSearchRequest.add(searchRequest); - } - ctx.registerAsyncAction((client, listener) -> { - client.execute(TransportMultiSearchAction.TYPE, multiSearchRequest, new ActionListener<>() { - @Override - public void onResponse(MultiSearchResponse items) { - List topDocs = new ArrayList<>(); - for (int i = 0; i < items.getResponses().length; i++) { - var item = items.getResponses()[i]; - var rankDocs = getRankDocs(item.getResponse()); - sources.get(i).retriever().setRankDocs(rankDocs); - topDocs.add(rankDocs); - } - results.set(combineResults(topDocs)); - listener.onResponse(null); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); - }); - - return new RankDocsRetrieverBuilder( - rankWindowSize, - newRetrievers.stream().map(s -> s.retriever).toList(), - results::get, - newPreFilters - ); - } - - @Override - public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) { - throw new UnsupportedOperationException("should not be called"); - } - - @Override - public String getName() { - return "compound_retriever"; - } - - @Override - protected void doToXContent(XContentBuilder builder, Params params) throws IOException { - - } - - @Override - protected boolean doEquals(Object o) { - return false; - } - - @Override - protected int doHashCode() { - return 0; - } - - private RankDoc[] getRankDocs(SearchResponse searchResponse) { - assert searchResponse != null; - int size = Math.min(rankWindowSize, searchResponse.getHits().getHits().length); - RankDoc[] docs = new RankDoc[size]; - for (int i = 0; i < size; i++) { - var hit = searchResponse.getHits().getAt(i); - long sortValue = (long) hit.getRawSortValues()[hit.getRawSortValues().length - 1]; - int doc = ShardDocSortField.decodeDoc(sortValue); - int shardRequestIndex = ShardDocSortField.decodeShardRequestIndex(sortValue); - docs[i] = new RankDoc(doc, hit.getScore(), shardRequestIndex); - docs[i].rank = i + 1; - } - return docs; - } - - record RankDocAndHitRatio(RankDoc rankDoc, float hitRatio) {} - - /** - * Combines the provided {@code rankResults} to return the final top documents. - */ - public RankDoc[] combineResults(List rankResults) { - int totalQueries = rankResults.size(); - final float step = 1.0f / totalQueries; - Map docsToRankResults = Maps.newMapWithExpectedSize(rankWindowSize); - for (var rankResult : rankResults) { - for (RankDoc scoreDoc : rankResult) { - docsToRankResults.compute(new RankDoc.RankKey(scoreDoc.doc, scoreDoc.shardIndex), (key, value) -> { - if (value == null) { - RankDoc res = new RankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex); - res.rank = scoreDoc.rank; - return new RankDocAndHitRatio(res, step); - } else { - RankDoc res = new RankDoc(scoreDoc.doc, Math.max(scoreDoc.score, value.rankDoc.score), scoreDoc.shardIndex); - res.rank = Math.min(scoreDoc.rank, value.rankDoc.rank); - return new RankDocAndHitRatio(res, value.hitRatio + step); - } - }); - } - } - // sort the results based on hit ratio, then doc, then rank, and final tiebreaker is based on smaller doc id - RankDocAndHitRatio[] sortedResults = docsToRankResults.values().toArray(RankDocAndHitRatio[]::new); - Arrays.sort(sortedResults, (RankDocAndHitRatio doc1, RankDocAndHitRatio doc2) -> { - if (doc1.hitRatio != doc2.hitRatio) { - return doc1.hitRatio < doc2.hitRatio ? 1 : -1; - } - if (false == (Float.isNaN(doc1.rankDoc.score) || Float.isNaN(doc2.rankDoc.score)) - && (doc1.rankDoc.score != doc2.rankDoc.score)) { - return doc1.rankDoc.score < doc2.rankDoc.score ? 1 : -1; - } - if (doc1.rankDoc.rank != doc2.rankDoc.rank) { - return doc1.rankDoc.rank < doc2.rankDoc.rank ? -1 : 1; - } - return doc1.rankDoc.doc < doc2.rankDoc.doc ? -1 : 1; - }); - // trim the results if needed, otherwise each shard will always return `rank_window_size` results. - // pagination and all else will happen on the coordinator when combining the shard responses - RankDoc[] topResults = new RankDoc[Math.min(rankWindowSize, sortedResults.length)]; - for (int rank = 0; rank < topResults.length; ++rank) { - topResults[rank] = sortedResults[rank].rankDoc; - topResults[rank].rank = rank + 1; - topResults[rank].score = sortedResults[rank].hitRatio; - } - return topResults; - } - } - - private SearchSourceBuilder createSearchSourceBuilder(PointInTimeBuilder pit, RetrieverBuilder retrieverBuilder) { - var sourceBuilder = new SearchSourceBuilder().pointInTimeBuilder(pit).trackTotalHits(false).size(100); - retrieverBuilder.extractToSearchSourceBuilder(sourceBuilder, false); - - // Record the shard id in the sort result - List> sortBuilders = sourceBuilder.sorts() != null ? new ArrayList<>(sourceBuilder.sorts()) : new ArrayList<>(); - if (sortBuilders.isEmpty()) { - sortBuilders.add(new ScoreSortBuilder()); - } - sortBuilders.add(new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME)); - sourceBuilder.sort(sortBuilders); - return sourceBuilder; - } -} diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index c55436f85a6e3..1911013cbe8e9 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -234,6 +234,8 @@ static TransportVersion def(int id) { public static final TransportVersion RRF_QUERY_REWRITE = def(8_758_00_0); public static final TransportVersion SEARCH_FAILURE_STATS = def(8_759_00_0); public static final TransportVersion INGEST_GEO_DATABASE_PROVIDERS = def(8_760_00_0); + public static final TransportVersion DATE_TIME_DOC_VALUES_LOCALES = def(8_761_00_0); + public static final TransportVersion FAST_REFRESH_RCO = def(8_762_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java index 9581279201be2..4aa6ed60afe43 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java @@ -21,6 +21,8 @@ import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.core.UpdateForV10; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.xcontent.ToXContent; @@ -43,6 +45,8 @@ public class ClusterRerouteResponse extends ActionResponse implements IsAcknowle /** * To be removed when REST compatibility with {@link org.elasticsearch.Version#V_8_6_0} / {@link RestApiVersion#V_8} no longer needed */ + @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) // to remove from the v9 API only + @UpdateForV10(owner = UpdateForV10.Owner.DISTRIBUTED_COORDINATION) // to remove entirely private final ClusterState state; private final RoutingExplanations explanations; private final boolean acknowledged; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java index 7857e9a22e9b9..cb667400240f0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java @@ -23,7 +23,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.injection.guice.Inject; @@ -120,27 +119,18 @@ public void onPrimaryOperationComplete( ActionListener listener ) { assert replicaRequest.primaryRefreshResult.refreshed() : "primary has not refreshed"; - boolean fastRefresh = IndexSettings.INDEX_FAST_REFRESH_SETTING.get( - clusterService.state().metadata().index(indexShardRoutingTable.shardId().getIndex()).getSettings() + UnpromotableShardRefreshRequest unpromotableReplicaRequest = new UnpromotableShardRefreshRequest( + indexShardRoutingTable, + replicaRequest.primaryRefreshResult.primaryTerm(), + replicaRequest.primaryRefreshResult.generation(), + false + ); + transportService.sendRequest( + transportService.getLocalNode(), + TransportUnpromotableShardRefreshAction.NAME, + unpromotableReplicaRequest, + new ActionListenerResponseHandler<>(listener.safeMap(r -> null), in -> ActionResponse.Empty.INSTANCE, refreshExecutor) ); - - // Indices marked with fast refresh do not rely on refreshing the unpromotables - if (fastRefresh) { - listener.onResponse(null); - } else { - UnpromotableShardRefreshRequest unpromotableReplicaRequest = new UnpromotableShardRefreshRequest( - indexShardRoutingTable, - replicaRequest.primaryRefreshResult.primaryTerm(), - replicaRequest.primaryRefreshResult.generation(), - false - ); - transportService.sendRequest( - transportService.getLocalNode(), - TransportUnpromotableShardRefreshAction.NAME, - unpromotableReplicaRequest, - new ActionListenerResponseHandler<>(listener.safeMap(r -> null), in -> ActionResponse.Empty.INSTANCE, refreshExecutor) - ); - } } } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java index 6c24ec2d17604..f91a983d47885 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java @@ -24,6 +24,9 @@ import java.util.List; +import static org.elasticsearch.TransportVersions.FAST_REFRESH_RCO; +import static org.elasticsearch.index.IndexSettings.INDEX_FAST_REFRESH_SETTING; + public class TransportUnpromotableShardRefreshAction extends TransportBroadcastUnpromotableAction< UnpromotableShardRefreshRequest, ActionResponse.Empty> { @@ -73,6 +76,18 @@ protected void unpromotableShardOperation( return; } + // During an upgrade to FAST_REFRESH_RCO, we expect search shards to be first upgraded before the primary is upgraded. Thus, + // when the primary is upgraded, and starts to deliver unpromotable refreshes, we expect the search shards to be upgraded already. + // Note that the fast refresh setting is final. + // TODO: remove assertion (ES-9563) + assert INDEX_FAST_REFRESH_SETTING.get(shard.indexSettings().getSettings()) == false + || transportService.getLocalNodeConnection().getTransportVersion().onOrAfter(FAST_REFRESH_RCO) + : "attempted to refresh a fast refresh search shard " + + shard + + " on transport version " + + transportService.getLocalNodeConnection().getTransportVersion() + + " (before FAST_REFRESH_RCO)"; + ActionListener.run(responseListener, listener -> { shard.waitForPrimaryTermAndGeneration( request.getPrimaryTerm(), diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 3561a4d0e2cb4..fdced5fc18ac9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -16,7 +16,6 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.AliasMetadata; -import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; @@ -157,8 +156,7 @@ protected void masterOperation( xContentRegistry, indicesService, systemIndices, - indexSettingProviders, - Map.of() + indexSettingProviders ); final Map> overlapping = new HashMap<>(); @@ -235,8 +233,7 @@ public static Template resolveTemplate( final NamedXContentRegistry xContentRegistry, final IndicesService indicesService, final SystemIndices systemIndices, - Set indexSettingProviders, - Map componentTemplateSubstitutions + Set indexSettingProviders ) throws Exception { var metadata = simulatedState.getMetadata(); Settings templateSettings = resolveSettings(simulatedState.metadata(), matchingTemplate); @@ -266,7 +263,6 @@ public static Template resolveTemplate( null, // empty request mapping as the user can't specify any explicit mappings via the simulate api simulatedState, matchingTemplate, - componentTemplateSubstitutions, xContentRegistry, simulatedIndexName ); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java index af7a253b5a042..30bbad0b57df0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -170,8 +170,7 @@ protected void masterOperation( xContentRegistry, indicesService, systemIndices, - indexSettingProviders, - Map.of() + indexSettingProviders ); if (request.includeDefaults()) { listener.onResponse( diff --git a/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java b/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java index d8c2389dd7d69..58ffe25e08e49 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java @@ -105,6 +105,7 @@ public static class Handler implements Releasable { private boolean closed = false; private boolean globalFailure = false; private boolean incrementalRequestSubmitted = false; + private boolean bulkInProgress = false; private ThreadContext.StoredContext requestContext; private Exception bulkActionLevelFailure = null; private long currentBulkSize = 0L; @@ -130,6 +131,7 @@ protected Handler( public void addItems(List> items, Releasable releasable, Runnable nextItems) { assert closed == false; + assert bulkInProgress == false; if (bulkActionLevelFailure != null) { shortCircuitDueToTopLevelFailure(items, releasable); nextItems.run(); @@ -143,6 +145,7 @@ public void addItems(List> items, Releasable releasable, Runn requestContext.restore(); final ArrayList toRelease = new ArrayList<>(releasables); releasables.clear(); + bulkInProgress = true; client.bulk(bulkRequest, ActionListener.runAfter(new ActionListener<>() { @Override @@ -158,6 +161,7 @@ public void onFailure(Exception e) { handleBulkFailure(isFirstRequest, e); } }, () -> { + bulkInProgress = false; requestContext = threadContext.newStoredContext(); toRelease.forEach(Releasable::close); nextItems.run(); @@ -177,6 +181,7 @@ private boolean shouldBackOff() { } public void lastItems(List> items, Releasable releasable, ActionListener listener) { + assert bulkInProgress == false; if (bulkActionLevelFailure != null) { shortCircuitDueToTopLevelFailure(items, releasable); errorResponse(listener); @@ -187,6 +192,8 @@ public void lastItems(List> items, Releasable releasable, Act requestContext.restore(); final ArrayList toRelease = new ArrayList<>(releasables); releasables.clear(); + // We do not need to set this back to false as this will be the last request. + bulkInProgress = true; client.bulk(bulkRequest, ActionListener.runBefore(new ActionListener<>() { private final boolean isFirstRequest = incrementalRequestSubmitted == false; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java index 5eae1c660d7d0..8c6565e52daa7 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java @@ -41,6 +41,7 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.concurrent.Executor; @@ -180,12 +181,48 @@ protected void doRun() throws IOException { private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor executor, ActionListener listener) throws IOException { boolean hasIndexRequestsWithPipelines = false; - final Metadata metadata = clusterService.state().getMetadata(); - Map templateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); + final Metadata metadata; + Map componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); + if (bulkRequest.isSimulated() && componentTemplateSubstitutions.isEmpty() == false) { + /* + * If this is a simulated request, and there are template substitutions, then we want to create and use a new metadata that has + * those templates. That is, we want to add the new templates (which will replace any that already existed with the same name), + * and remove the indices and data streams that are referred to from the bulkRequest so that we get settings from the templates + * rather than from the indices/data streams. + */ + Metadata.Builder simulatedMetadataBuilder = Metadata.builder(clusterService.state().getMetadata()); + if (componentTemplateSubstitutions.isEmpty() == false) { + Map updatedComponentTemplates = new HashMap<>(); + updatedComponentTemplates.putAll(clusterService.state().metadata().componentTemplates()); + updatedComponentTemplates.putAll(componentTemplateSubstitutions); + simulatedMetadataBuilder.componentTemplates(updatedComponentTemplates); + } + /* + * We now remove the index from the simulated metadata to force the templates to be used. Note that simulated requests are + * always index requests -- no other type of request is supported. + */ + for (DocWriteRequest actionRequest : bulkRequest.requests) { + assert actionRequest != null : "Requests cannot be null in simulate mode"; + assert actionRequest instanceof IndexRequest + : "Only IndexRequests are supported in simulate mode, but got " + actionRequest.getClass(); + if (actionRequest != null) { + IndexRequest indexRequest = (IndexRequest) actionRequest; + String indexName = indexRequest.index(); + if (indexName != null) { + simulatedMetadataBuilder.remove(indexName); + simulatedMetadataBuilder.removeDataStream(indexName); + } + } + } + metadata = simulatedMetadataBuilder.build(); + } else { + metadata = clusterService.state().getMetadata(); + } + for (DocWriteRequest actionRequest : bulkRequest.requests) { IndexRequest indexRequest = getIndexWriteRequest(actionRequest); if (indexRequest != null) { - IngestService.resolvePipelinesAndUpdateIndexRequest(actionRequest, indexRequest, metadata, templateSubstitutions); + IngestService.resolvePipelinesAndUpdateIndexRequest(actionRequest, indexRequest, metadata); hasIndexRequestsWithPipelines |= IngestService.hasPipeline(indexRequest); } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index c860c49809cb5..713116c4cf98e 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -51,6 +51,7 @@ import org.elasticsearch.xcontent.NamedXContentRegistry; import java.io.IOException; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -197,12 +198,33 @@ private Exception validateMappings(Map componentTempl * path for when the index does not exist). And it does not deal with system indices since we do not intend for users to * simulate writing to system indices. */ - // First, we remove the index from the cluster state if necessary (since we're going to use the templates) - ClusterState simulatedState = indexAbstraction == null - ? state - : new ClusterState.Builder(state).metadata(Metadata.builder(state.metadata()).remove(request.index()).build()).build(); + ClusterState.Builder simulatedClusterStateBuilder = new ClusterState.Builder(state); + Metadata.Builder simulatedMetadata = Metadata.builder(state.metadata()); + if (indexAbstraction != null) { + /* + * We remove the index or data stream from the cluster state so that we are forced to fall back to the templates to get + * mappings. + */ + String indexRequest = request.index(); + assert indexRequest != null : "Index requests cannot be null in a simulate bulk call"; + if (indexRequest != null) { + simulatedMetadata.remove(indexRequest); + simulatedMetadata.removeDataStream(indexRequest); + } + } + if (componentTemplateSubstitutions.isEmpty() == false) { + /* + * We put the template substitutions into the cluster state. If they have the same name as an existing one, the + * existing one is replaced. + */ + Map updatedComponentTemplates = new HashMap<>(); + updatedComponentTemplates.putAll(state.metadata().componentTemplates()); + updatedComponentTemplates.putAll(componentTemplateSubstitutions); + simulatedMetadata.componentTemplates(updatedComponentTemplates); + } + ClusterState simulatedState = simulatedClusterStateBuilder.metadata(simulatedMetadata).build(); - String matchingTemplate = findV2Template(state.metadata(), request.index(), false); + String matchingTemplate = findV2Template(simulatedState.metadata(), request.index(), false); if (matchingTemplate != null) { final Template template = TransportSimulateIndexTemplateAction.resolveTemplate( matchingTemplate, @@ -212,8 +234,7 @@ private Exception validateMappings(Map componentTempl xContentRegistry, indicesService, systemIndices, - indexSettingProviders, - componentTemplateSubstitutions + indexSettingProviders ); CompressedXContent mappings = template.mappings(); if (mappings != null) { @@ -247,7 +268,7 @@ private Exception validateMappings(Map componentTempl }); } } else { - List matchingTemplates = findV1Templates(state.metadata(), request.index(), false); + List matchingTemplates = findV1Templates(simulatedState.metadata(), request.index(), false); final Map mappingsMap = MetadataCreateIndexService.parseV1Mappings( "{}", matchingTemplates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()), diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java index 189aa1c95d865..99eac250641ae 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java @@ -125,11 +125,10 @@ protected void asyncShardOperation(GetRequest request, ShardId shardId, ActionLi IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); IndexShard indexShard = indexService.getShard(shardId.id()); if (indexShard.routingEntry().isPromotableToPrimary() == false) { - assert indexShard.indexSettings().isFastRefresh() == false - : "a search shard should not receive a TransportGetAction for an index with fast refresh"; handleGetOnUnpromotableShard(request, indexShard, listener); return; } + // TODO: adapt assertion to assert only that it is not stateless (ES-9563) assert DiscoveryNode.isStateless(clusterService.getSettings()) == false || indexShard.indexSettings().isFastRefresh() : "in Stateless a promotable to primary shard can receive a TransportGetAction only if an index has the fast refresh setting"; if (request.realtime()) { // we are not tied to a refresh cycle here anyway diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java index 8d5760307c3fe..633e7ef6793ab 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java @@ -124,11 +124,10 @@ protected void asyncShardOperation(MultiGetShardRequest request, ShardId shardId IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); IndexShard indexShard = indexService.getShard(shardId.id()); if (indexShard.routingEntry().isPromotableToPrimary() == false) { - assert indexShard.indexSettings().isFastRefresh() == false - : "a search shard should not receive a TransportShardMultiGetAction for an index with fast refresh"; handleMultiGetOnUnpromotableShard(request, indexShard, listener); return; } + // TODO: adapt assertion to assert only that it is not stateless (ES-9563) assert DiscoveryNode.isStateless(clusterService.getSettings()) == false || indexShard.indexSettings().isFastRefresh() : "in Stateless a promotable to primary shard can receive a TransportShardMultiGetAction only if an index has " + "the fast refresh setting"; diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index 7ee9be25b3d59..5457ca60d0da4 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -10,6 +10,7 @@ package org.elasticsearch.action.search; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.Maps; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.InnerHitBuilder; @@ -82,8 +83,15 @@ private void doRun() { CollapseBuilder innerCollapseBuilder = innerHitBuilder.getInnerCollapseBuilder(); SearchSourceBuilder sourceBuilder = buildExpandSearchSourceBuilder(innerHitBuilder, innerCollapseBuilder).query(groupQuery) .postFilter(searchRequest.source().postFilter()) - .runtimeMappings(searchRequest.source().runtimeMappings()); + .runtimeMappings(searchRequest.source().runtimeMappings()) + .pointInTimeBuilder(searchRequest.source().pointInTimeBuilder()); SearchRequest groupRequest = new SearchRequest(searchRequest); + if (searchRequest.pointInTimeBuilder() != null) { + // if the original request has a point in time, we propagate it to the inner search request + // and clear the indices and preference from the inner search request + groupRequest.indices(Strings.EMPTY_ARRAY); + groupRequest.preference(null); + } groupRequest.source(sourceBuilder); multiRequest.add(groupRequest); } diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/PostWriteRefresh.java b/server/src/main/java/org/elasticsearch/action/support/replication/PostWriteRefresh.java index 683c3589c893d..7414aeeb2c405 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/PostWriteRefresh.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/PostWriteRefresh.java @@ -19,7 +19,6 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.translog.Translog; @@ -53,9 +52,7 @@ public void refreshShard( case WAIT_UNTIL -> waitUntil(indexShard, location, new ActionListener<>() { @Override public void onResponse(Boolean forced) { - // Fast refresh indices do not depend on the unpromotables being refreshed - boolean fastRefresh = IndexSettings.INDEX_FAST_REFRESH_SETTING.get(indexShard.indexSettings().getSettings()); - if (location != null && (indexShard.routingEntry().isSearchable() == false && fastRefresh == false)) { + if (location != null && indexShard.routingEntry().isSearchable() == false) { refreshUnpromotables(indexShard, location, listener, forced, postWriteRefreshTimeout); } else { listener.onResponse(forced); @@ -68,9 +65,7 @@ public void onFailure(Exception e) { } }); case IMMEDIATE -> immediate(indexShard, listener.delegateFailureAndWrap((l, r) -> { - // Fast refresh indices do not depend on the unpromotables being refreshed - boolean fastRefresh = IndexSettings.INDEX_FAST_REFRESH_SETTING.get(indexShard.indexSettings().getSettings()); - if (indexShard.getReplicationGroup().getRoutingTable().unpromotableShards().size() > 0 && fastRefresh == false) { + if (indexShard.getReplicationGroup().getRoutingTable().unpromotableShards().size() > 0) { sendUnpromotableRequests(indexShard, r.generation(), true, l, postWriteRefreshTimeout); } else { l.onResponse(true); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 4cdf1508a7987..f43f1c6b05a15 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -665,7 +665,6 @@ private ClusterState applyCreateIndexRequestWithV2Template( request.mappings(), currentState, templateName, - Map.of(), xContentRegistry, request.index() ); @@ -824,7 +823,6 @@ private static List collectSystemV2Mappings( List templateMappings = MetadataIndexTemplateService.collectMappings( composableIndexTemplate, componentTemplates, - Map.of(), indexName ); return collectV2Mappings(null, templateMappings, xContentRegistry); @@ -834,16 +832,10 @@ public static List collectV2Mappings( @Nullable final String requestMappings, final ClusterState currentState, final String templateName, - Map componentTemplateSubstitutions, final NamedXContentRegistry xContentRegistry, final String indexName ) throws Exception { - List templateMappings = MetadataIndexTemplateService.collectMappings( - currentState, - templateName, - componentTemplateSubstitutions, - indexName - ); + List templateMappings = MetadataIndexTemplateService.collectMappings(currentState, templateName, indexName); return collectV2Mappings(requestMappings, templateMappings, xContentRegistry); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 1f9f6f636c1cf..abeb3279b7b50 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -698,7 +698,7 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT final var now = Instant.now(); final var metadata = currentState.getMetadata(); - final var combinedMappings = collectMappings(indexTemplate, metadata.componentTemplates(), Map.of(), "tmp_idx"); + final var combinedMappings = collectMappings(indexTemplate, metadata.componentTemplates(), "tmp_idx"); final var combinedSettings = resolveSettings(indexTemplate, metadata.componentTemplates()); // First apply settings sourced from index setting providers: for (var provider : indexSettingProviders) { @@ -1341,12 +1341,7 @@ private static boolean isGlobalAndHasIndexHiddenSetting(Metadata metadata, Compo /** * Collect the given v2 template into an ordered list of mappings. */ - public static List collectMappings( - final ClusterState state, - final String templateName, - Map componentTemplateSubstitutions, - final String indexName - ) { + public static List collectMappings(final ClusterState state, final String templateName, final String indexName) { final ComposableIndexTemplate template = state.metadata().templatesV2().get(templateName); assert template != null : "attempted to resolve mappings for a template [" + templateName + "] that did not exist in the cluster state"; @@ -1355,7 +1350,7 @@ public static List collectMappings( } final Map componentTemplates = state.metadata().componentTemplates(); - return collectMappings(template, componentTemplates, componentTemplateSubstitutions, indexName); + return collectMappings(template, componentTemplates, indexName); } /** @@ -1364,7 +1359,6 @@ public static List collectMappings( public static List collectMappings( final ComposableIndexTemplate template, final Map componentTemplates, - final Map componentTemplateSubstitutions, final String indexName ) { Objects.requireNonNull(template, "Composable index template must be provided"); @@ -1375,12 +1369,9 @@ public static List collectMappings( ComposableIndexTemplate.DataStreamTemplate.DATA_STREAM_MAPPING_SNIPPET ); } - final Map combinedComponentTemplates = new HashMap<>(); - combinedComponentTemplates.putAll(componentTemplates); - combinedComponentTemplates.putAll(componentTemplateSubstitutions); List mappings = template.composedOf() .stream() - .map(combinedComponentTemplates::get) + .map(componentTemplates::get) .filter(Objects::nonNull) .map(ComponentTemplate::template) .map(Template::mappings) @@ -1716,7 +1707,7 @@ private static void validateCompositeTemplate( String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName; // Parse mappings to ensure they are valid after being composed - List mappings = collectMappings(stateWithIndex, templateName, Map.of(), indexName); + List mappings = collectMappings(stateWithIndex, templateName, indexName); try { MapperService mapperService = tempIndexService.mapperService(); mapperService.merge(MapperService.SINGLE_MAPPING_NAME, mappings, MapperService.MergeReason.INDEX_TEMPLATE); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java index f7812d284f2af..9120e25b443d7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java @@ -32,6 +32,7 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.elasticsearch.TransportVersions.FAST_REFRESH_RCO; import static org.elasticsearch.index.IndexSettings.INDEX_FAST_REFRESH_SETTING; public class OperationRouting { @@ -305,8 +306,14 @@ public ShardId shardId(ClusterState clusterState, String index, String id, @Null } public static boolean canSearchShard(ShardRouting shardRouting, ClusterState clusterState) { + // TODO: remove if and always return isSearchable (ES-9563) if (INDEX_FAST_REFRESH_SETTING.get(clusterState.metadata().index(shardRouting.index()).getSettings())) { - return shardRouting.isPromotableToPrimary(); + // Until all the cluster is upgraded, we send searches/gets to the primary (even if it has been upgraded) to execute locally. + if (clusterState.getMinTransportVersion().onOrAfter(FAST_REFRESH_RCO)) { + return shardRouting.isSearchable(); + } else { + return shardRouting.isPromotableToPrimary(); + } } else { return shardRouting.isSearchable(); } diff --git a/server/src/main/java/org/elasticsearch/common/util/AbstractHash.java b/server/src/main/java/org/elasticsearch/common/util/AbstractHash.java index e975f138e8b7d..687e9cb3fd9dc 100644 --- a/server/src/main/java/org/elasticsearch/common/util/AbstractHash.java +++ b/server/src/main/java/org/elasticsearch/common/util/AbstractHash.java @@ -32,7 +32,17 @@ public long id(long index) { return ids.get(index) - 1; } - protected final long id(long index, long id) { + /** + * Set the id provided key at 0 <= index <= capacity() . + */ + protected final void setId(long index, long id) { + ids.set(index, id + 1); + } + + /** + * Set the id provided key at 0 <= index <= capacity() and get the previous value or -1 if this slot is unused. + */ + protected final long getAndSetId(long index, long id) { return ids.getAndSet(index, id + 1) - 1; } diff --git a/server/src/main/java/org/elasticsearch/common/util/ArrayUtils.java b/server/src/main/java/org/elasticsearch/common/util/ArrayUtils.java index 96b694e04bd5e..be40bf16e20e4 100644 --- a/server/src/main/java/org/elasticsearch/common/util/ArrayUtils.java +++ b/server/src/main/java/org/elasticsearch/common/util/ArrayUtils.java @@ -126,4 +126,20 @@ public static void reverseSubArray(long[] array, int offset, int length) { end--; } } + + /** + * Reverse the {@code length} values on the array starting from {@code offset}. + */ + public static void reverseArray(byte[] array, int offset, int length) { + int start = offset; + int end = offset + length; + while (start < end) { + final byte temp = array[start]; + array[start] = array[end - 1]; + array[end - 1] = temp; + start++; + end--; + } + } + } diff --git a/server/src/main/java/org/elasticsearch/common/util/BigArrays.java b/server/src/main/java/org/elasticsearch/common/util/BigArrays.java index a33ee4c2edeac..4f0aae9380a01 100644 --- a/server/src/main/java/org/elasticsearch/common/util/BigArrays.java +++ b/server/src/main/java/org/elasticsearch/common/util/BigArrays.java @@ -452,7 +452,13 @@ public T get(long index) { } @Override - public T set(long index, T value) { + public void set(long index, T value) { + assert index >= 0 && index < size(); + array[(int) index] = value; + } + + @Override + public T getAndSet(long index, T value) { assert index >= 0 && index < size(); @SuppressWarnings("unchecked") T ret = (T) array[(int) index]; diff --git a/server/src/main/java/org/elasticsearch/common/util/BigObjectArray.java b/server/src/main/java/org/elasticsearch/common/util/BigObjectArray.java index 019ef341de8dc..95707b64b9a1e 100644 --- a/server/src/main/java/org/elasticsearch/common/util/BigObjectArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/BigObjectArray.java @@ -46,7 +46,15 @@ public T get(long index) { } @Override - public T set(long index, T value) { + public void set(long index, T value) { + final int pageIndex = pageIndex(index); + final int indexInPage = indexInPage(index); + final Object[] page = pages[pageIndex]; + page[indexInPage] = value; + } + + @Override + public T getAndSet(long index, T value) { final int pageIndex = pageIndex(index); final int indexInPage = indexInPage(index); final Object[] page = pages[pageIndex]; diff --git a/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java b/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java index 48a810789308f..208d29edad71d 100644 --- a/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java +++ b/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java @@ -169,7 +169,7 @@ private long set(BytesRef key, int code, long id) { for (long index = slot;; index = nextSlot(index, mask)) { final long curId = id(index); if (curId == -1) { // means unset - id(index, id); + setId(index, id); append(id, key, code); ++size; return id; @@ -197,7 +197,7 @@ private void reset(int code, long id) { for (long index = slot;; index = nextSlot(index, mask)) { final long curId = id(index); if (curId == -1) { // means unset - id(index, id); + setId(index, id); break; } } @@ -223,7 +223,7 @@ public long add(BytesRef key) { @Override protected void removeAndAdd(long index) { - final long id = id(index, -1); + final long id = getAndSetId(index, -1); assert id >= 0; final int code = hashes.get(id); reset(code, id); diff --git a/server/src/main/java/org/elasticsearch/common/util/Int3Hash.java b/server/src/main/java/org/elasticsearch/common/util/Int3Hash.java index f2a4288bf7c9b..dc49b39a031a1 100644 --- a/server/src/main/java/org/elasticsearch/common/util/Int3Hash.java +++ b/server/src/main/java/org/elasticsearch/common/util/Int3Hash.java @@ -79,7 +79,7 @@ private long set(int key1, int key2, int key3, long id) { while (true) { final long curId = id(index); if (curId == -1) { // means unset - id(index, id); + setId(index, id); append(id, key1, key2, key3); ++size; return id; @@ -106,7 +106,7 @@ private void reset(int key1, int key2, int key3, long id) { while (true) { final long curId = id(index); if (curId == -1) { // means unset - id(index, id); + setId(index, id); append(id, key1, key2, key3); break; } @@ -130,7 +130,7 @@ public long add(int key1, int key2, int key3) { @Override protected void removeAndAdd(long index) { - final long id = id(index, -1); + final long id = getAndSetId(index, -1); assert id >= 0; long keyOffset = id * 3; final int key1 = keys.getAndSet(keyOffset, 0); diff --git a/server/src/main/java/org/elasticsearch/common/util/LocaleUtils.java b/server/src/main/java/org/elasticsearch/common/util/LocaleUtils.java index 86e82886ed263..bdbf843ad2b53 100644 --- a/server/src/main/java/org/elasticsearch/common/util/LocaleUtils.java +++ b/server/src/main/java/org/elasticsearch/common/util/LocaleUtils.java @@ -68,16 +68,16 @@ private static Locale parseParts(String[] parts) { switch (parts.length) { case 3: // lang, country, variant - return new Locale(parts[0], parts[1], parts[2]); + return Locale.of(parts[0], parts[1], parts[2]); case 2: // lang, country - return new Locale(parts[0], parts[1]); + return Locale.of(parts[0], parts[1]); case 1: if ("ROOT".equalsIgnoreCase(parts[0])) { return Locale.ROOT; } // lang - return new Locale(parts[0]); + return Locale.of(parts[0]); default: throw new IllegalArgumentException( "Locales can have at most 3 parts but got " + parts.length + ": " + Arrays.asList(parts) diff --git a/server/src/main/java/org/elasticsearch/common/util/LongHash.java b/server/src/main/java/org/elasticsearch/common/util/LongHash.java index 4de6772d22447..0c681063c50b0 100644 --- a/server/src/main/java/org/elasticsearch/common/util/LongHash.java +++ b/server/src/main/java/org/elasticsearch/common/util/LongHash.java @@ -67,7 +67,7 @@ private long set(long key, long id) { for (long index = slot;; index = nextSlot(index, mask)) { final long curId = id(index); if (curId == -1) { // means unset - id(index, id); + setId(index, id); append(id, key); ++size; return id; @@ -82,13 +82,13 @@ private void append(long id, long key) { keys.set(id, key); } - private void reset(long key, long id) { + private void reset(long id) { + final long key = keys.get(id); final long slot = slot(hash(key), mask); for (long index = slot;; index = nextSlot(index, mask)) { final long curId = id(index); if (curId == -1) { // means unset - id(index, id); - append(id, key); + setId(index, id); break; } } @@ -109,10 +109,9 @@ public long add(long key) { @Override protected void removeAndAdd(long index) { - final long id = id(index, -1); + final long id = getAndSetId(index, -1); assert id >= 0; - final long key = keys.getAndSet(id, 0); - reset(key, id); + reset(id); } @Override diff --git a/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java b/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java index f160ecdaa7079..f7708af59dde2 100644 --- a/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java +++ b/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java @@ -84,7 +84,7 @@ private long set(long key1, long key2, long id) { for (long index = slot;; index = nextSlot(index, mask)) { final long curId = id(index); if (curId == -1) { // means unset - id(index, id); + setId(index, id); append(id, key1, key2); ++size; return id; @@ -104,13 +104,16 @@ private void append(long id, long key1, long key2) { keys.set(keyOffset + 1, key2); } - private void reset(long key1, long key2, long id) { + private void reset(long id) { + final LongArray keys = this.keys; + final long keyOffset = id * 2; + final long key1 = keys.get(keyOffset); + final long key2 = keys.get(keyOffset + 1); final long slot = slot(hash(key1, key2), mask); for (long index = slot;; index = nextSlot(index, mask)) { final long curId = id(index); if (curId == -1) { // means unset - id(index, id); - append(id, key1, key2); + setId(index, id); break; } } @@ -132,12 +135,9 @@ public long add(long key1, long key2) { @Override protected void removeAndAdd(long index) { - final long id = id(index, -1); + final long id = getAndSetId(index, -1); assert id >= 0; - long keyOffset = id * 2; - final long key1 = keys.getAndSet(keyOffset, 0); - final long key2 = keys.getAndSet(keyOffset + 1, 0); - reset(key1, key2, id); + reset(id); } @Override diff --git a/server/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java b/server/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java index 8ef3b568b0396..d955863caa091 100644 --- a/server/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java +++ b/server/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java @@ -77,7 +77,7 @@ public T put(long key, T value) { */ public T remove(long key) { for (long i = slot(hash(key), mask);; i = nextSlot(i, mask)) { - final T previous = values.set(i, null); + final T previous = values.getAndSet(i, null); if (previous == null) { return null; } else if (keys.get(i) == key) { @@ -98,7 +98,7 @@ private T set(long key, T value) { throw new IllegalArgumentException("Null values are not supported"); } for (long i = slot(hash(key), mask);; i = nextSlot(i, mask)) { - final T previous = values.set(i, value); + final T previous = values.getAndSet(i, value); if (previous == null) { // slot was free keys.set(i, key); @@ -180,7 +180,7 @@ protected boolean used(long bucket) { @Override protected void removeAndAdd(long index) { final long key = keys.get(index); - final T value = values.set(index, null); + final T value = values.getAndSet(index, null); --size; final T removed = set(key, value); assert removed == null; diff --git a/server/src/main/java/org/elasticsearch/common/util/ObjectArray.java b/server/src/main/java/org/elasticsearch/common/util/ObjectArray.java index 24b010eb62aad..034b7b3c85692 100644 --- a/server/src/main/java/org/elasticsearch/common/util/ObjectArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/ObjectArray.java @@ -19,9 +19,14 @@ public interface ObjectArray extends BigArray { */ T get(long index); + /** + * Set a value at the given index. + */ + void set(long index, T value); + /** * Set a value at the given index and return the previous value. */ - T set(long index, T value); + T getAndSet(long index, T value); } diff --git a/server/src/main/java/org/elasticsearch/common/util/ObjectObjectPagedHashMap.java b/server/src/main/java/org/elasticsearch/common/util/ObjectObjectPagedHashMap.java index 58722c18b2434..298f910d65a9f 100644 --- a/server/src/main/java/org/elasticsearch/common/util/ObjectObjectPagedHashMap.java +++ b/server/src/main/java/org/elasticsearch/common/util/ObjectObjectPagedHashMap.java @@ -82,7 +82,7 @@ public V put(K key, V value) { public V remove(K key) { final long slot = slot(key.hashCode(), mask); for (long index = slot;; index = nextSlot(index, mask)) { - final V previous = values.set(index, null); + final V previous = values.getAndSet(index, null); if (previous == null) { return null; } else if (keys.get(index).equals(key)) { @@ -104,7 +104,7 @@ private V set(K key, int code, V value) { assert size < maxSize; final long slot = slot(code, mask); for (long index = slot;; index = nextSlot(index, mask)) { - final V previous = values.set(index, value); + final V previous = values.getAndSet(index, value); if (previous == null) { // slot was free keys.set(index, key); @@ -186,7 +186,7 @@ protected boolean used(long bucket) { @Override protected void removeAndAdd(long index) { final K key = keys.get(index); - final V value = values.set(index, null); + final V value = values.getAndSet(index, null); --size; final V removed = set(key, key.hashCode(), value); assert removed == null; diff --git a/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java b/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java index c19e3ca353569..3b37afc3b297b 100644 --- a/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java +++ b/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java @@ -105,7 +105,7 @@ static boolean shouldLoadRandomAccessFiltersEagerly(IndexSettings settings) { boolean loadFiltersEagerlySetting = settings.getValue(INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING); boolean isStateless = DiscoveryNode.isStateless(settings.getNodeSettings()); if (isStateless) { - return DiscoveryNode.hasRole(settings.getNodeSettings(), DiscoveryNodeRole.INDEX_ROLE) + return DiscoveryNode.hasRole(settings.getNodeSettings(), DiscoveryNodeRole.SEARCH_ROLE) && loadFiltersEagerlySetting && INDEX_FAST_REFRESH_SETTING.get(settings.getSettings()); } else { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java index 651d9e76e84a2..481901f7c03ce 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java @@ -165,10 +165,11 @@ public void doValidate(MappingLookup lookup) { Map configuredSettings = XContentHelper.convertToMap(BytesReference.bytes(builder), false, XContentType.JSON).v2(); configuredSettings = (Map) configuredSettings.values().iterator().next(); - // Only type, meta and format attributes are allowed: + // Only type, meta, format, and locale attributes are allowed: configuredSettings.remove("type"); configuredSettings.remove("meta"); configuredSettings.remove("format"); + configuredSettings.remove("locale"); // ignoring malformed values is disallowed (see previous check), // however if `index.mapping.ignore_malformed` has been set to true then diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 2e602033442c7..7be5ee2200b5c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -79,11 +79,16 @@ public final class DateFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "date"; public static final String DATE_NANOS_CONTENT_TYPE = "date_nanos"; - public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time||epoch_millis"); + public static final Locale DEFAULT_LOCALE = Locale.ENGLISH; + // although the locale doesn't affect the results, tests still check formatter equality, which does include locale + public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time||epoch_millis") + .withLocale(DEFAULT_LOCALE); public static final DateFormatter DEFAULT_DATE_TIME_NANOS_FORMATTER = DateFormatter.forPattern( "strict_date_optional_time_nanos||epoch_millis" - ); - private static final DateMathParser EPOCH_MILLIS_PARSER = DateFormatter.forPattern("epoch_millis").toDateMathParser(); + ).withLocale(DEFAULT_LOCALE); + private static final DateMathParser EPOCH_MILLIS_PARSER = DateFormatter.forPattern("epoch_millis") + .withLocale(DEFAULT_LOCALE) + .toDateMathParser(); public enum Resolution { MILLISECONDS(CONTENT_TYPE, NumericType.DATE, DateMillisDocValuesField::new) { @@ -232,7 +237,7 @@ public static final class Builder extends FieldMapper.Builder { private final Parameter locale = new Parameter<>( "locale", false, - () -> Locale.ROOT, + () -> DEFAULT_LOCALE, (n, c, o) -> LocaleUtils.parse(o.toString()), m -> toType(m).locale, (xContentBuilder, n, v) -> xContentBuilder.field(n, v.toString()), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java index e519fec09ce78..341944c3d687a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java @@ -69,7 +69,7 @@ private static class Builder extends AbstractScriptFieldType.Builder o == null ? null : LocaleUtils.parse(o.toString()), RuntimeField.initializerNotSupported(), (b, n, v) -> { - if (v != null && false == v.equals(Locale.ROOT)) { + if (v != null && false == v.equals(DateFieldMapper.DEFAULT_LOCALE)) { b.field(n, v.toString()); } }, @@ -97,7 +97,7 @@ protected AbstractScriptFieldType createFieldType( OnScriptError onScriptError ) { String pattern = format.getValue() == null ? DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.pattern() : format.getValue(); - Locale locale = this.locale.getValue() == null ? Locale.ROOT : this.locale.getValue(); + Locale locale = this.locale.getValue() == null ? DateFieldMapper.DEFAULT_LOCALE : this.locale.getValue(); DateFormatter dateTimeFormatter = DateFormatter.forPattern(pattern, supportedVersion).withLocale(locale); return new DateScriptFieldType(name, factory, dateTimeFormatter, script, meta, onScriptError); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 0b9727aa66c8a..5e63fee8c5adc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -63,7 +63,7 @@ public enum Subobjects { this.printedValue = printedValue; } - static Subobjects from(Object node) { + public static Subobjects from(Object node) { if (node instanceof Boolean value) { return value ? Subobjects.ENABLED : Subobjects.DISABLED; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index af0dc0c0ad7fe..6ca30304201b2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -59,6 +59,7 @@ public class RangeFieldMapper extends FieldMapper { public static class Defaults { public static final DateFormatter DATE_FORMATTER = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; + public static final Locale LOCALE = DateFieldMapper.DEFAULT_LOCALE; } // this is private since it has a different default @@ -83,7 +84,7 @@ public static class Builder extends FieldMapper.Builder { private final Parameter locale = new Parameter<>( "locale", false, - () -> Locale.ROOT, + () -> Defaults.LOCALE, (n, c, o) -> LocaleUtils.parse(o.toString()), m -> toType(m).locale, (xContentBuilder, n, v) -> xContentBuilder.field(n, v.toString()), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java index f086526eec78e..b60feb4d5746e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java @@ -179,7 +179,7 @@ public static boolean parseMultiField( public static DateFormatter parseDateTimeFormatter(Object node) { if (node instanceof String) { - return DateFormatter.forPattern((String) node); + return DateFormatter.forPattern((String) node).withLocale(DateFieldMapper.DEFAULT_LOCALE); } throw new IllegalArgumentException("Invalid format: [" + node.toString() + "]: expected string value"); } diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java index aeffff28269dd..20874a736b1ec 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java @@ -10,7 +10,6 @@ package org.elasticsearch.index.query; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; @@ -19,8 +18,8 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.lucene.queries.SpanMatchNoDocsQuery; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentParser; @@ -76,40 +75,37 @@ public SpanTermQueryBuilder(StreamInput in) throws IOException { } @Override - protected SpanQuery doToQuery(SearchExecutionContext context) throws IOException { + protected Query doToQuery(SearchExecutionContext context) throws IOException { MappedFieldType mapper = context.getFieldType(fieldName); - Term term; if (mapper == null) { - term = new Term(fieldName, BytesRefs.toBytesRef(value)); - } else { - if (mapper.getTextSearchInfo().hasPositions() == false) { - throw new IllegalArgumentException( - "Span term query requires position data, but field " + fieldName + " was indexed without position data" - ); - } - Query termQuery = mapper.termQuery(value, context); - List termsList = new ArrayList<>(); - termQuery.visit(new QueryVisitor() { - @Override - public QueryVisitor getSubVisitor(BooleanClause.Occur occur, Query parent) { - if (occur == BooleanClause.Occur.MUST || occur == BooleanClause.Occur.FILTER) { - return this; - } - return EMPTY_VISITOR; + return new SpanMatchNoDocsQuery(fieldName, "unmapped field: " + fieldName); + } + if (mapper.getTextSearchInfo().hasPositions() == false) { + throw new IllegalArgumentException( + "Span term query requires position data, but field " + fieldName + " was indexed without position data" + ); + } + Query termQuery = mapper.termQuery(value, context); + List termsList = new ArrayList<>(); + termQuery.visit(new QueryVisitor() { + @Override + public QueryVisitor getSubVisitor(BooleanClause.Occur occur, Query parent) { + if (occur == BooleanClause.Occur.MUST || occur == BooleanClause.Occur.FILTER) { + return this; } + return EMPTY_VISITOR; + } - @Override - public void consumeTerms(Query query, Term... terms) { - termsList.addAll(Arrays.asList(terms)); - } - }); - if (termsList.size() != 1) { - // This is for safety, but we have called mapper.termQuery above: we really should get one and only one term from the query? - throw new IllegalArgumentException("Cannot extract a term from a query of type " + termQuery.getClass() + ": " + termQuery); + @Override + public void consumeTerms(Query query, Term... terms) { + termsList.addAll(Arrays.asList(terms)); } - term = termsList.get(0); + }); + if (termsList.size() != 1) { + // This is for safety, but we have called mapper.termQuery above: we really should get one and only one term from the query? + throw new IllegalArgumentException("Cannot extract a term from a query of type " + termQuery.getClass() + ": " + termQuery); } - return new SpanTermQuery(term); + return new SpanTermQuery(termsList.get(0)); } public static SpanTermQueryBuilder fromXContent(XContentParser parser) throws IOException, ParsingException { diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 0275e988ce39d..0f63d2a8dcc1b 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -33,7 +33,6 @@ import org.elasticsearch.cluster.ClusterStateApplier; import org.elasticsearch.cluster.ClusterStateTaskExecutor; import org.elasticsearch.cluster.ClusterStateTaskListener; -import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -271,30 +270,14 @@ public static void resolvePipelinesAndUpdateIndexRequest( final IndexRequest indexRequest, final Metadata metadata ) { - resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, Map.of()); - } - - public static void resolvePipelinesAndUpdateIndexRequest( - final DocWriteRequest originalRequest, - final IndexRequest indexRequest, - final Metadata metadata, - Map componentTemplateSubstitutions - ) { - resolvePipelinesAndUpdateIndexRequest( - originalRequest, - indexRequest, - metadata, - System.currentTimeMillis(), - componentTemplateSubstitutions - ); + resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, System.currentTimeMillis()); } static void resolvePipelinesAndUpdateIndexRequest( final DocWriteRequest originalRequest, final IndexRequest indexRequest, final Metadata metadata, - final long epochMillis, - final Map componentTemplateSubstitutions + final long epochMillis ) { if (indexRequest.isPipelineResolved()) { return; @@ -302,21 +285,11 @@ static void resolvePipelinesAndUpdateIndexRequest( /* * Here we look for the pipelines associated with the index if the index exists. If the index does not exist we fall back to using - * templates to find the pipelines. But if a user has passed in component template substitutions, they want the settings from those - * used in place of the settings used to create any previous indices. So in that case we use the templates to find the pipelines -- - * we don't fall back to the existing index if we don't find any because it is possible the user has intentionally removed the - * pipeline. + * templates to find the pipelines. */ - final Pipelines pipelines; - if (componentTemplateSubstitutions.isEmpty()) { - pipelines = resolvePipelinesFromMetadata(originalRequest, indexRequest, metadata, epochMillis) // - .or(() -> resolvePipelinesFromIndexTemplates(indexRequest, metadata, Map.of())) - .orElse(Pipelines.NO_PIPELINES_DEFINED); - } else { - pipelines = resolvePipelinesFromIndexTemplates(indexRequest, metadata, componentTemplateSubstitutions).orElse( - Pipelines.NO_PIPELINES_DEFINED - ); - } + final Pipelines pipelines = resolvePipelinesFromMetadata(originalRequest, indexRequest, metadata, epochMillis).or( + () -> resolvePipelinesFromIndexTemplates(indexRequest, metadata) + ).orElse(Pipelines.NO_PIPELINES_DEFINED); // The pipeline coming as part of the request always has priority over the resolved one from metadata or templates String requestPipeline = indexRequest.getPipeline(); @@ -1466,11 +1439,7 @@ private static Optional resolvePipelinesFromMetadata( return Optional.of(new Pipelines(IndexSettings.DEFAULT_PIPELINE.get(settings), IndexSettings.FINAL_PIPELINE.get(settings))); } - private static Optional resolvePipelinesFromIndexTemplates( - IndexRequest indexRequest, - Metadata metadata, - Map componentTemplateSubstitutions - ) { + private static Optional resolvePipelinesFromIndexTemplates(IndexRequest indexRequest, Metadata metadata) { if (indexRequest.index() == null) { return Optional.empty(); } @@ -1480,7 +1449,7 @@ private static Optional resolvePipelinesFromIndexTemplates( // precedence), or if a V2 template does not match, any V1 templates String v2Template = MetadataIndexTemplateService.findV2Template(metadata, indexRequest.index(), false); if (v2Template != null) { - final Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template, componentTemplateSubstitutions); + final Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template); return Optional.of(new Pipelines(IndexSettings.DEFAULT_PIPELINE.get(settings), IndexSettings.FINAL_PIPELINE.get(settings))); } diff --git a/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java b/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java index 17e290283d5e0..e07f6908330df 100644 --- a/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java +++ b/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java @@ -18,8 +18,10 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.search.stats.SearchStats; import org.elasticsearch.index.shard.IllegalIndexShardStateException; import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.IndexingStats; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.telemetry.metric.LongWithAttributes; import org.elasticsearch.telemetry.metric.MeterRegistry; @@ -50,8 +52,8 @@ public IndicesMetrics(MeterRegistry meterRegistry, IndicesService indicesService } private static List registerAsyncMetrics(MeterRegistry registry, IndicesStatsCache cache) { - List metrics = new ArrayList<>(IndexMode.values().length * 3); - assert IndexMode.values().length == 3 : "index modes have changed"; + final int TOTAL_METRICS = 36; + List metrics = new ArrayList<>(TOTAL_METRICS); for (IndexMode indexMode : IndexMode.values()) { String name = indexMode.getName(); metrics.add( @@ -72,13 +74,89 @@ private static List registerAsyncMetrics(MeterRegistry registry, ); metrics.add( registry.registerLongGauge( - "es.indices." + name + ".bytes.total", + "es.indices." + name + ".size", "total size in bytes of " + name + " indices", - "unit", + "bytes", () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).numBytes) ) ); + // query (count, took, failures) - use gauges as shards can be removed + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".query.total", + "total queries of " + name + " indices", + "unit", + () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).search.getQueryCount()) + ) + ); + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".query.time", + "total query time of " + name + " indices", + "ms", + () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).search.getQueryTimeInMillis()) + ) + ); + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".query.failure.total", + "total query failures of " + name + " indices", + "unit", + () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).search.getQueryFailure()) + ) + ); + // fetch (count, took, failures) - use gauges as shards can be removed + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".fetch.total", + "total fetches of " + name + " indices", + "unit", + () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).search.getFetchCount()) + ) + ); + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".fetch.time", + "total fetch time of " + name + " indices", + "ms", + () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).search.getFetchTimeInMillis()) + ) + ); + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".fetch.failure.total", + "total fetch failures of " + name + " indices", + "unit", + () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).search.getFetchFailure()) + ) + ); + // indexing + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".indexing.total", + "total indexing operations of " + name + " indices", + "unit", + () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).indexing.getIndexCount()) + ) + ); + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".indexing.time", + "total indexing time of " + name + " indices", + "ms", + () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).indexing.getIndexTime().millis()) + ) + ); + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".indexing.failure.total", + "total indexing failures of " + name + " indices", + "unit", + () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).indexing.getIndexFailedCount()) + ) + ); } + assert metrics.size() == TOTAL_METRICS : "total number of metrics has changed"; return metrics; } @@ -107,6 +185,8 @@ static class IndexStats { int numIndices = 0; long numDocs = 0; long numBytes = 0; + SearchStats.Stats search = new SearchStats().getTotal(); + IndexingStats.Stats indexing = new IndexingStats().getTotal(); } private static class IndicesStatsCache extends SingleObjectCache> { @@ -152,6 +232,8 @@ private Map internalGetIndicesStats() { try { indexStats.numDocs += indexShard.commitStats().getNumDocs(); indexStats.numBytes += indexShard.storeStats().sizeInBytes(); + indexStats.search.add(indexShard.searchStats().getTotal()); + indexStats.indexing.add(indexShard.indexingStats().getTotal()); } catch (IllegalIndexShardStateException | AlreadyClosedException ignored) { // ignored } diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java index cce3c764fe7a4..2cd6e2b11ef7a 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java @@ -9,10 +9,17 @@ package org.elasticsearch.repositories; +import org.elasticsearch.cluster.metadata.RepositoryMetadata; +import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.telemetry.metric.LongCounter; import org.elasticsearch.telemetry.metric.LongHistogram; import org.elasticsearch.telemetry.metric.MeterRegistry; +import java.util.Map; + +/** + * The common set of metrics that we publish for {@link org.elasticsearch.repositories.blobstore.BlobStoreRepository} implementations. + */ public record RepositoriesMetrics( MeterRegistry meterRegistry, LongCounter requestCounter, @@ -28,15 +35,65 @@ public record RepositoriesMetrics( public static RepositoriesMetrics NOOP = new RepositoriesMetrics(MeterRegistry.NOOP); + /** + * Is incremented for each request sent to the blob store (including retries) + * + * Exposed as {@link #requestCounter()} + */ public static final String METRIC_REQUESTS_TOTAL = "es.repositories.requests.total"; + /** + * Is incremented for each request which returns a non 2xx response OR fails to return a response + * (includes throttling and retryable errors) + * + * Exposed as {@link #exceptionCounter()} + */ public static final String METRIC_EXCEPTIONS_TOTAL = "es.repositories.exceptions.total"; + /** + * Is incremented each time an operation ends with a 416 response + * + * Exposed as {@link #requestRangeNotSatisfiedExceptionCounter()} + */ public static final String METRIC_EXCEPTIONS_REQUEST_RANGE_NOT_SATISFIED_TOTAL = "es.repositories.exceptions.request_range_not_satisfied.total"; + /** + * Is incremented each time we are throttled by the blob store, e.g. upon receiving an HTTP 429 response + * + * Exposed as {@link #throttleCounter()} + */ public static final String METRIC_THROTTLES_TOTAL = "es.repositories.throttles.total"; + /** + * Is incremented for each operation we attempt, whether it succeeds or fails, this doesn't include retries + * + * Exposed via {@link #operationCounter()} + */ public static final String METRIC_OPERATIONS_TOTAL = "es.repositories.operations.total"; + /** + * Is incremented for each operation that ends with a non 2xx response or throws an exception + * + * Exposed via {@link #unsuccessfulOperationCounter()} + */ public static final String METRIC_UNSUCCESSFUL_OPERATIONS_TOTAL = "es.repositories.operations.unsuccessful.total"; + /** + * Each time an operation has one or more failed requests (from non 2xx response or exception), the + * count of those is sampled + * + * Exposed via {@link #exceptionHistogram()} + */ public static final String METRIC_EXCEPTIONS_HISTOGRAM = "es.repositories.exceptions.histogram"; + /** + * Each time an operation has one or more throttled requests, the count of those is sampled + * + * Exposed via {@link #throttleHistogram()} + */ public static final String METRIC_THROTTLES_HISTOGRAM = "es.repositories.throttles.histogram"; + /** + * Every operation that is attempted will record a time. The value recorded here is the sum of the duration of + * each of the requests executed to try and complete the operation. The duration of each request is the time + * between sending the request and either a response being received, or the request failing. Does not include + * the consumption of the body of the response or any time spent pausing between retries. + * + * Exposed via {@link #httpRequestTimeInMillisHistogram()} + */ public static final String HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM = "es.repositories.requests.http_request_time.histogram"; public RepositoriesMetrics(MeterRegistry meterRegistry) { @@ -61,4 +118,25 @@ public RepositoriesMetrics(MeterRegistry meterRegistry) { ) ); } + + /** + * Create the map of attributes we expect to see on repository metrics + */ + public static Map createAttributesMap( + RepositoryMetadata repositoryMetadata, + OperationPurpose purpose, + String operation + ) { + return Map.of( + "repo_type", + repositoryMetadata.type(), + "repo_name", + repositoryMetadata.name(), + "operation", + operation, + "purpose", + purpose.getKey() + ); + } + } diff --git a/server/src/main/java/org/elasticsearch/rest/DeprecationRestHandler.java b/server/src/main/java/org/elasticsearch/rest/DeprecationRestHandler.java index 98ab7d53ffbe6..8d363f6e63511 100644 --- a/server/src/main/java/org/elasticsearch/rest/DeprecationRestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/DeprecationRestHandler.java @@ -13,7 +13,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.core.Nullable; import java.util.Objects; @@ -29,7 +28,6 @@ public class DeprecationRestHandler extends FilterRestHandler implements RestHan private final DeprecationLogger deprecationLogger; private final boolean compatibleVersionWarning; private final String deprecationKey; - @Nullable private final Level deprecationLevel; /** @@ -39,6 +37,8 @@ public class DeprecationRestHandler extends FilterRestHandler implements RestHan * @param handler The rest handler to deprecate (it's possible that the handler is reused with a different name!) * @param method a method of a deprecated endpoint * @param path a path of a deprecated endpoint + * @param deprecationLevel The level of the deprecation warning, must be non-null + * and either {@link Level#WARN} or {@link DeprecationLogger#CRITICAL} * @param deprecationMessage The message to warn users with when they use the {@code handler} * @param deprecationLogger The deprecation logger * @param compatibleVersionWarning set to false so that a deprecation warning will be issued for the handled request, @@ -51,7 +51,7 @@ public DeprecationRestHandler( RestHandler handler, RestRequest.Method method, String path, - @Nullable Level deprecationLevel, + Level deprecationLevel, String deprecationMessage, DeprecationLogger deprecationLogger, boolean compatibleVersionWarning @@ -61,7 +61,7 @@ public DeprecationRestHandler( this.deprecationLogger = Objects.requireNonNull(deprecationLogger); this.compatibleVersionWarning = compatibleVersionWarning; this.deprecationKey = DEPRECATED_ROUTE_KEY + "_" + method + "_" + path; - if (deprecationLevel != null && (deprecationLevel != Level.WARN && deprecationLevel != DeprecationLogger.CRITICAL)) { + if (deprecationLevel != Level.WARN && deprecationLevel != DeprecationLogger.CRITICAL) { throw new IllegalArgumentException( "unexpected deprecation logger level: " + deprecationLevel + ", expected either 'CRITICAL' or 'WARN'" ); @@ -77,19 +77,18 @@ public DeprecationRestHandler( @Override public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception { if (compatibleVersionWarning == false) { - // The default value for deprecated requests without a version warning is WARN - if (deprecationLevel == null || deprecationLevel == Level.WARN) { + // emit a standard deprecation warning + if (Level.WARN == deprecationLevel) { deprecationLogger.warn(DeprecationCategory.API, deprecationKey, deprecationMessage); - } else { + } else if (DeprecationLogger.CRITICAL == deprecationLevel) { deprecationLogger.critical(DeprecationCategory.API, deprecationKey, deprecationMessage); } } else { - // The default value for deprecated requests with a version warning is CRITICAL, - // because they have a specific version where the endpoint is removed - if (deprecationLevel == null || deprecationLevel == DeprecationLogger.CRITICAL) { - deprecationLogger.compatibleCritical(deprecationKey, deprecationMessage); - } else { + // emit a compatibility warning + if (Level.WARN == deprecationLevel) { deprecationLogger.compatible(Level.WARN, deprecationKey, deprecationMessage); + } else if (DeprecationLogger.CRITICAL == deprecationLevel) { + deprecationLogger.compatibleCritical(deprecationKey, deprecationMessage); } } @@ -139,4 +138,9 @@ public static String requireValidHeader(String value) { return value; } + + // test only + Level getDeprecationLevel() { + return deprecationLevel; + } } diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java index 924cd361c671d..c2064fdd931de 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestController.java +++ b/server/src/main/java/org/elasticsearch/rest/RestController.java @@ -144,25 +144,6 @@ public ServerlessApiProtections getApiProtections() { return apiProtections; } - /** - * Registers a REST handler to be executed when the provided {@code method} and {@code path} match the request. - * - * @param method GET, POST, etc. - * @param path Path to handle (e.g. "/{index}/{type}/_bulk") - * @param version API version to handle (e.g. RestApiVersion.V_8) - * @param handler The handler to actually execute - * @param deprecationMessage The message to log and send as a header in the response - */ - protected void registerAsDeprecatedHandler( - RestRequest.Method method, - String path, - RestApiVersion version, - RestHandler handler, - String deprecationMessage - ) { - registerAsDeprecatedHandler(method, path, version, handler, deprecationMessage, null); - } - /** * Registers a REST handler to be executed when the provided {@code method} and {@code path} match the request. * @@ -179,40 +160,23 @@ protected void registerAsDeprecatedHandler( RestApiVersion version, RestHandler handler, String deprecationMessage, - @Nullable Level deprecationLevel + Level deprecationLevel ) { assert (handler instanceof DeprecationRestHandler) == false; - if (version == RestApiVersion.current()) { - // e.g. it was marked as deprecated in 8.x, and we're currently running 8.x - registerHandler( - method, - path, - version, - new DeprecationRestHandler(handler, method, path, deprecationLevel, deprecationMessage, deprecationLogger, false) - ); - } else if (version == RestApiVersion.minimumSupported()) { - // e.g. it was marked as deprecated in 7.x, and we're currently running 8.x + if (RestApiVersion.onOrAfter(RestApiVersion.minimumSupported()).test(version)) { registerHandler( method, path, version, - new DeprecationRestHandler(handler, method, path, deprecationLevel, deprecationMessage, deprecationLogger, true) - ); - } else { - // e.g. it was marked as deprecated in 7.x, and we're currently running *9.x* - logger.debug( - "Deprecated route [" - + method - + " " - + path - + "] for handler [" - + handler.getClass() - + "] " - + "with version [" - + version - + "], which is less than the minimum supported version [" - + RestApiVersion.minimumSupported() - + "]" + new DeprecationRestHandler( + handler, + method, + path, + deprecationLevel, + deprecationMessage, + deprecationLogger, + version != RestApiVersion.current() + ) ); } } @@ -250,21 +214,12 @@ protected void registerAsReplacedHandler( RestHandler handler, RestRequest.Method replacedMethod, String replacedPath, - RestApiVersion replacedVersion + RestApiVersion replacedVersion, + String replacedMessage, + Level deprecationLevel ) { - // e.g. [POST /_optimize] is deprecated! Use [POST /_forcemerge] instead. - final String replacedMessage = "[" - + replacedMethod.name() - + " " - + replacedPath - + "] is deprecated! Use [" - + method.name() - + " " - + path - + "] instead."; - registerHandler(method, path, version, handler); - registerAsDeprecatedHandler(replacedMethod, replacedPath, replacedVersion, handler, replacedMessage); + registerAsDeprecatedHandler(replacedMethod, replacedPath, replacedVersion, handler, replacedMessage, deprecationLevel); } /** @@ -284,7 +239,15 @@ protected void registerHandler(RestRequest.Method method, String path, RestApiVe private void registerHandlerNoWrap(RestRequest.Method method, String path, RestApiVersion version, RestHandler handler) { assert RestApiVersion.minimumSupported() == version || RestApiVersion.current() == version - : "REST API compatibility is only supported for version " + RestApiVersion.minimumSupported().major; + : "REST API compatibility is only supported for version " + + RestApiVersion.minimumSupported().major + + " [method=" + + method + + ", path=" + + path + + ", handler=" + + handler.getClass().getCanonicalName() + + "]"; if (RESERVED_PATHS.contains(path)) { throw new IllegalArgumentException("path [" + path + "] is a reserved path and may not be registered"); @@ -299,7 +262,7 @@ private void registerHandlerNoWrap(RestRequest.Method method, String path, RestA } public void registerHandler(final Route route, final RestHandler handler) { - if (route.isReplacement()) { + if (route.hasReplacement()) { Route replaced = route.getReplacedRoute(); registerAsReplacedHandler( route.getMethod(), @@ -308,7 +271,9 @@ public void registerHandler(final Route route, final RestHandler handler) { handler, replaced.getMethod(), replaced.getPath(), - replaced.getRestApiVersion() + replaced.getRestApiVersion(), + replaced.getDeprecationMessage(), + replaced.getDeprecationLevel() ); } else if (route.isDeprecated()) { registerAsDeprecatedHandler( diff --git a/server/src/main/java/org/elasticsearch/rest/RestHandler.java b/server/src/main/java/org/elasticsearch/rest/RestHandler.java index ede295fee9f4d..0e3b8d37dd25c 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/RestHandler.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.Level; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.RestRequest.Method; @@ -136,11 +137,10 @@ class Route { private final Method method; private final String path; private final RestApiVersion restApiVersion; - - private final String deprecationMessage; @Nullable + private final String deprecationMessage; private final Level deprecationLevel; - + @Nullable private final Route replacedRoute; private Route( @@ -153,12 +153,16 @@ private Route( ) { this.method = Objects.requireNonNull(method); this.path = Objects.requireNonNull(path); + // the last version in which this route was fully supported this.restApiVersion = Objects.requireNonNull(restApiVersion); - // a deprecated route will have a deprecation message, and the restApiVersion - // will represent the version when the route was deprecated + // a route marked as deprecated to keep or remove will have a deprecation message and level (warn for keep, critical for remove) this.deprecationMessage = deprecationMessage; - this.deprecationLevel = deprecationLevel; + this.deprecationLevel = Objects.requireNonNull(deprecationLevel); + + if (deprecationMessage == null && deprecationLevel != Level.OFF) { + throw new IllegalArgumentException("deprecationMessage must be set if deprecationLevel is not OFF"); + } // a route that replaces another route will have a reference to the route that was replaced this.replacedRoute = replacedRoute; @@ -173,7 +177,7 @@ private Route( * @param path the path, e.g. "/" */ public Route(Method method, String path) { - this(method, path, RestApiVersion.current(), null, null, null); + this(method, path, RestApiVersion.current(), null, Level.OFF, null); } public static class RouteBuilder { @@ -183,7 +187,6 @@ public static class RouteBuilder { private RestApiVersion restApiVersion; private String deprecationMessage; - @Nullable private Level deprecationLevel; private Route replacedRoute; @@ -194,6 +197,16 @@ private RouteBuilder(Method method, String path) { this.restApiVersion = RestApiVersion.current(); } + /** + * @deprecated Use {@link #deprecatedForRemoval(String, RestApiVersion)} if the intent is deprecate the path and remove in the + * next major version. Use {@link #deprecateAndKeep(String)} if the intent is to deprecate the path but not remove it. + * This method will delegate to {@link #deprecatedForRemoval(String, RestApiVersion)}. + */ + @Deprecated(since = "9.0.0", forRemoval = true) + public RouteBuilder deprecated(String deprecationMessage, RestApiVersion lastFullySupportedVersion) { + return deprecatedForRemoval(deprecationMessage, lastFullySupportedVersion); + } + /** * Marks that the route being built has been deprecated (for some reason -- the deprecationMessage) for removal. Notes the last * major version in which the path is fully supported without compatibility headers. If this path is being replaced by another @@ -202,7 +215,7 @@ private RouteBuilder(Method method, String path) { * For example: * 
 {@code
              * Route.builder(GET, "_upgrade")
-             *  .deprecated("The _upgrade API is no longer useful and will be removed.", RestApiVersion.V_7)
+             *  .deprecatedForRemoval("The _upgrade API is no longer useful and will be removed.", RestApiVersion.V_7)
              *  .build()}
* * @param deprecationMessage the user-visible explanation of this deprecation @@ -211,10 +224,12 @@ private RouteBuilder(Method method, String path) { * The next major version (i.e. 9) will have no support whatsoever for this route. * @return a reference to this object. */ - public RouteBuilder deprecated(String deprecationMessage, RestApiVersion lastFullySupportedVersion) { + public RouteBuilder deprecatedForRemoval(String deprecationMessage, RestApiVersion lastFullySupportedVersion) { assert this.replacedRoute == null; this.restApiVersion = Objects.requireNonNull(lastFullySupportedVersion); this.deprecationMessage = Objects.requireNonNull(deprecationMessage); + // if being deprecated for removal in the current version, then it's a warning, otherwise it's critical + this.deprecationLevel = lastFullySupportedVersion == RestApiVersion.current() ? Level.WARN : DeprecationLogger.CRITICAL; return this; } @@ -227,16 +242,38 @@ public RouteBuilder deprecated(String deprecationMessage, RestApiVersion lastFul * Route.builder(GET, "/_security/user/") * .replaces(GET, "/_xpack/security/user/", RestApiVersion.V_7).build()} * - * @param method the method being replaced - * @param path the path being replaced + * @param replacedMethod the method being replaced + * @param replacedPath the path being replaced * @param lastFullySupportedVersion the last {@link RestApiVersion} (i.e. 7) for which this route is fully supported. * The next major version (i.e. 8) will require compatibility header(s). (;compatible-with=7) * The next major version (i.e. 9) will have no support whatsoever for this route. * @return a reference to this object. */ - public RouteBuilder replaces(Method method, String path, RestApiVersion lastFullySupportedVersion) { + public RouteBuilder replaces(Method replacedMethod, String replacedPath, RestApiVersion lastFullySupportedVersion) { assert this.deprecationMessage == null; - this.replacedRoute = new Route(method, path, lastFullySupportedVersion, null, null, null); + + // if being replaced in the current version, then it's a warning, otherwise it's critical + Level deprecationLevel = lastFullySupportedVersion == RestApiVersion.current() ? Level.WARN : DeprecationLogger.CRITICAL; + + // e.g. [POST /_optimize] is deprecated! Use [POST /_forcemerge] instead. + final String replacedMessage = "[" + + replacedMethod.name() + + " " + + replacedPath + + "] is deprecated! Use [" + + this.method.name() + + " " + + this.path + + "] instead."; + + this.replacedRoute = new Route( + replacedMethod, + replacedPath, + lastFullySupportedVersion, + replacedMessage, + deprecationLevel, + null + ); return this; } @@ -246,7 +283,7 @@ public RouteBuilder replaces(Method method, String path, RestApiVersion lastFull * For example: * 
 {@code
              * Route.builder(GET, "_upgrade")
-             *  .deprecated("The _upgrade API is no longer useful but will not be removed.")
+             *  .deprecateAndKeep("The _upgrade API is no longer useful but will not be removed.")
              *  .build()}
* * @param deprecationMessage the user-visible explanation of this deprecation @@ -261,14 +298,15 @@ public RouteBuilder deprecateAndKeep(String deprecationMessage) { } public Route build() { - if (replacedRoute != null) { - return new Route(method, path, restApiVersion, null, null, replacedRoute); - } else if (deprecationMessage != null) { - return new Route(method, path, restApiVersion, deprecationMessage, deprecationLevel, null); - } else { - // this is a little silly, but perfectly legal - return new Route(method, path, restApiVersion, null, null, null); - } + assert (deprecationMessage != null) == (deprecationLevel != null); // both must be set or neither + return new Route( + method, + path, + restApiVersion, + deprecationMessage, + deprecationLevel == null ? Level.OFF : deprecationLevel, + replacedRoute + ); } } @@ -288,11 +326,11 @@ public RestApiVersion getRestApiVersion() { return restApiVersion; } + @Nullable public String getDeprecationMessage() { return deprecationMessage; } - @Nullable public Level getDeprecationLevel() { return deprecationLevel; } @@ -301,11 +339,12 @@ public boolean isDeprecated() { return deprecationMessage != null; } + @Nullable public Route getReplacedRoute() { return replacedRoute; } - public boolean isReplacement() { + public boolean hasReplacement() { return replacedRoute != null; } } diff --git a/server/src/main/java/org/elasticsearch/rest/RestStatus.java b/server/src/main/java/org/elasticsearch/rest/RestStatus.java index 72227b2d26ec0..569b63edda00b 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestStatus.java +++ b/server/src/main/java/org/elasticsearch/rest/RestStatus.java @@ -571,4 +571,16 @@ public static RestStatus status(int successfulShards, int totalShards, ShardOper public static RestStatus fromCode(int code) { return CODE_TO_STATUS.get(code); } + + /** + * Utility method to determine if an HTTP status code is "Successful" + * + * as defined by RFC 9110 + * + * @param code An HTTP status code + * @return true if it is a 2xx code, false otherwise + */ + public static boolean isSuccessful(int code) { + return code >= 200 && code < 300; + } } diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index be9ad0ed0a9cd..f1d4f678c5fb9 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateMathParser; +import org.elasticsearch.common.util.LocaleUtils; import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; @@ -236,9 +237,12 @@ private DateTime(DateFormatter formatter, ZoneId timeZone, DateFieldMapper.Resol public DateTime(StreamInput in) throws IOException { String formatterPattern = in.readString(); + Locale locale = in.getTransportVersion().onOrAfter(TransportVersions.DATE_TIME_DOC_VALUES_LOCALES) + ? LocaleUtils.parse(in.readString()) + : DateFieldMapper.DEFAULT_LOCALE; String zoneId = in.readString(); this.timeZone = ZoneId.of(zoneId); - this.formatter = DateFormatter.forPattern(formatterPattern).withZone(this.timeZone); + this.formatter = DateFormatter.forPattern(formatterPattern).withZone(this.timeZone).withLocale(locale); this.parser = formatter.toDateMathParser(); this.resolution = DateFieldMapper.Resolution.ofOrdinal(in.readVInt()); if (in.getTransportVersion().between(TransportVersions.V_7_7_0, TransportVersions.V_8_0_0)) { @@ -259,6 +263,9 @@ public String getWriteableName() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(formatter.pattern()); + if (out.getTransportVersion().onOrAfter(TransportVersions.DATE_TIME_DOC_VALUES_LOCALES)) { + out.writeString(formatter.locale().toString()); + } out.writeString(timeZone.getId()); out.writeVInt(resolution.ordinal()); if (out.getTransportVersion().between(TransportVersions.V_7_7_0, TransportVersions.V_8_0_0)) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java index cd4a0bac7f429..6fdd41d374d0e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java @@ -76,7 +76,7 @@ public InternalStats( } private void verifyFormattingStats() { - if (format != DocValueFormat.RAW) { + if (format != DocValueFormat.RAW && count != 0) { verifyFormattingStat(Fields.MIN, format, min); verifyFormattingStat(Fields.MAX, format, max); verifyFormattingStat(Fields.AVG, format, getAvg()); diff --git a/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java index 1328106896bcb..1c6f8c4a7ce44 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java @@ -251,11 +251,19 @@ public ActionRequestValidationException validate( @Override public final XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(); + builder.startObject(getName()); if (preFilterQueryBuilders.isEmpty() == false) { builder.field(PRE_FILTER_FIELD.getPreferredName(), preFilterQueryBuilders); } + if (minScore != null) { + builder.field(MIN_SCORE_FIELD.getPreferredName(), minScore); + } + if (retrieverName != null) { + builder.field(NAME_FIELD.getPreferredName(), retrieverName); + } doToXContent(builder, params); builder.endObject(); + builder.endObject(); return builder; } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java index 9b1d8c15619ad..8f0ff82beab4b 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java @@ -87,8 +87,7 @@ public Settings getAdditionalIndexSettings( xContentRegistry(), indicesService, systemIndices, - indexSettingsProviders, - Map.of() + indexSettingsProviders ); assertThat(resolvedTemplate.settings().getAsInt("test-setting", -1), is(1)); diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index 45c4b3bd2d7c7..5240d704dea3b 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -11,7 +11,10 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -19,6 +22,7 @@ import org.elasticsearch.search.AbstractSearchTestCase; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.test.ESTestCase; @@ -317,14 +321,12 @@ public void testExpandRequestOptions() throws IOException { @Override void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener listener) { final QueryBuilder postFilter = QueryBuilders.existsQuery("foo"); - assertTrue(request.requests().stream().allMatch((r) -> "foo".equals(r.preference()))); + assertTrue(request.requests().stream().allMatch((r) -> "foobar".equals(r.preference()))); assertTrue(request.requests().stream().allMatch((r) -> "baz".equals(r.routing()))); assertTrue(request.requests().stream().allMatch((r) -> version == r.source().version())); assertTrue(request.requests().stream().allMatch((r) -> seqNoAndTerm == r.source().seqNoAndPrimaryTerm())); assertTrue(request.requests().stream().allMatch((r) -> postFilter.equals(r.source().postFilter()))); - assertTrue(request.requests().stream().allMatch((r) -> r.source().fetchSource().fetchSource() == false)); - assertTrue(request.requests().stream().allMatch((r) -> r.source().fetchSource().includes().length == 0)); - assertTrue(request.requests().stream().allMatch((r) -> r.source().fetchSource().excludes().length == 0)); + assertTrue(request.requests().stream().allMatch((r) -> r.source().fetchSource() == null)); } }; mockSearchPhaseContext.getRequest() @@ -338,17 +340,72 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL .preference("foobar") .routing("baz"); - SearchHits hits = SearchHits.empty(new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0f); - ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, hits, () -> new SearchPhase("test") { + SearchHit hit = new SearchHit(1, "ID"); + hit.setDocumentField("someField", new DocumentField("someField", Collections.singletonList("foo"))); + SearchHits hits = new SearchHits(new SearchHit[] { hit }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F); + try { + ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, hits, () -> new SearchPhase("test") { + @Override + public void run() { + mockSearchPhaseContext.sendSearchResponse(new SearchResponseSections(hits, null, null, false, null, null, 1), null); + } + }); + phase.run(); + mockSearchPhaseContext.assertNoFailure(); + } finally { + hits.decRef(); + } + } finally { + var resp = mockSearchPhaseContext.searchResponse.get(); + if (resp != null) { + resp.decRef(); + } + } + } + + public void testExpandSearchRespectsOriginalPIT() { + MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1); + final PointInTimeBuilder pit = new PointInTimeBuilder(new BytesArray("foo")); + try { + boolean version = randomBoolean(); + final boolean seqNoAndTerm = randomBoolean(); + + mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null, null) { @Override - public void run() { - mockSearchPhaseContext.sendSearchResponse(new SearchResponseSections(hits, null, null, false, null, null, 1), null); + void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener listener) { + assertTrue(request.requests().stream().allMatch((r) -> r.preference() == null)); + assertTrue(request.requests().stream().allMatch((r) -> r.indices() == Strings.EMPTY_ARRAY)); + assertTrue(request.requests().stream().allMatch((r) -> r.source().pointInTimeBuilder().equals(pit))); } - }); - phase.run(); - mockSearchPhaseContext.assertNoFailure(); - assertNotNull(mockSearchPhaseContext.searchResponse.get()); - mockSearchPhaseContext.execute(() -> {}); + }; + mockSearchPhaseContext.getRequest() + .source( + new SearchSourceBuilder().collapse( + new CollapseBuilder("someField").setInnerHits( + new InnerHitBuilder().setName("foobarbaz").setVersion(version).setSeqNoAndPrimaryTerm(seqNoAndTerm) + ) + ).fetchSource(false).postFilter(QueryBuilders.existsQuery("foo")).pointInTimeBuilder(pit) + ) + .routing("baz"); + + SearchHit hit = new SearchHit(1, "ID"); + hit.setDocumentField("someField", new DocumentField("someField", Collections.singletonList("foo"))); + SearchHits hits = new SearchHits(new SearchHit[] { hit }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F); + try { + ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, hits, () -> new SearchPhase("test") { + @Override + public void run() { + mockSearchPhaseContext.sendSearchResponse( + new SearchResponseSections(hits, null, null, false, null, null, 1), + new AtomicArray<>(0) + ); + } + }); + phase.run(); + mockSearchPhaseContext.assertNoFailure(); + } finally { + hits.decRef(); + } } finally { var resp = mockSearchPhaseContext.searchResponse.get(); if (resp != null) { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 5fadd8f263f7c..8d4b04746e7a4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -1074,7 +1074,7 @@ public void testResolveConflictingMappings() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "my-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", Map.of(), "my-index"); + List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(3)); @@ -1136,7 +1136,7 @@ public void testResolveMappings() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "my-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", Map.of(), "my-index"); + List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(3)); @@ -1190,7 +1190,6 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex List mappings = MetadataIndexTemplateService.collectMappings( state, "logs-data-stream-template", - Map.of(), DataStream.getDefaultBackingIndexName("logs", 1L) ); @@ -1242,12 +1241,7 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex .build(); state = service.addIndexTemplateV2(state, true, "timeseries-template", it); - List mappings = MetadataIndexTemplateService.collectMappings( - state, - "timeseries-template", - Map.of(), - "timeseries" - ); + List mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template", "timeseries"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(2)); @@ -1269,7 +1263,6 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex mappings = MetadataIndexTemplateService.collectMappings( state, "timeseries-template", - Map.of(), DataStream.getDefaultBackingIndexName("timeseries", 1L) ); @@ -1318,7 +1311,6 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception List mappings = MetadataIndexTemplateService.collectMappings( state, "logs-template", - Map.of(), DataStream.getDefaultBackingIndexName("logs", 1L) ); @@ -1375,7 +1367,6 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception List mappings = MetadataIndexTemplateService.collectMappings( state, "timeseries-template", - Map.of(), DataStream.getDefaultBackingIndexName("timeseries-template", 1L) ); @@ -2442,12 +2433,7 @@ public void testComposableTemplateWithSubobjectsFalse() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "composable-template", it); - List mappings = MetadataIndexTemplateService.collectMappings( - state, - "composable-template", - Map.of(), - "test-index" - ); + List mappings = MetadataIndexTemplateService.collectMappings(state, "composable-template", "test-index"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(2)); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTableTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTableTests.java index 21b30557cafea..6a7f4bb27a324 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTableTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTableTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.cluster.routing; +import org.elasticsearch.TransportVersion; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; @@ -19,6 +20,7 @@ import java.util.List; +import static org.elasticsearch.TransportVersions.FAST_REFRESH_RCO; import static org.elasticsearch.index.IndexSettings.INDEX_FAST_REFRESH_SETTING; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; @@ -27,16 +29,22 @@ public class IndexRoutingTableTests extends ESTestCase { public void testReadyForSearch() { - innerReadyForSearch(false); - innerReadyForSearch(true); + innerReadyForSearch(false, false); + innerReadyForSearch(false, true); + innerReadyForSearch(true, false); + innerReadyForSearch(true, true); } - private void innerReadyForSearch(boolean fastRefresh) { + // TODO: remove if (fastRefresh && beforeFastRefreshRCO) branches (ES-9563) + private void innerReadyForSearch(boolean fastRefresh, boolean beforeFastRefreshRCO) { Index index = new Index(randomIdentifier(), UUIDs.randomBase64UUID()); ClusterState clusterState = mock(ClusterState.class, Mockito.RETURNS_DEEP_STUBS); when(clusterState.metadata().index(any(Index.class)).getSettings()).thenReturn( Settings.builder().put(INDEX_FAST_REFRESH_SETTING.getKey(), fastRefresh).build() ); + when(clusterState.getMinTransportVersion()).thenReturn( + beforeFastRefreshRCO ? TransportVersion.fromId(FAST_REFRESH_RCO.id() - 1_00_0) : TransportVersion.current() + ); // 2 primaries that are search and index ShardId p1 = new ShardId(index, 0); IndexShardRoutingTable shardTable1 = new IndexShardRoutingTable( @@ -55,7 +63,7 @@ private void innerReadyForSearch(boolean fastRefresh) { shardTable1 = new IndexShardRoutingTable(p1, List.of(getShard(p1, true, ShardRoutingState.STARTED, ShardRouting.Role.INDEX_ONLY))); shardTable2 = new IndexShardRoutingTable(p2, List.of(getShard(p2, true, ShardRoutingState.STARTED, ShardRouting.Role.INDEX_ONLY))); indexRoutingTable = new IndexRoutingTable(index, new IndexShardRoutingTable[] { shardTable1, shardTable2 }); - if (fastRefresh) { + if (fastRefresh && beforeFastRefreshRCO) { assertTrue(indexRoutingTable.readyForSearch(clusterState)); } else { assertFalse(indexRoutingTable.readyForSearch(clusterState)); @@ -91,7 +99,7 @@ private void innerReadyForSearch(boolean fastRefresh) { ) ); indexRoutingTable = new IndexRoutingTable(index, new IndexShardRoutingTable[] { shardTable1, shardTable2 }); - if (fastRefresh) { + if (fastRefresh && beforeFastRefreshRCO) { assertTrue(indexRoutingTable.readyForSearch(clusterState)); } else { assertFalse(indexRoutingTable.readyForSearch(clusterState)); @@ -118,8 +126,6 @@ private void innerReadyForSearch(boolean fastRefresh) { assertTrue(indexRoutingTable.readyForSearch(clusterState)); // 2 unassigned primaries that are index only with some replicas that are all available - // Fast refresh indices do not support replicas so this can not practically happen. If we add support we will want to ensure - // that readyForSearch allows for searching replicas when the index shard is not available. shardTable1 = new IndexShardRoutingTable( p1, List.of( @@ -137,8 +143,8 @@ private void innerReadyForSearch(boolean fastRefresh) { ) ); indexRoutingTable = new IndexRoutingTable(index, new IndexShardRoutingTable[] { shardTable1, shardTable2 }); - if (fastRefresh) { - assertFalse(indexRoutingTable.readyForSearch(clusterState)); // if we support replicas for fast refreshes this needs to change + if (fastRefresh && beforeFastRefreshRCO) { + assertFalse(indexRoutingTable.readyForSearch(clusterState)); } else { assertTrue(indexRoutingTable.readyForSearch(clusterState)); } diff --git a/server/src/test/java/org/elasticsearch/common/util/BigArraysTests.java b/server/src/test/java/org/elasticsearch/common/util/BigArraysTests.java index fc3096de8b8c0..3ef010f760ab6 100644 --- a/server/src/test/java/org/elasticsearch/common/util/BigArraysTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/BigArraysTests.java @@ -135,6 +135,7 @@ public void testObjectArrayGrowth() { ref[i] = randomFrom(pool); array = bigArrays.grow(array, i + 1); array.set(i, ref[i]); + assertEquals(ref[i], array.getAndSet(i, ref[i])); } for (int i = 0; i < totalLen; ++i) { assertSame(ref[i], array.get(i)); diff --git a/server/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java b/server/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java index 77635fd0312f8..4cb3ce418f761 100644 --- a/server/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java +++ b/server/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java @@ -276,7 +276,7 @@ public void testShouldLoadRandomAccessFiltersEagerly() { for (var isStateless : values) { if (isStateless) { assertEquals( - loadFiltersEagerly && indexFastRefresh && hasIndexRole, + loadFiltersEagerly && indexFastRefresh && hasIndexRole == false, BitsetFilterCache.shouldLoadRandomAccessFiltersEagerly( bitsetFilterCacheSettings(isStateless, hasIndexRole, loadFiltersEagerly, indexFastRefresh) ) diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 883723de31d46..c8ca3d17de797 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -55,7 +55,7 @@ import org.apache.lucene.search.SortedSetSortField; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TotalHitCountCollector; +import org.apache.lucene.search.TotalHitCountCollectorManager; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.Lock; @@ -640,9 +640,8 @@ public void testTranslogMultipleOperationsSameDocument() throws IOException { recoverFromTranslog(recoveringEngine, translogHandler, Long.MAX_VALUE); recoveringEngine.refresh("test"); try (Engine.Searcher searcher = recoveringEngine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new MatchAllDocsQuery(), collector); - assertThat(collector.getTotalHits(), equalTo(operations.get(operations.size() - 1) instanceof Engine.Delete ? 0 : 1)); + Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager()); + assertThat(totalHits, equalTo(operations.get(operations.size() - 1) instanceof Engine.Delete ? 0 : 1)); } } } @@ -2009,16 +2008,20 @@ public void testConcurrentOutOfOrderDocsOnReplica() throws IOException, Interrup if (lastFieldValueDoc1 != null) { try (Engine.Searcher searcher = engine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new TermQuery(new Term("value", lastFieldValueDoc1)), collector); - assertThat(collector.getTotalHits(), equalTo(1)); + Integer totalHits = searcher.search( + new TermQuery(new Term("value", lastFieldValueDoc1)), + new TotalHitCountCollectorManager() + ); + assertThat(totalHits, equalTo(1)); } } if (lastFieldValueDoc2 != null) { try (Engine.Searcher searcher = engine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new TermQuery(new Term("value", lastFieldValueDoc2)), collector); - assertThat(collector.getTotalHits(), equalTo(1)); + Integer totalHits = searcher.search( + new TermQuery(new Term("value", lastFieldValueDoc2)), + new TotalHitCountCollectorManager() + ); + assertThat(totalHits, equalTo(1)); } } @@ -2244,9 +2247,11 @@ private int assertOpsOnPrimary(List ops, long currentOpVersion // first op and it failed. if (docDeleted == false && lastFieldValue != null) { try (Engine.Searcher searcher = engine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new TermQuery(new Term("value", lastFieldValue)), collector); - assertThat(collector.getTotalHits(), equalTo(1)); + Integer totalHits = searcher.search( + new TermQuery(new Term("value", lastFieldValue)), + new TotalHitCountCollectorManager() + ); + assertThat(totalHits, equalTo(1)); } } } @@ -2270,9 +2275,8 @@ private int assertOpsOnPrimary(List ops, long currentOpVersion assertVisibleCount(engine, docDeleted ? 0 : 1); if (docDeleted == false) { try (Engine.Searcher searcher = engine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new TermQuery(new Term("value", lastFieldValue)), collector); - assertThat(collector.getTotalHits(), equalTo(1)); + Integer totalHits = searcher.search(new TermQuery(new Term("value", lastFieldValue)), new TotalHitCountCollectorManager()); + assertThat(totalHits, equalTo(1)); } } return opsPerformed; @@ -2357,9 +2361,8 @@ public void testNonInternalVersioningOnPrimary() throws IOException { if (docDeleted == false) { logger.info("searching for [{}]", lastFieldValue); try (Engine.Searcher searcher = engine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new TermQuery(new Term("value", lastFieldValue)), collector); - assertThat(collector.getTotalHits(), equalTo(1)); + Integer totalHits = searcher.search(new TermQuery(new Term("value", lastFieldValue)), new TotalHitCountCollectorManager()); + assertThat(totalHits, equalTo(1)); } } } @@ -2375,9 +2378,8 @@ public void testVersioningPromotedReplica() throws IOException { final int opsOnPrimary = assertOpsOnPrimary(primaryOps, finalReplicaVersion, deletedOnReplica, replicaEngine); final long currentSeqNo = getSequenceID(replicaEngine, new Engine.Get(false, false, Term.toString(lastReplicaOp.uid()))).v1(); try (Engine.Searcher searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new MatchAllDocsQuery(), collector); - if (collector.getTotalHits() > 0) { + Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager()); + if (totalHits > 0) { // last op wasn't delete assertThat(currentSeqNo, equalTo(finalReplicaSeqNo + opsOnPrimary)); } @@ -2400,9 +2402,8 @@ public void testConcurrentExternalVersioningOnPrimary() throws IOException, Inte assertVisibleCount(engine, lastFieldValue == null ? 0 : 1); if (lastFieldValue != null) { try (Engine.Searcher searcher = engine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new TermQuery(new Term("value", lastFieldValue)), collector); - assertThat(collector.getTotalHits(), equalTo(1)); + Integer totalHits = searcher.search(new TermQuery(new Term("value", lastFieldValue)), new TotalHitCountCollectorManager()); + assertThat(totalHits, equalTo(1)); } } } diff --git a/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java index bc2ba833536ff..3890b53e29ffb 100644 --- a/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java @@ -9,13 +9,12 @@ package org.elasticsearch.index.query; -import org.apache.lucene.index.Term; import org.apache.lucene.queries.spans.FieldMaskingSpanQuery; -import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; import org.elasticsearch.common.ParsingException; import org.elasticsearch.core.Strings; +import org.elasticsearch.lucene.queries.SpanMatchNoDocsQuery; import org.elasticsearch.test.AbstractQueryTestCase; import java.io.IOException; @@ -105,7 +104,7 @@ public void testJsonWithTopLevelBoost() throws IOException { } }""", NAME.getPreferredName()); Query q = parseQuery(json).toQuery(createSearchExecutionContext()); - assertEquals(new BoostQuery(new FieldMaskingSpanQuery(new SpanTermQuery(new Term("value", "foo")), "mapped_geo_shape"), 42.0f), q); + assertEquals(new BoostQuery(new FieldMaskingSpanQuery(new SpanMatchNoDocsQuery("value", null), "mapped_geo_shape"), 42.0f), q); } public void testJsonWithDeprecatedName() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanGapQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanGapQueryBuilderTests.java index cef43a635541e..5adca6d562dca 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanGapQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanGapQueryBuilderTests.java @@ -13,6 +13,7 @@ import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.Query; +import org.elasticsearch.lucene.queries.SpanMatchNoDocsQuery; import org.elasticsearch.test.AbstractQueryTestCase; import java.io.IOException; @@ -50,7 +51,9 @@ protected SpanNearQueryBuilder doCreateTestQueryBuilder() { protected void doAssertLuceneQuery(SpanNearQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException { assertThat( query, - either(instanceOf(SpanNearQuery.class)).or(instanceOf(SpanTermQuery.class)).or(instanceOf(MatchAllQueryBuilder.class)) + either(instanceOf(SpanNearQuery.class)).or(instanceOf(SpanTermQuery.class)) + .or(instanceOf(MatchAllQueryBuilder.class)) + .or(instanceOf(SpanMatchNoDocsQuery.class)) ); if (query instanceof SpanNearQuery spanNearQuery) { assertThat(spanNearQuery.getSlop(), equalTo(queryBuilder.slop())); diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java index f0f23d8539e13..c4a9267ff68a0 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java @@ -15,9 +15,9 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.lucene.queries.SpanMatchNoDocsQuery; import org.elasticsearch.xcontent.json.JsonStringEncoder; import java.io.IOException; @@ -49,18 +49,16 @@ protected SpanTermQueryBuilder createQueryBuilder(String fieldName, Object value @Override protected void doAssertLuceneQuery(SpanTermQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException { - assertThat(query, instanceOf(SpanTermQuery.class)); - SpanTermQuery spanTermQuery = (SpanTermQuery) query; - - String expectedFieldName = expectedFieldName(queryBuilder.fieldName); - assertThat(spanTermQuery.getTerm().field(), equalTo(expectedFieldName)); - MappedFieldType mapper = context.getFieldType(queryBuilder.fieldName()); if (mapper != null) { + String expectedFieldName = expectedFieldName(queryBuilder.fieldName); + assertThat(query, instanceOf(SpanTermQuery.class)); + SpanTermQuery spanTermQuery = (SpanTermQuery) query; + assertThat(spanTermQuery.getTerm().field(), equalTo(expectedFieldName)); Term term = ((TermQuery) mapper.termQuery(queryBuilder.value(), null)).getTerm(); assertThat(spanTermQuery.getTerm(), equalTo(term)); } else { - assertThat(spanTermQuery.getTerm().bytes(), equalTo(BytesRefs.toBytesRef(queryBuilder.value()))); + assertThat(query, instanceOf(SpanMatchNoDocsQuery.class)); } } @@ -117,23 +115,13 @@ public void testParseFailsWithMultipleFields() throws IOException { assertEquals("[span_term] query doesn't support multiple fields, found [message1] and [message2]", e.getMessage()); } - public void testWithMetadataField() throws IOException { - SearchExecutionContext context = createSearchExecutionContext(); - for (String field : new String[] { "field1", "field2" }) { - SpanTermQueryBuilder spanTermQueryBuilder = new SpanTermQueryBuilder(field, "toto"); - Query query = spanTermQueryBuilder.toQuery(context); - Query expected = new SpanTermQuery(new Term(field, "toto")); - assertEquals(expected, query); - } - } - public void testWithBoost() throws IOException { SearchExecutionContext context = createSearchExecutionContext(); - for (String field : new String[] { "field1", "field2" }) { + for (String field : new String[] { TEXT_FIELD_NAME, TEXT_ALIAS_FIELD_NAME }) { SpanTermQueryBuilder spanTermQueryBuilder = new SpanTermQueryBuilder(field, "toto"); spanTermQueryBuilder.boost(10); Query query = spanTermQueryBuilder.toQuery(context); - Query expected = new BoostQuery(new SpanTermQuery(new Term(field, "toto")), 10); + Query expected = new BoostQuery(new SpanTermQuery(new Term(TEXT_FIELD_NAME, "toto")), 10); assertEquals(expected, query); } } diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 49e75a71aa7f7..3adaf398624de 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -32,20 +32,16 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; -import org.elasticsearch.cluster.metadata.ComponentTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.ClusterStateTaskExecutorUtils; import org.elasticsearch.common.TriConsumer; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.util.Maps; @@ -77,7 +73,6 @@ import org.mockito.ArgumentMatcher; import org.mockito.invocation.InvocationOnMock; -import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -2493,7 +2488,7 @@ public void testResolveFinalPipelineWithDateMathExpression() { // index name matches with IDM: IndexRequest indexRequest = new IndexRequest(""); - IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, epochMillis, Map.of()); + IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, epochMillis); assertTrue(hasPipeline(indexRequest)); assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("_none")); @@ -2858,83 +2853,6 @@ public void testResolvePipelinesWithNonePipeline() { } } - public void testResolvePipelinesAndUpdateIndexRequestWithComponentTemplateSubstitutions() throws IOException { - final String componentTemplateName = "test-component-template"; - final String indexName = "my-index-1"; - final String indexPipeline = "index-pipeline"; - final String realTemplatePipeline = "template-pipeline"; - final String substitutePipeline = "substitute-pipeline"; - - Metadata metadata; - { - // Build up cluster state metadata - IndexMetadata.Builder builder = IndexMetadata.builder(indexName) - .settings(settings(IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0); - ComponentTemplate realComponentTemplate = new ComponentTemplate( - new Template( - Settings.builder().put("index.default_pipeline", realTemplatePipeline).build(), - CompressedXContent.fromJSON("{}"), - null - ), - null, - null - ); - ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.builder() - .indexPatterns(List.of("my-index-*")) - .componentTemplates(List.of(componentTemplateName)) - .build(); - metadata = Metadata.builder() - .put(builder) - .indexTemplates(Map.of("my-index-template", composableIndexTemplate)) - .componentTemplates(Map.of("test-component-template", realComponentTemplate)) - .build(); - } - - Map componentTemplateSubstitutions; - { - ComponentTemplate simulatedComponentTemplate = new ComponentTemplate( - new Template( - Settings.builder().put("index.default_pipeline", substitutePipeline).build(), - CompressedXContent.fromJSON("{}"), - null - ), - null, - null - ); - componentTemplateSubstitutions = Map.of(componentTemplateName, simulatedComponentTemplate); - } - - { - /* - * Here there is a pipeline in the request. This takes precedence over anything in the index or templates or component template - * substitutions. - */ - IndexRequest indexRequest = new IndexRequest(indexName).setPipeline(indexPipeline); - IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, 0, componentTemplateSubstitutions); - assertThat(indexRequest.getPipeline(), equalTo(indexPipeline)); - } - { - /* - * Here there is no pipeline in the request, but there is one in the substitute component template. So it takes precedence. - */ - IndexRequest indexRequest = new IndexRequest(indexName); - IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, 0, componentTemplateSubstitutions); - assertThat(indexRequest.getPipeline(), equalTo(substitutePipeline)); - } - { - /* - * This one is tricky. Since the index exists and there are no component template substitutions, we're going to use the actual - * index in this case rather than its template. The index does not have a default pipeline set, so it's "_none" instead of - * realTemplatePipeline. - */ - IndexRequest indexRequest = new IndexRequest(indexName); - IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, 0, Map.of()); - assertThat(indexRequest.getPipeline(), equalTo("_none")); - } - } - private static Tuple randomMapEntry() { return tuple(randomAlphaOfLength(5), randomObject()); } diff --git a/server/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java b/server/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java index 4e0fe14fb1def..b534a6be0dc5f 100644 --- a/server/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java @@ -73,7 +73,7 @@ public void testHandleRequestLogsThenForwards() throws Exception { RestChannel channel = mock(RestChannel.class); NodeClient client = mock(NodeClient.class); - final Level deprecationLevel = randomBoolean() ? null : randomFrom(Level.WARN, DeprecationLogger.CRITICAL); + final Level deprecationLevel = randomFrom(Level.WARN, DeprecationLogger.CRITICAL); DeprecationRestHandler deprecatedHandler = new DeprecationRestHandler( handler, @@ -159,17 +159,55 @@ public void testInvalidHeaderValueEmpty() { public void testSupportsBulkContentTrue() { when(handler.supportsBulkContent()).thenReturn(true); assertTrue( - new DeprecationRestHandler(handler, METHOD, PATH, null, deprecationMessage, deprecationLogger, false).supportsBulkContent() + new DeprecationRestHandler(handler, METHOD, PATH, Level.WARN, deprecationMessage, deprecationLogger, false) + .supportsBulkContent() ); } public void testSupportsBulkContentFalse() { when(handler.supportsBulkContent()).thenReturn(false); assertFalse( - new DeprecationRestHandler(handler, METHOD, PATH, null, deprecationMessage, deprecationLogger, false).supportsBulkContent() + new DeprecationRestHandler(handler, METHOD, PATH, Level.WARN, deprecationMessage, deprecationLogger, false) + .supportsBulkContent() ); } + public void testDeprecationLevel() { + DeprecationRestHandler handler = new DeprecationRestHandler( + this.handler, + METHOD, + PATH, + Level.WARN, + deprecationMessage, + deprecationLogger, + false + ); + assertEquals(Level.WARN, handler.getDeprecationLevel()); + + handler = new DeprecationRestHandler( + this.handler, + METHOD, + PATH, + DeprecationLogger.CRITICAL, + deprecationMessage, + deprecationLogger, + false + ); + assertEquals(DeprecationLogger.CRITICAL, handler.getDeprecationLevel()); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> new DeprecationRestHandler(this.handler, METHOD, PATH, null, deprecationMessage, deprecationLogger, false) + ); + assertEquals(exception.getMessage(), "unexpected deprecation logger level: null, expected either 'CRITICAL' or 'WARN'"); + + exception = expectThrows( + IllegalArgumentException.class, + () -> new DeprecationRestHandler(this.handler, METHOD, PATH, Level.OFF, deprecationMessage, deprecationLogger, false) + ); + assertEquals(exception.getMessage(), "unexpected deprecation logger level: OFF, expected either 'CRITICAL' or 'WARN'"); + } + /** * {@code ASCIIHeaderGenerator} only uses characters expected to be valid in headers (simplified US-ASCII). */ diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java index 1d946681661e7..8f1904ce42438 100644 --- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java +++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.rest; +import org.apache.logging.log4j.Level; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.breaker.CircuitBreaker; @@ -17,6 +18,7 @@ import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.io.stream.BytesStream; import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; @@ -85,6 +87,7 @@ import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -349,18 +352,22 @@ public void testRegisterAsDeprecatedHandler() { String path = "/_" + randomAlphaOfLengthBetween(1, 6); RestHandler handler = (request, channel, client) -> {}; String deprecationMessage = randomAlphaOfLengthBetween(1, 10); - RestApiVersion deprecatedInVersion = RestApiVersion.current(); - Route route = Route.builder(method, path).deprecated(deprecationMessage, deprecatedInVersion).build(); + List replacedInVersions = List.of(RestApiVersion.current(), RestApiVersion.minimumSupported()); + for (RestApiVersion replacedInVersion : replacedInVersions) { + Level level = replacedInVersion == RestApiVersion.current() ? Level.WARN : DeprecationLogger.CRITICAL; + clearInvocations(controller); + Route route = Route.builder(method, path).deprecatedForRemoval(deprecationMessage, replacedInVersion).build(); - // don't want to test everything -- just that it actually wraps the handler - doCallRealMethod().when(controller).registerHandler(route, handler); - doCallRealMethod().when(controller) - .registerAsDeprecatedHandler(method, path, deprecatedInVersion, handler, deprecationMessage, null); + // don't want to test everything -- just that it actually wraps the handler + doCallRealMethod().when(controller).registerHandler(route, handler); + doCallRealMethod().when(controller) + .registerAsDeprecatedHandler(method, path, replacedInVersion, handler, deprecationMessage, level); - controller.registerHandler(route, handler); + controller.registerHandler(route, handler); - verify(controller).registerHandler(eq(method), eq(path), eq(deprecatedInVersion), any(DeprecationRestHandler.class)); + verify(controller).registerHandler(eq(method), eq(path), eq(replacedInVersion), any(DeprecationRestHandler.class)); + } } public void testRegisterAsReplacedHandler() { @@ -383,17 +390,40 @@ public void testRegisterAsReplacedHandler() { + path + "] instead."; - final Route route = Route.builder(method, path).replaces(replacedMethod, replacedPath, previous).build(); - - // don't want to test everything -- just that it actually wraps the handlers - doCallRealMethod().when(controller).registerHandler(route, handler); - doCallRealMethod().when(controller) - .registerAsReplacedHandler(method, path, current, handler, replacedMethod, replacedPath, previous); - - controller.registerHandler(route, handler); + List replacedInVersions = List.of(current, previous); + for (RestApiVersion replacedInVersion : replacedInVersions) { + clearInvocations(controller); + Route route = Route.builder(method, path).replaces(replacedMethod, replacedPath, replacedInVersion).build(); + // don't want to test everything -- just that it actually wraps the handler + doCallRealMethod().when(controller).registerHandler(route, handler); + Level level = replacedInVersion == current ? Level.WARN : DeprecationLogger.CRITICAL; + doCallRealMethod().when(controller) + .registerAsReplacedHandler( + method, + path, + current, + handler, + replacedMethod, + replacedPath, + replacedInVersion, + deprecationMessage, + level + ); - verify(controller).registerHandler(method, path, current, handler); - verify(controller).registerAsDeprecatedHandler(replacedMethod, replacedPath, previous, handler, deprecationMessage); + controller.registerHandler(route, handler); + + // verify we registered the primary handler + verify(controller).registerHandler(method, path, current, handler); + // verify we register the replaced handler with the correct deprecation message and level + verify(controller).registerAsDeprecatedHandler( + replacedMethod, + replacedPath, + replacedInVersion, + handler, + deprecationMessage, + level + ); + } } public void testRegisterSecondMethodWithDifferentNamedWildcard() { diff --git a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java index fff5dcb4bb80b..5e1296c354015 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java @@ -22,7 +22,7 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.TotalHitCountCollector; +import org.apache.lucene.search.TotalHitCountCollectorManager; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.tests.util.TestUtil; @@ -96,7 +96,6 @@ public void testAddingCancellationActions() throws IOException { } public void testCancellableCollector() throws IOException { - TotalHitCountCollector collector1 = new TotalHitCountCollector(); Runnable cancellation = () -> { throw new TaskCancelledException("cancelled"); }; ContextIndexSearcher searcher = new ContextIndexSearcher( reader, @@ -106,16 +105,15 @@ public void testCancellableCollector() throws IOException { true ); - searcher.search(new MatchAllDocsQuery(), collector1); - assertThat(collector1.getTotalHits(), equalTo(reader.numDocs())); + Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager()); + assertThat(totalHits, equalTo(reader.numDocs())); searcher.addQueryCancellation(cancellation); - expectThrows(TaskCancelledException.class, () -> searcher.search(new MatchAllDocsQuery(), collector1)); + expectThrows(TaskCancelledException.class, () -> searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager())); searcher.removeQueryCancellation(cancellation); - TotalHitCountCollector collector2 = new TotalHitCountCollector(); - searcher.search(new MatchAllDocsQuery(), collector2); - assertThat(collector2.getTotalHits(), equalTo(reader.numDocs())); + Integer totalHits2 = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager()); + assertThat(totalHits2, equalTo(reader.numDocs())); } public void testExitableDirectoryReader() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java index 8b8a4f97d540e..ae4ed3568683a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java @@ -14,16 +14,20 @@ import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import java.io.IOException; +import java.util.Map; import java.util.function.Consumer; import static java.util.Collections.singleton; +import static org.elasticsearch.search.aggregations.AggregationBuilders.stats; public class ExtendedStatsAggregatorTests extends AggregatorTestCase { private static final double TOLERANCE = 1e-5; @@ -49,6 +53,37 @@ public void testEmpty() throws IOException { }); } + public void testEmptyDate() throws IOException { + DateFormatter.forPattern("epoch_millis"); + final MappedFieldType ft = new DateFieldMapper.DateFieldType( + "field", + true, + true, + false, + true, + DateFormatter.forPattern("epoch_millis"), + DateFieldMapper.Resolution.MILLISECONDS, + null, + null, + Map.of() + ); + testCase(ft, iw -> {}, stats -> { + assertEquals(0d, stats.getCount(), 0); + assertEquals(0d, stats.getSum(), 0); + assertEquals(Float.NaN, stats.getAvg(), 0); + assertEquals(Double.POSITIVE_INFINITY, stats.getMin(), 0); + assertEquals(Double.NEGATIVE_INFINITY, stats.getMax(), 0); + assertEquals(Double.NaN, stats.getVariance(), 0); + assertEquals(Double.NaN, stats.getVariancePopulation(), 0); + assertEquals(Double.NaN, stats.getVarianceSampling(), 0); + assertEquals(Double.NaN, stats.getStdDeviation(), 0); + assertEquals(Double.NaN, stats.getStdDeviationPopulation(), 0); + assertEquals(Double.NaN, stats.getStdDeviationSampling(), 0); + assertEquals(0d, stats.getSumOfSquares(), 0); + assertFalse(AggregationInspectionHelper.hasValue(stats)); + }); + } + public void testRandomDoubles() throws IOException { MappedFieldType ft = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.DOUBLE); final ExtendedSimpleStatsAggregator expected = new ExtendedSimpleStatsAggregator(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java index 9f48cb2279320..ddd1fa987ad2b 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java @@ -18,7 +18,9 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; @@ -73,6 +75,30 @@ public void testEmpty() throws IOException { }, ft); } + public void testEmptyDate() throws IOException { + DateFormatter.forPattern("epoch_millis"); + final MappedFieldType ft = new DateFieldMapper.DateFieldType( + "field", + true, + true, + false, + true, + DateFormatter.forPattern("epoch_millis"), + DateFieldMapper.Resolution.MILLISECONDS, + null, + null, + Map.of() + ); + testCase(stats("_name").field(ft.name()), iw -> {}, stats -> { + assertEquals(0d, stats.getCount(), 0); + assertEquals(0d, stats.getSum(), 0); + assertEquals(Float.NaN, stats.getAvg(), 0); + assertEquals(Double.POSITIVE_INFINITY, stats.getMin(), 0); + assertEquals(Double.NEGATIVE_INFINITY, stats.getMax(), 0); + assertFalse(AggregationInspectionHelper.hasValue(stats)); + }, ft); + } + public void testRandomDoubles() throws IOException { final MappedFieldType ft = new NumberFieldMapper.NumberFieldType("field", NumberType.DOUBLE); final SimpleStatsAggregator expected = new SimpleStatsAggregator(); diff --git a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index 3f33bbfe6f6cb..240a677f4cbfd 100644 --- a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -41,6 +41,8 @@ import org.elasticsearch.search.collapse.CollapseBuilderTests; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.rescore.QueryRescorerBuilder; +import org.elasticsearch.search.retriever.KnnRetrieverBuilder; +import org.elasticsearch.search.retriever.StandardRetrieverBuilder; import org.elasticsearch.search.slice.SliceBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; @@ -600,6 +602,75 @@ public void testNegativeTrackTotalHits() throws IOException { } } + public void testStandardRetrieverParsing() throws IOException { + String restContent = "{" + + " \"retriever\": {" + + " \"standard\": {" + + " \"query\": {" + + " \"match_all\": {}" + + " }," + + " \"min_score\": 10," + + " \"_name\": \"foo_standard\"" + + " }" + + " }" + + "}"; + SearchUsageHolder searchUsageHolder = new UsageService().getSearchUsageHolder(); + try (XContentParser jsonParser = createParser(JsonXContent.jsonXContent, restContent)) { + SearchSourceBuilder source = new SearchSourceBuilder().parseXContent(jsonParser, true, searchUsageHolder, nf -> true); + assertThat(source.retriever(), instanceOf(StandardRetrieverBuilder.class)); + StandardRetrieverBuilder parsed = (StandardRetrieverBuilder) source.retriever(); + assertThat(parsed.minScore(), equalTo(10f)); + assertThat(parsed.retrieverName(), equalTo("foo_standard")); + try (XContentParser parseSerialized = createParser(JsonXContent.jsonXContent, Strings.toString(source))) { + SearchSourceBuilder deserializedSource = new SearchSourceBuilder().parseXContent( + parseSerialized, + true, + searchUsageHolder, + nf -> true + ); + assertThat(deserializedSource.retriever(), instanceOf(StandardRetrieverBuilder.class)); + StandardRetrieverBuilder deserialized = (StandardRetrieverBuilder) source.retriever(); + assertThat(parsed, equalTo(deserialized)); + } + } + } + + public void testKnnRetrieverParsing() throws IOException { + String restContent = "{" + + " \"retriever\": {" + + " \"knn\": {" + + " \"query_vector\": [" + + " 3" + + " ]," + + " \"field\": \"vector\"," + + " \"k\": 10," + + " \"num_candidates\": 15," + + " \"min_score\": 10," + + " \"_name\": \"foo_knn\"" + + " }" + + " }" + + "}"; + SearchUsageHolder searchUsageHolder = new UsageService().getSearchUsageHolder(); + try (XContentParser jsonParser = createParser(JsonXContent.jsonXContent, restContent)) { + SearchSourceBuilder source = new SearchSourceBuilder().parseXContent(jsonParser, true, searchUsageHolder, nf -> true); + assertThat(source.retriever(), instanceOf(KnnRetrieverBuilder.class)); + KnnRetrieverBuilder parsed = (KnnRetrieverBuilder) source.retriever(); + assertThat(parsed.minScore(), equalTo(10f)); + assertThat(parsed.retrieverName(), equalTo("foo_knn")); + try (XContentParser parseSerialized = createParser(JsonXContent.jsonXContent, Strings.toString(source))) { + SearchSourceBuilder deserializedSource = new SearchSourceBuilder().parseXContent( + parseSerialized, + true, + searchUsageHolder, + nf -> true + ); + assertThat(deserializedSource.retriever(), instanceOf(KnnRetrieverBuilder.class)); + KnnRetrieverBuilder deserialized = (KnnRetrieverBuilder) source.retriever(); + assertThat(parsed, equalTo(deserialized)); + } + } + } + public void testStoredFieldsUsage() throws IOException { Set storedFieldRestVariations = Set.of( "{\"stored_fields\" : [\"_none_\"]}", diff --git a/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java b/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java index f3dd86e0b1fa2..b0bf7e6636498 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java +++ b/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java @@ -74,7 +74,7 @@ protected KnnRetrieverBuilder createTestInstance() { @Override protected KnnRetrieverBuilder doParseInstance(XContentParser parser) throws IOException { - return KnnRetrieverBuilder.fromXContent( + return (KnnRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( parser, new RetrieverParserContext( new SearchUsage(), diff --git a/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java b/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java index d2a1cac43c154..eacd949077bc4 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java +++ b/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java @@ -98,7 +98,7 @@ protected StandardRetrieverBuilder createTestInstance() { @Override protected StandardRetrieverBuilder doParseInstance(XContentParser parser) throws IOException { - return StandardRetrieverBuilder.fromXContent( + return (StandardRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( parser, new RetrieverParserContext( new SearchUsage(), diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java index e45ac8a9e0f70..a24bd91206ac0 100644 --- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java +++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java @@ -355,7 +355,6 @@ public void testManyEval() throws IOException { assertMap(map, mapMatcher.entry("columns", columns).entry("values", hasSize(10_000)).entry("took", greaterThanOrEqualTo(0))); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-serverless/issues/1874") public void testTooManyEval() throws IOException { initManyLongs(); assertCircuitBreaks(() -> manyEval(490)); @@ -616,14 +615,13 @@ private void initMvLongsIndex(int docs, int fields, int fieldValues) throws IOEx private void bulk(String name, String bulk) throws IOException { Request request = new Request("POST", "/" + name + "/_bulk"); - request.addParameter("filter_path", "errors"); request.setJsonEntity(bulk); request.setOptions( RequestOptions.DEFAULT.toBuilder() .setRequestConfig(RequestConfig.custom().setSocketTimeout(Math.toIntExact(TimeValue.timeValueMinutes(5).millis())).build()) ); Response response = client().performRequest(request); - assertThat(EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8), equalTo("{\"errors\":false}")); + assertThat(entityAsMap(response), matchesMap().entry("errors", false).extraOk()); } private void initIndex(String name, String bulk) throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java index 1cc6043e0be17..de87772d5ae82 100644 --- a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java +++ b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java @@ -695,8 +695,13 @@ public T get(long index) { } @Override - public T set(long index, T value) { - return in.set(index, value); + public void set(long index, T value) { + in.set(index, value); + } + + @Override + public T getAndSet(long index, T value) { + return in.getAndSet(index, value); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index 3e4925bb97efd..0b5803e9887d6 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -42,7 +42,7 @@ import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Sort; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TotalHitCountCollector; +import org.apache.lucene.search.TotalHitCountCollectorManager; import org.apache.lucene.search.Weight; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; @@ -178,9 +178,8 @@ protected static void assertVisibleCount(Engine engine, int numDocs, boolean ref engine.refresh("test"); } try (Engine.Searcher searcher = engine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new MatchAllDocsQuery(), collector); - assertThat(collector.getTotalHits(), equalTo(numDocs)); + Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager()); + assertThat(totalHits, equalTo(numDocs)); } } @@ -971,9 +970,8 @@ protected static void assertVisibleCount(InternalEngine engine, int numDocs, boo engine.refresh("test"); } try (Engine.Searcher searcher = engine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new MatchAllDocsQuery(), collector); - assertThat(collector.getTotalHits(), equalTo(numDocs)); + Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager()); + assertThat(totalHits, equalTo(numDocs)); } } @@ -1170,9 +1168,8 @@ public static void assertOpsOnReplica( assertVisibleCount(replicaEngine, lastFieldValue == null ? 0 : 1); if (lastFieldValue != null) { try (Engine.Searcher searcher = replicaEngine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new TermQuery(new Term("value", lastFieldValue)), collector); - assertThat(collector.getTotalHits(), equalTo(1)); + Integer totalHits = searcher.search(new TermQuery(new Term("value", lastFieldValue)), new TotalHitCountCollectorManager()); + assertThat(totalHits, equalTo(1)); } } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java index 067d1b96e965e..8dee5876aa207 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java @@ -9,10 +9,10 @@ package org.elasticsearch.logsdb.datageneration.datasource; +import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping; -import org.elasticsearch.test.ESTestCase; import java.util.Set; @@ -116,15 +116,11 @@ public DataSourceResponse.LeafMappingParametersGenerator accept(DataSourceHandle } } - record ObjectMappingParametersGenerator(boolean isRoot, boolean isNested) + record ObjectMappingParametersGenerator(boolean isRoot, boolean isNested, ObjectMapper.Subobjects parentSubobjects) implements DataSourceRequest { public DataSourceResponse.ObjectMappingParametersGenerator accept(DataSourceHandler handler) { return handler.handle(this); } - - public String syntheticSourceKeepValue() { - return isRoot() ? ESTestCase.randomFrom("none", "arrays") : ESTestCase.randomFrom("none", "arrays", "all"); - } } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java index 69f839d461b40..81bd80f464525 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -10,6 +10,7 @@ package org.elasticsearch.logsdb.datageneration.datasource; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping; import org.elasticsearch.test.ESTestCase; @@ -78,8 +79,11 @@ private Supplier> scaledFloatMapping(Map inj @Override public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) { if (request.isNested()) { + assert request.parentSubobjects() != ObjectMapper.Subobjects.DISABLED; + return new DataSourceResponse.ObjectMappingParametersGenerator(() -> { var parameters = new HashMap(); + if (ESTestCase.randomBoolean()) { parameters.put("dynamic", ESTestCase.randomFrom("true", "false", "strict")); } @@ -93,14 +97,43 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequ return new DataSourceResponse.ObjectMappingParametersGenerator(() -> { var parameters = new HashMap(); + + // Changing subobjects from subobjects: false is not supported, but we can f.e. go from "true" to "false". + // TODO enable subobjects: auto + // It is disabled because it currently does not have auto flattening and that results in asserts being triggered when using + // copy_to. + if (ESTestCase.randomBoolean()) { + parameters.put( + "subobjects", + ESTestCase.randomValueOtherThan( + ObjectMapper.Subobjects.AUTO, + () -> ESTestCase.randomFrom(ObjectMapper.Subobjects.values()) + ).toString() + ); + } + + if (request.parentSubobjects() == ObjectMapper.Subobjects.DISABLED + || parameters.getOrDefault("subobjects", "true").equals("false")) { + // "enabled: false" is not compatible with subobjects: false + // changing "dynamic" from parent context is not compatible with subobjects: false + // changing subobjects value is not compatible with subobjects: false + if (ESTestCase.randomBoolean()) { + parameters.put("enabled", "true"); + } + + return parameters; + } + if (ESTestCase.randomBoolean()) { parameters.put("dynamic", ESTestCase.randomFrom("true", "false", "strict", "runtime")); } if (ESTestCase.randomBoolean()) { parameters.put("enabled", ESTestCase.randomFrom("true", "false")); } + if (ESTestCase.randomBoolean()) { - parameters.put(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, request.syntheticSourceKeepValue()); + var value = request.isRoot() ? ESTestCase.randomFrom("none", "arrays") : ESTestCase.randomFrom("none", "arrays", "all"); + parameters.put(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, value); } return parameters; diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java index ebf13eb93ff4b..c1ec15a3479b3 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java @@ -9,6 +9,7 @@ package org.elasticsearch.logsdb.datageneration.fields; +import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; @@ -31,9 +32,14 @@ class Context { private final AtomicInteger nestedFieldsCount; private final Set eligibleCopyToDestinations; private final DynamicMapping parentDynamicMapping; + private final ObjectMapper.Subobjects currentSubobjectsConfig; - Context(DataGeneratorSpecification specification, DynamicMapping parentDynamicMapping) { - this(specification, "", 0, new AtomicInteger(0), new HashSet<>(), parentDynamicMapping); + Context( + DataGeneratorSpecification specification, + DynamicMapping parentDynamicMapping, + ObjectMapper.Subobjects currentSubobjectsConfig + ) { + this(specification, "", 0, new AtomicInteger(0), new HashSet<>(), parentDynamicMapping, currentSubobjectsConfig); } private Context( @@ -42,7 +48,8 @@ private Context( int objectDepth, AtomicInteger nestedFieldsCount, Set eligibleCopyToDestinations, - DynamicMapping parentDynamicMapping + DynamicMapping parentDynamicMapping, + ObjectMapper.Subobjects currentSubobjectsConfig ) { this.specification = specification; this.childFieldGenerator = specification.dataSource().get(new DataSourceRequest.ChildFieldGenerator(specification)); @@ -52,6 +59,7 @@ private Context( this.nestedFieldsCount = nestedFieldsCount; this.eligibleCopyToDestinations = eligibleCopyToDestinations; this.parentDynamicMapping = parentDynamicMapping; + this.currentSubobjectsConfig = currentSubobjectsConfig; } public DataGeneratorSpecification specification() { @@ -66,21 +74,30 @@ public DataSourceResponse.FieldTypeGenerator fieldTypeGenerator(DynamicMapping d return specification.dataSource().get(new DataSourceRequest.FieldTypeGenerator(dynamicMapping)); } - public Context subObject(String name, DynamicMapping dynamicMapping) { + public Context subObject(String name, DynamicMapping dynamicMapping, ObjectMapper.Subobjects subobjects) { return new Context( specification, pathToField(name), objectDepth + 1, nestedFieldsCount, eligibleCopyToDestinations, - dynamicMapping + dynamicMapping, + subobjects ); } - public Context nestedObject(String name, DynamicMapping dynamicMapping) { + public Context nestedObject(String name, DynamicMapping dynamicMapping, ObjectMapper.Subobjects subobjects) { nestedFieldsCount.incrementAndGet(); // copy_to can't be used across nested documents so all currently eligible fields are not eligible inside nested document. - return new Context(specification, pathToField(name), objectDepth + 1, nestedFieldsCount, new HashSet<>(), dynamicMapping); + return new Context( + specification, + pathToField(name), + objectDepth + 1, + nestedFieldsCount, + new HashSet<>(), + dynamicMapping, + subobjects + ); } public boolean shouldAddDynamicObjectField(DynamicMapping dynamicMapping) { @@ -99,10 +116,11 @@ public boolean shouldAddObjectField() { return childFieldGenerator.generateRegularSubObject(); } - public boolean shouldAddNestedField() { + public boolean shouldAddNestedField(ObjectMapper.Subobjects subobjects) { if (objectDepth >= specification.maxObjectDepth() || nestedFieldsCount.get() >= specification.nestedFieldsLimit() - || parentDynamicMapping == DynamicMapping.FORCED) { + || parentDynamicMapping == DynamicMapping.FORCED + || subobjects == ObjectMapper.Subobjects.DISABLED) { return false; } @@ -131,6 +149,14 @@ public DynamicMapping determineDynamicMapping(Map mappingParamet return dynamicParameter.equals("strict") ? DynamicMapping.FORBIDDEN : DynamicMapping.SUPPORTED; } + public ObjectMapper.Subobjects determineSubobjects(Map mappingParameters) { + if (currentSubobjectsConfig == ObjectMapper.Subobjects.DISABLED) { + return ObjectMapper.Subobjects.DISABLED; + } + + return ObjectMapper.Subobjects.from(mappingParameters.getOrDefault("subobjects", "true")); + } + public Set getEligibleCopyToDestinations() { return eligibleCopyToDestinations; } @@ -142,4 +168,8 @@ public void markFieldAsEligibleForCopyTo(String field) { private String pathToField(String field) { return path.isEmpty() ? field : path + "." + field; } + + public ObjectMapper.Subobjects getCurrentSubobjectsConfig() { + return currentSubobjectsConfig; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java index ba03b2f91c53c..83a68519d5de1 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java @@ -10,6 +10,7 @@ package org.elasticsearch.logsdb.datageneration.fields; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; @@ -31,7 +32,7 @@ public class GenericSubObjectFieldDataGenerator { this.context = context; } - List generateChildFields(DynamicMapping dynamicMapping) { + List generateChildFields(DynamicMapping dynamicMapping, ObjectMapper.Subobjects subobjects) { var existingFieldNames = new HashSet(); // no child fields is legal var childFieldsCount = context.childFieldGenerator().generateChildFieldCount(); @@ -42,12 +43,24 @@ List generateChildFields(DynamicMapping dynamicMapping) { if (context.shouldAddDynamicObjectField(dynamicMapping)) { result.add( - new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject(fieldName, DynamicMapping.FORCED)), true) + new ChildField( + fieldName, + new ObjectFieldDataGenerator(context.subObject(fieldName, DynamicMapping.FORCED, subobjects)), + true + ) ); } else if (context.shouldAddObjectField()) { - result.add(new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject(fieldName, dynamicMapping)), false)); - } else if (context.shouldAddNestedField()) { - result.add(new ChildField(fieldName, new NestedFieldDataGenerator(context.nestedObject(fieldName, dynamicMapping)), false)); + result.add( + new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject(fieldName, dynamicMapping, subobjects)), false) + ); + } else if (context.shouldAddNestedField(subobjects)) { + result.add( + new ChildField( + fieldName, + new NestedFieldDataGenerator(context.nestedObject(fieldName, dynamicMapping, subobjects)), + false + ) + ); } else { var fieldTypeInfo = context.fieldTypeGenerator(dynamicMapping).generator().get(); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java index ba168b221f572..69853debf9b77 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java @@ -28,13 +28,14 @@ public class NestedFieldDataGenerator implements FieldDataGenerator { this.mappingParameters = context.specification() .dataSource() - .get(new DataSourceRequest.ObjectMappingParametersGenerator(false, true)) + .get(new DataSourceRequest.ObjectMappingParametersGenerator(false, true, context.getCurrentSubobjectsConfig())) .mappingGenerator() .get(); var dynamicMapping = context.determineDynamicMapping(mappingParameters); + var subobjects = context.determineSubobjects(mappingParameters); var genericGenerator = new GenericSubObjectFieldDataGenerator(context); - this.childFields = genericGenerator.generateChildFields(dynamicMapping); + this.childFields = genericGenerator.generateChildFields(dynamicMapping, subobjects); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java index 084310ac967fc..701642c57619b 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java @@ -28,13 +28,14 @@ public class ObjectFieldDataGenerator implements FieldDataGenerator { this.mappingParameters = context.specification() .dataSource() - .get(new DataSourceRequest.ObjectMappingParametersGenerator(false, false)) + .get(new DataSourceRequest.ObjectMappingParametersGenerator(false, false, context.getCurrentSubobjectsConfig())) .mappingGenerator() .get(); var dynamicMapping = context.determineDynamicMapping(mappingParameters); + var subobjects = context.determineSubobjects(mappingParameters); var genericGenerator = new GenericSubObjectFieldDataGenerator(context); - this.childFields = genericGenerator.generateChildFields(dynamicMapping); + this.childFields = genericGenerator.generateChildFields(dynamicMapping, subobjects); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java index 2c7aa65d8c6d1..1374362df7f4a 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java @@ -10,6 +10,7 @@ package org.elasticsearch.logsdb.datageneration.fields; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.xcontent.XContentBuilder; @@ -37,7 +38,12 @@ public TopLevelObjectFieldDataGenerator(DataGeneratorSpecification specification this.mappingParameters = Map.of(); } else { this.mappingParameters = new HashMap<>( - specification.dataSource().get(new DataSourceRequest.ObjectMappingParametersGenerator(true, false)).mappingGenerator().get() + // Value of subobjects here is for a parent of this object. + // Since there is no parent we pass ENABLED to allow to set subobjects to any value at top level. + specification.dataSource() + .get(new DataSourceRequest.ObjectMappingParametersGenerator(true, false, ObjectMapper.Subobjects.ENABLED)) + .mappingGenerator() + .get() ); // Top-level object can't be disabled because @timestamp is a required field in data streams. this.mappingParameters.remove("enabled"); @@ -46,11 +52,15 @@ public TopLevelObjectFieldDataGenerator(DataGeneratorSpecification specification ? DynamicMapping.FORBIDDEN : DynamicMapping.SUPPORTED; } - this.context = new Context(specification, dynamicMapping); + var subobjects = ObjectMapper.Subobjects.from(mappingParameters.getOrDefault("subobjects", "true")); + + // Value of subobjects here is for a parent of this object. + // Since there is no parent we pass ENABLED to allow to set subobjects to any value at top level. + this.context = new Context(specification, dynamicMapping, ObjectMapper.Subobjects.ENABLED); var genericGenerator = new GenericSubObjectFieldDataGenerator(context); this.predefinedFields = genericGenerator.generateChildFields(specification.predefinedFields()); - this.generatedChildFields = genericGenerator.generateChildFields(dynamicMapping); + this.generatedChildFields = genericGenerator.generateChildFields(dynamicMapping, subobjects); } public CheckedConsumer mappingWriter(Map customMappingParameters) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 068a666d78d74..31c8e5bc3d457 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -900,10 +900,11 @@ public static long randomLongBetween(long min, long max) { * @return a random instant between a min and a max value with a random nanosecond precision */ public static Instant randomInstantBetween(Instant minInstant, Instant maxInstant) { - return Instant.ofEpochSecond( - randomLongBetween(minInstant.getEpochSecond(), maxInstant.getEpochSecond()), - randomLongBetween(0, 999999999) - ); + long epochSecond = randomLongBetween(minInstant.getEpochSecond(), maxInstant.getEpochSecond()); + long minNanos = epochSecond == minInstant.getEpochSecond() ? minInstant.getNano() : 0; + long maxNanos = epochSecond == maxInstant.getEpochSecond() ? maxInstant.getNano() : 999999999; + long nanos = randomLongBetween(minNanos, maxNanos); + return Instant.ofEpochSecond(epochSecond, nanos); } /** diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 215973b5dece2..d17016f850300 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -719,10 +719,6 @@ protected boolean preserveTemplatesUponCompletion() { * all feature states, deleting system indices, system associated indices, and system data streams. */ protected boolean resetFeatureStates() { - if (clusterHasFeature(RestTestLegacyFeatures.FEATURE_STATE_RESET_SUPPORTED) == false) { - return false; - } - // ML reset fails when ML is disabled in versions before 8.7 if (isMlEnabled() == false && clusterHasFeature(RestTestLegacyFeatures.ML_STATE_RESET_FALLBACK_ON_DISABLED) == false) { return false; @@ -917,22 +913,10 @@ private void wipeCluster() throws Exception { .filter(name -> isXPackTemplate(name) == false) .collect(Collectors.toList()); if (names.isEmpty() == false) { - // Ideally we would want to check if the elected master node supports this feature and send the delete request - // directly to that node, but node-specific feature checks is something we want to avoid if possible. - if (clusterHasFeature(RestTestLegacyFeatures.DELETE_TEMPLATE_MULTIPLE_NAMES_SUPPORTED)) { - try { - adminClient().performRequest(new Request("DELETE", "_index_template/" + String.join(",", names))); - } catch (ResponseException e) { - logger.warn(() -> format("unable to remove multiple composable index templates %s", names), e); - } - } else { - for (String name : names) { - try { - adminClient().performRequest(new Request("DELETE", "_index_template/" + name)); - } catch (ResponseException e) { - logger.warn(() -> format("unable to remove composable index template %s", name), e); - } - } + try { + adminClient().performRequest(new Request("DELETE", "_index_template/" + String.join(",", names))); + } catch (ResponseException e) { + logger.warn(() -> format("unable to remove multiple composable index templates %s", names), e); } } } catch (Exception e) { @@ -948,22 +932,10 @@ private void wipeCluster() throws Exception { .filter(name -> isXPackTemplate(name) == false) .collect(Collectors.toList()); if (names.isEmpty() == false) { - // Ideally we would want to check if the elected master node supports this feature and send the delete request - // directly to that node, but node-specific feature checks is something we want to avoid if possible. - if (clusterHasFeature(RestTestLegacyFeatures.DELETE_TEMPLATE_MULTIPLE_NAMES_SUPPORTED)) { - try { - adminClient().performRequest(new Request("DELETE", "_component_template/" + String.join(",", names))); - } catch (ResponseException e) { - logger.warn(() -> format("unable to remove multiple component templates %s", names), e); - } - } else { - for (String componentTemplate : names) { - try { - adminClient().performRequest(new Request("DELETE", "_component_template/" + componentTemplate)); - } catch (ResponseException e) { - logger.warn(() -> format("unable to remove component template %s", componentTemplate), e); - } - } + try { + adminClient().performRequest(new Request("DELETE", "_component_template/" + String.join(",", names))); + } catch (ResponseException e) { + logger.warn(() -> format("unable to remove multiple component templates %s", names), e); } } } catch (Exception e) { @@ -1141,7 +1113,6 @@ protected static void wipeAllIndices() throws IOException { } protected static void wipeAllIndices(boolean preserveSecurityIndices) throws IOException { - boolean includeHidden = clusterHasFeature(RestTestLegacyFeatures.HIDDEN_INDICES_SUPPORTED); try { // remove all indices except some history indices which can pop up after deleting all data streams but shouldn't interfere final List indexPatterns = new ArrayList<>( @@ -1151,7 +1122,7 @@ protected static void wipeAllIndices(boolean preserveSecurityIndices) throws IOE indexPatterns.add("-.security-*"); } final Request deleteRequest = new Request("DELETE", Strings.collectionToCommaDelimitedString(indexPatterns)); - deleteRequest.addParameter("expand_wildcards", "open,closed" + (includeHidden ? ",hidden" : "")); + deleteRequest.addParameter("expand_wildcards", "open,closed,hidden"); final Response response = adminClient().performRequest(deleteRequest); try (InputStream is = response.getEntity().getContent()) { assertTrue((boolean) XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true).get("acknowledged")); @@ -1320,9 +1291,8 @@ private void wipeRollupJobs() throws IOException { } protected void refreshAllIndices() throws IOException { - boolean includeHidden = clusterHasFeature(RestTestLegacyFeatures.HIDDEN_INDICES_SUPPORTED); Request refreshRequest = new Request("POST", "/_refresh"); - refreshRequest.addParameter("expand_wildcards", "open" + (includeHidden ? ",hidden" : "")); + refreshRequest.addParameter("expand_wildcards", "open,hidden"); // Allow system index deprecation warnings refreshRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(warnings -> { if (warnings.isEmpty()) { @@ -2488,18 +2458,6 @@ public static void setIgnoredErrorResponseCodes(Request request, RestStatus... r } private static XContentType randomSupportedContentType() { - if (clusterHasFeature(RestTestLegacyFeatures.SUPPORTS_TRUE_BINARY_RESPONSES) == false) { - // Very old versions encode binary stored fields using base64 in all formats, not just JSON, but we expect to see raw binary - // fields in non-JSON formats, so we stick to JSON in these cases. - return XContentType.JSON; - } - - if (clusterHasFeature(RestTestLegacyFeatures.SUPPORTS_VENDOR_XCONTENT_TYPES) == false) { - // The VND_* formats were introduced part-way through the 7.x series for compatibility with 8.x, but are not supported by older - // 7.x versions. - return randomFrom(XContentType.JSON, XContentType.CBOR, XContentType.YAML, XContentType.SMILE); - } - return randomFrom(XContentType.values()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestLegacyFeatures.java b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestLegacyFeatures.java index 427398b9a8c0e..194dfc057b84f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestLegacyFeatures.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestLegacyFeatures.java @@ -27,23 +27,8 @@ public class RestTestLegacyFeatures implements FeatureSpecification { public static final NodeFeature ML_STATE_RESET_FALLBACK_ON_DISABLED = new NodeFeature("ml.state_reset_fallback_on_disabled"); @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature FEATURE_STATE_RESET_SUPPORTED = new NodeFeature("system_indices.feature_state_reset_supported"); - public static final NodeFeature SYSTEM_INDICES_REST_ACCESS_ENFORCED = new NodeFeature("system_indices.rest_access_enforced"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature SYSTEM_INDICES_REST_ACCESS_DEPRECATED = new NodeFeature("system_indices.rest_access_deprecated"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature HIDDEN_INDICES_SUPPORTED = new NodeFeature("indices.hidden_supported"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) public static final NodeFeature COMPONENT_TEMPLATE_SUPPORTED = new NodeFeature("indices.component_template_supported"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature DELETE_TEMPLATE_MULTIPLE_NAMES_SUPPORTED = new NodeFeature( - "indices.delete_template_multiple_names_supported" - ); public static final NodeFeature ML_NEW_MEMORY_FORMAT = new NodeFeature("ml.new_memory_format"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature SUPPORTS_VENDOR_XCONTENT_TYPES = new NodeFeature("rest.supports_vendor_xcontent_types"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature SUPPORTS_TRUE_BINARY_RESPONSES = new NodeFeature("rest.supports_true_binary_responses"); /** These are "pure test" features: normally we would not need them, and test for TransportVersion/fallback to Version (see for example * {@code ESRestTestCase#minimumTransportVersion()}. However, some tests explicitly check and validate the content of a response, so @@ -61,21 +46,6 @@ public class RestTestLegacyFeatures implements FeatureSpecification { public static final NodeFeature DESIRED_NODE_API_SUPPORTED = new NodeFeature("desired_node_supported"); public static final NodeFeature SECURITY_UPDATE_API_KEY = new NodeFeature("security.api_key_update"); public static final NodeFeature SECURITY_BULK_UPDATE_API_KEY = new NodeFeature("security.api_key_bulk_update"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature WATCHES_VERSION_IN_META = new NodeFeature("watcher.version_in_meta"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature SECURITY_ROLE_DESCRIPTORS_OPTIONAL = new NodeFeature("security.role_descriptors_optional"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature SEARCH_AGGREGATIONS_FORCE_INTERVAL_SELECTION_DATE_HISTOGRAM = new NodeFeature( - "search.aggregations.force_interval_selection_on_date_histogram" - ); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature TRANSFORM_NEW_API_ENDPOINT = new NodeFeature("transform.new_api_endpoint"); - // Ref: https://github.com/elastic/elasticsearch/pull/65205 - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature ML_INDICES_HIDDEN = new NodeFeature("ml.indices_hidden"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature ML_ANALYTICS_MAPPINGS = new NodeFeature("ml.analytics_mappings"); public static final NodeFeature TSDB_NEW_INDEX_FORMAT = new NodeFeature("indices.tsdb_new_format"); public static final NodeFeature TSDB_GENERALLY_AVAILABLE = new NodeFeature("indices.tsdb_supported"); @@ -104,14 +74,10 @@ public class RestTestLegacyFeatures implements FeatureSpecification { @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) public static final NodeFeature REPLICATION_OF_CLOSED_INDICES = new NodeFeature("indices.closed_replication_supported"); @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature TASK_INDEX_SYSTEM_INDEX = new NodeFeature("tasks.moved_to_system_index"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) public static final NodeFeature SOFT_DELETES_ENFORCED = new NodeFeature("indices.soft_deletes_enforced"); @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) public static final NodeFeature NEW_TRANSPORT_COMPRESSED_SETTING = new NodeFeature("transport.new_compressed_setting"); @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature SHUTDOWN_SUPPORTED = new NodeFeature("shutdown.supported"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) public static final NodeFeature SERVICE_ACCOUNTS_SUPPORTED = new NodeFeature("auth.service_accounts_supported"); @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) public static final NodeFeature TRANSFORM_SUPPORTED = new NodeFeature("transform.supported"); @@ -140,27 +106,14 @@ public class RestTestLegacyFeatures implements FeatureSpecification { @Override public Map getHistoricalFeatures() { return Map.ofEntries( - entry(FEATURE_STATE_RESET_SUPPORTED, Version.V_7_13_0), - entry(SYSTEM_INDICES_REST_ACCESS_ENFORCED, Version.V_8_0_0), - entry(SYSTEM_INDICES_REST_ACCESS_DEPRECATED, Version.V_7_10_0), - entry(HIDDEN_INDICES_SUPPORTED, Version.V_7_7_0), entry(COMPONENT_TEMPLATE_SUPPORTED, Version.V_7_8_0), - entry(DELETE_TEMPLATE_MULTIPLE_NAMES_SUPPORTED, Version.V_7_13_0), entry(ML_STATE_RESET_FALLBACK_ON_DISABLED, Version.V_8_7_0), entry(SECURITY_UPDATE_API_KEY, Version.V_8_4_0), entry(SECURITY_BULK_UPDATE_API_KEY, Version.V_8_5_0), entry(ML_NEW_MEMORY_FORMAT, Version.V_8_11_0), - entry(SUPPORTS_VENDOR_XCONTENT_TYPES, Version.V_7_11_0), - entry(SUPPORTS_TRUE_BINARY_RESPONSES, Version.V_7_7_0), entry(TRANSPORT_VERSION_SUPPORTED, VERSION_INTRODUCING_TRANSPORT_VERSIONS), entry(STATE_REPLACED_TRANSPORT_VERSION_WITH_NODES_VERSION, Version.V_8_11_0), entry(ML_MEMORY_OVERHEAD_FIXED, Version.V_8_2_1), - entry(WATCHES_VERSION_IN_META, Version.V_7_13_0), - entry(SECURITY_ROLE_DESCRIPTORS_OPTIONAL, Version.V_7_3_0), - entry(SEARCH_AGGREGATIONS_FORCE_INTERVAL_SELECTION_DATE_HISTOGRAM, Version.V_7_2_0), - entry(TRANSFORM_NEW_API_ENDPOINT, Version.V_7_5_0), - entry(ML_INDICES_HIDDEN, Version.V_7_7_0), - entry(ML_ANALYTICS_MAPPINGS, Version.V_7_3_0), entry(REST_ELASTIC_PRODUCT_HEADER_PRESENT, Version.V_8_0_1), entry(DESIRED_NODE_API_SUPPORTED, Version.V_8_1_0), entry(TSDB_NEW_INDEX_FORMAT, Version.V_8_2_0), @@ -173,10 +126,8 @@ public Map getHistoricalFeatures() { entry(INDEXING_SLOWLOG_LEVEL_SETTING_REMOVED, Version.V_8_0_0), entry(DEPRECATION_WARNINGS_LEAK_FIXED, Version.V_7_17_9), entry(REPLICATION_OF_CLOSED_INDICES, Version.V_7_2_0), - entry(TASK_INDEX_SYSTEM_INDEX, Version.V_7_10_0), entry(SOFT_DELETES_ENFORCED, Version.V_8_0_0), entry(NEW_TRANSPORT_COMPRESSED_SETTING, Version.V_7_14_0), - entry(SHUTDOWN_SUPPORTED, Version.V_7_15_0), entry(SERVICE_ACCOUNTS_SUPPORTED, Version.V_7_13_0), entry(TRANSFORM_SUPPORTED, Version.V_7_2_0), entry(SLM_SUPPORTED, Version.V_7_4_0), diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java index 7cfbd2b3a57f1..86c3f42a6a8ec 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java @@ -371,14 +371,10 @@ public void execute(ClientYamlTestExecutionContext executionContext) throws IOEx ? executionContext.getClientYamlTestCandidate().getTestPath() : null; - // #84038 and #84089 mean that this assertion fails when running against < 7.17.2 and 8.0.0 released versions - // This is really difficult to express just with features, so I will break it down into 2 parts: version check for v7, - // and feature check for v8. This way the version check can be removed once we move to v9 - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - var fixedInV7 = executionContext.clusterHasFeature("gte_v7.17.2", false) - && executionContext.clusterHasFeature("gte_v8.0.0", false) == false; - var fixedProductionHeader = fixedInV7 - || executionContext.clusterHasFeature(RestTestLegacyFeatures.REST_ELASTIC_PRODUCT_HEADER_PRESENT.id(), false); + var fixedProductionHeader = executionContext.clusterHasFeature( + RestTestLegacyFeatures.REST_ELASTIC_PRODUCT_HEADER_PRESENT.id(), + false + ); if (fixedProductionHeader) { checkElasticProductHeader(response.getHeaders("X-elastic-product")); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/GetRoleMappingsResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/GetRoleMappingsResponse.java index 13a751829797f..4f18411ac3af6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/GetRoleMappingsResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/GetRoleMappingsResponse.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; import java.io.IOException; +import java.util.Collection; /** * Response to {@link GetRoleMappingsAction get role-mappings API}. @@ -21,6 +22,10 @@ public class GetRoleMappingsResponse extends ActionResponse { private final ExpressionRoleMapping[] mappings; + public GetRoleMappingsResponse(Collection mappings) { + this(mappings.toArray(new ExpressionRoleMapping[0])); + } + public GetRoleMappingsResponse(ExpressionRoleMapping... mappings) { this.mappings = mappings; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java index da6ff6ad24c34..8f78fdbccd923 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.AbstractNamedDiffable; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.NamedDiff; +import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; @@ -57,6 +58,7 @@ public final class RoleMappingMetadata extends AbstractNamedDiffable deprecatedWarnings = getWarningHeaders(response.getHeaders()); + assertThat( + extractWarningValuesFromWarningHeaders(deprecatedWarnings), + containsInAnyOrder("[/_test_cluster/deprecated_but_dont_remove] is deprecated, but no plans to remove quite yet") + ); + + assertBusy(() -> { + List> documents = DeprecationTestUtils.getIndexedDeprecations(client(), xOpaqueId()); + + logger.warn(documents); + + // only assert the relevant fields: level, message, and category + assertThat( + documents, + containsInAnyOrder( + allOf( + hasEntry("elasticsearch.event.category", "api"), + hasEntry("log.level", "WARN"), + hasEntry("message", "[/_test_cluster/deprecated_but_dont_remove] is deprecated, but no plans to remove quite yet") + ) + ) + ); + }, 30, TimeUnit.SECONDS); + } + + public void testReplacesInCurrentVersion() throws Exception { + final Request request = new Request("GET", "/_test_cluster/old_name1"); // deprecated in current version + request.setEntity(buildSettingsRequest(Collections.singletonList(TEST_NOT_DEPRECATED_SETTING), "settings")); + Response response = performScopedRequest(request); + + final List deprecatedWarnings = getWarningHeaders(response.getHeaders()); + assertThat( + extractWarningValuesFromWarningHeaders(deprecatedWarnings), + containsInAnyOrder("[GET /_test_cluster/old_name1] is deprecated! Use [GET /_test_cluster/new_name1] instead.") + ); + + assertBusy(() -> { + List> documents = DeprecationTestUtils.getIndexedDeprecations(client(), xOpaqueId()); + + logger.warn(documents); + + // only assert the relevant fields: level, message, and category + assertThat( + documents, + containsInAnyOrder( + allOf( + hasEntry("elasticsearch.event.category", "api"), + hasEntry("log.level", "WARN"), + hasEntry("message", "[GET /_test_cluster/old_name1] is deprecated! Use [GET /_test_cluster/new_name1] instead.") + ) + ) + ); + }, 30, TimeUnit.SECONDS); + } + + public void testReplacesInCompatibleVersion() throws Exception { + final Request request = new Request("GET", "/_test_cluster/old_name2"); // deprecated in minimum supported version + request.setEntity(buildSettingsRequest(Collections.singletonList(TEST_DEPRECATED_SETTING_TRUE1), "deprecated_settings")); + final RequestOptions compatibleOptions = request.getOptions() + .toBuilder() + .addHeader("Accept", "application/vnd.elasticsearch+json;compatible-with=" + RestApiVersion.minimumSupported().major) + .addHeader("Content-Type", "application/vnd.elasticsearch+json;compatible-with=" + RestApiVersion.minimumSupported().major) + .build(); + request.setOptions(compatibleOptions); + Response response = performScopedRequest(request); + + final List deprecatedWarnings = getWarningHeaders(response.getHeaders()); + assertThat( + extractWarningValuesFromWarningHeaders(deprecatedWarnings), + containsInAnyOrder( + "[GET /_test_cluster/old_name2] is deprecated! Use [GET /_test_cluster/new_name2] instead.", + "You are using a compatible API for this request" + ) + ); + assertBusy(() -> { + List> documents = DeprecationTestUtils.getIndexedDeprecations(client(), xOpaqueId()); + + logger.warn(documents); + + // only assert the relevant fields: level, message, and category + assertThat( + documents, + containsInAnyOrder( + allOf( + + hasEntry("elasticsearch.event.category", "compatible_api"), + hasEntry("log.level", "CRITICAL"), + hasEntry("message", "[GET /_test_cluster/old_name2] is deprecated! Use [GET /_test_cluster/new_name2] instead.") + ), + allOf( + hasEntry("elasticsearch.event.category", "compatible_api"), + hasEntry("log.level", "CRITICAL"), + // this message comes from the test, not production code. this is the message for setting the deprecated setting + hasEntry("message", "You are using a compatible API for this request") + ) + ) + ); + }, 30, TimeUnit.SECONDS); + } + /** * Check that log messages about REST API compatibility are recorded to an index */ diff --git a/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationHeaderRestAction.java b/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationHeaderRestAction.java index 70942b04f85b8..9e5f999d1f825 100644 --- a/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationHeaderRestAction.java +++ b/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationHeaderRestAction.java @@ -97,10 +97,23 @@ public List routes() { return List.of( // note: RestApiVersion.current() is acceptable here because this is test code -- ordinary callers of `.deprecated(...)` // should use an actual version - Route.builder(GET, "/_test_cluster/deprecated_settings").deprecated(DEPRECATED_ENDPOINT, RestApiVersion.current()).build(), + Route.builder(GET, "/_test_cluster/deprecated_settings") + .deprecatedForRemoval(DEPRECATED_ENDPOINT, RestApiVersion.current()) + .build(), + // TODO: s/deprecated/deprecatedForRemoval when removing `deprecated` method Route.builder(POST, "/_test_cluster/deprecated_settings").deprecated(DEPRECATED_ENDPOINT, RestApiVersion.current()).build(), - Route.builder(GET, "/_test_cluster/compat_only").deprecated(DEPRECATED_ENDPOINT, RestApiVersion.minimumSupported()).build(), - Route.builder(GET, "/_test_cluster/only_deprecated_setting").build() + Route.builder(GET, "/_test_cluster/compat_only") + .deprecatedForRemoval(DEPRECATED_ENDPOINT, RestApiVersion.minimumSupported()) + .build(), + Route.builder(GET, "/_test_cluster/only_deprecated_setting").build(), + Route.builder(GET, "/_test_cluster/deprecated_but_dont_remove") + .deprecateAndKeep("[/_test_cluster/deprecated_but_dont_remove] is deprecated, but no plans to remove quite yet") + .build(), + Route.builder(GET, "/_test_cluster/new_name1").replaces(GET, "/_test_cluster/old_name1", RestApiVersion.current()).build(), + Route.builder(GET, "/_test_cluster/new_name2") + .replaces(GET, "/_test_cluster/old_name2", RestApiVersion.minimumSupported()) + .build() + ); } diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index c5ab20469bf77..974180526d750 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -117,7 +117,8 @@ public void testDoNotLogWithInfo() throws IOException { setLoggingLevel("INFO"); RequestObjectBuilder builder = requestObjectBuilder().query("ROW DO_NOT_LOG_ME = 1"); Map result = runEsql(builder); - assertEquals(2, result.size()); + assertEquals(3, result.size()); + assertThat(((Integer) result.get("took")).intValue(), greaterThanOrEqualTo(0)); Map colA = Map.of("name", "DO_NOT_LOG_ME", "type", "integer"); assertEquals(List.of(colA), result.get("columns")); assertEquals(List.of(List.of(1)), result.get("values")); @@ -136,7 +137,8 @@ public void testDoLogWithDebug() throws IOException { setLoggingLevel("DEBUG"); RequestObjectBuilder builder = requestObjectBuilder().query("ROW DO_LOG_ME = 1"); Map result = runEsql(builder); - assertEquals(2, result.size()); + assertEquals(3, result.size()); + assertThat(((Integer) result.get("took")).intValue(), greaterThanOrEqualTo(0)); Map colA = Map.of("name", "DO_LOG_ME", "type", "integer"); assertEquals(List.of(colA), result.get("columns")); assertEquals(List.of(List.of(1)), result.get("values")); diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java index 39340ab745a4d..c3e9652f51fb4 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java @@ -249,7 +249,8 @@ public static RequestObjectBuilder jsonBuilder() throws IOException { public void testGetAnswer() throws IOException { Map answer = runEsql(requestObjectBuilder().query("row a = 1, b = 2")); - assertEquals(2, answer.size()); + assertEquals(3, answer.size()); + assertThat(((Integer) answer.get("took")).intValue(), greaterThanOrEqualTo(0)); Map colA = Map.of("name", "a", "type", "integer"); Map colB = Map.of("name", "b", "type", "integer"); assertEquals(List.of(colA, colB), answer.get("columns")); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index f52829741ed6e..1fdb6150a0e81 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -1270,3 +1270,19 @@ emp_no:integer | birth_date:datetime 10007 | 1957-05-23T00:00:00Z 10008 | 1958-02-19T00:00:00Z ; + +Least for dates +required_capability: least_greatest_for_dates +ROW a = LEAST(TO_DATETIME("1957-05-23T00:00:00Z"), TO_DATETIME("1958-02-19T00:00:00Z")); + +a:datetime +1957-05-23T00:00:00 +; + +GREATEST for dates +required_capability: least_greatest_for_dates +ROW a = GREATEST(TO_DATETIME("1957-05-23T00:00:00Z"), TO_DATETIME("1958-02-19T00:00:00Z")); + +a:datetime +1958-02-19T00:00:00 +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec index 13c3857a5c497..6e8d5fba67cee 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec @@ -33,9 +33,9 @@ double e() "double exp(number:double|integer|long|unsigned_long)" "double|integer|long|unsigned_long floor(number:double|integer|long|unsigned_long)" "keyword from_base64(string:keyword|text)" -"boolean|double|integer|ip|keyword|long|text|version greatest(first:boolean|double|integer|ip|keyword|long|text|version, ?rest...:boolean|double|integer|ip|keyword|long|text|version)" +"boolean|date|double|integer|ip|keyword|long|text|version greatest(first:boolean|date|double|integer|ip|keyword|long|text|version, ?rest...:boolean|date|double|integer|ip|keyword|long|text|version)" "ip ip_prefix(ip:ip, prefixLengthV4:integer, prefixLengthV6:integer)" -"boolean|double|integer|ip|keyword|long|text|version least(first:boolean|double|integer|ip|keyword|long|text|version, ?rest...:boolean|double|integer|ip|keyword|long|text|version)" +"boolean|date|double|integer|ip|keyword|long|text|version least(first:boolean|date|double|integer|ip|keyword|long|text|version, ?rest...:boolean|date|double|integer|ip|keyword|long|text|version)" "keyword left(string:keyword|text, length:integer)" "integer length(string:keyword|text)" "integer locate(string:keyword|text, substring:keyword|text, ?start:integer)" @@ -69,6 +69,7 @@ double pi() "double pow(base:double|integer|long|unsigned_long, exponent:double|integer|long|unsigned_long)" "keyword repeat(string:keyword|text, number:integer)" "keyword replace(string:keyword|text, regex:keyword|text, newString:keyword|text)" +"keyword|text reverse(str:keyword|text)" "keyword right(string:keyword|text, length:integer)" "double|integer|long|unsigned_long round(number:double|integer|long|unsigned_long, ?decimals:integer)" "keyword|text rtrim(string:keyword|text)" @@ -165,9 +166,9 @@ ends_with |[str, suffix] |["keyword|text", "keyword|te exp |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. floor |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. from_base64 |string |"keyword|text" |A base64 string. -greatest |first |"boolean|double|integer|ip|keyword|long|text|version" |First of the columns to evaluate. +greatest |first |"boolean|date|double|integer|ip|keyword|long|text|version" |First of the columns to evaluate. ip_prefix |[ip, prefixLengthV4, prefixLengthV6]|[ip, integer, integer] |[IP address of type `ip` (both IPv4 and IPv6 are supported)., Prefix length for IPv4 addresses., Prefix length for IPv6 addresses.] -least |first |"boolean|double|integer|ip|keyword|long|text|version" |First of the columns to evaluate. +least |first |"boolean|date|double|integer|ip|keyword|long|text|version" |First of the columns to evaluate. left |[string, length] |["keyword|text", integer] |[The string from which to return a substring., The number of characters to return.] length |string |"keyword|text" |String expression. If `null`, the function returns `null`. locate |[string, substring, start] |["keyword|text", "keyword|text", "integer"] |[An input string, A substring to locate in the input string, The start index] @@ -201,6 +202,7 @@ pi |null |null pow |[base, exponent] |["double|integer|long|unsigned_long", "double|integer|long|unsigned_long"] |["Numeric expression for the base. If `null`\, the function returns `null`.", "Numeric expression for the exponent. If `null`\, the function returns `null`."] repeat |[string, number] |["keyword|text", integer] |[String expression., Number times to repeat.] replace |[string, regex, newString] |["keyword|text", "keyword|text", "keyword|text"] |[String expression., Regular expression., Replacement string.] +reverse |str |"keyword|text" |String expression. If `null`, the function returns `null`. right |[string, length] |["keyword|text", integer] |[The string from which to returns a substring., The number of characters to return.] round |[number, decimals] |["double|integer|long|unsigned_long", integer] |["The numeric value to round. If `null`\, the function returns `null`.", "The number of decimal places to round to. Defaults to 0. If `null`\, the function returns `null`."] rtrim |string |"keyword|text" |String expression. If `null`, the function returns `null`. @@ -333,6 +335,7 @@ pi |Returns {wikipedia}/Pi[Pi], the ratio of a circle's circumference pow |Returns the value of `base` raised to the power of `exponent`. repeat |Returns a string constructed by concatenating `string` with itself the specified `number` of times. replace |The function substitutes in the string `str` any match of the regular expression `regex` with the replacement string `newStr`. +reverse |Returns a new string representing the input string in reverse order. right |Return the substring that extracts 'length' chars from 'str' starting from the right. round |Rounds a number to the specified number of decimal places. Defaults to 0, which returns the nearest integer. If the precision is a negative number, rounds to the number of digits left of the decimal point. rtrim |Removes trailing whitespaces from a string. @@ -431,9 +434,9 @@ ends_with |boolean exp |double |false |false |false floor |"double|integer|long|unsigned_long" |false |false |false from_base64 |keyword |false |false |false -greatest |"boolean|double|integer|ip|keyword|long|text|version" |false |true |false +greatest |"boolean|date|double|integer|ip|keyword|long|text|version" |false |true |false ip_prefix |ip |[false, false, false] |false |false -least |"boolean|double|integer|ip|keyword|long|text|version" |false |true |false +least |"boolean|date|double|integer|ip|keyword|long|text|version" |false |true |false left |keyword |[false, false] |false |false length |integer |false |false |false locate |integer |[false, false, true] |false |false @@ -467,6 +470,7 @@ pi |double pow |double |[false, false] |false |false repeat |keyword |[false, false] |false |false replace |keyword |[false, false, false] |false |false +reverse |"keyword|text" |false |false |false right |keyword |[false, false] |false |false round |"double|integer|long|unsigned_long" |[false, true] |false |false rtrim |"keyword|text" |false |false |false @@ -544,5 +548,5 @@ required_capability: meta meta functions | stats a = count(*), b = count(*), c = count(*) | mv_expand c; a:long | b:long | c:long -121 | 121 | 121 +122 | 122 | 122 ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index ffcceab26bcaf..5313e6630c75d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -1194,6 +1194,113 @@ a:keyword | upper:keyword | lower:keyword π/2 + a + B + Λ ºC | Π/2 + A + B + Λ ºC | π/2 + a + b + λ ºc ; +reverse +required_capability: fn_reverse +from employees | sort emp_no | eval name_reversed = REVERSE(first_name) | keep emp_no, first_name, name_reversed | limit 1; + +emp_no:integer | first_name:keyword | name_reversed:keyword +10001 | Georgi | igroeG +; + +reverseRow +required_capability: fn_reverse +// tag::reverse[] +ROW message = "Some Text" | EVAL message_reversed = REVERSE(message); +// end::reverse[] + +// tag::reverse-result[] +message:keyword | message_reversed:keyword +Some Text | txeT emoS +// end::reverse-result[] +; + +reverseEmoji +required_capability: fn_reverse +// tag::reverseEmoji[] +ROW bending_arts = "💧🪨🔥💨" | EVAL bending_arts_reversed = REVERSE(bending_arts); +// end::reverseEmoji[] + +// tag::reverseEmoji-result[] +bending_arts:keyword | bending_arts_reversed:keyword +💧🪨🔥💨 | 💨🔥🪨💧 +// end::reverseEmoji-result[] +; + +reverseEmoji2 +required_capability: fn_reverse +ROW off_on_holiday = "🏠➡️🚌➡️✈️➡️🏝️" | EVAL back_home_again = REVERSE(off_on_holiday); + +off_on_holiday:keyword | back_home_again:keyword +🏠➡️🚌➡️✈️➡️🏝️ | 🏝️➡️✈️➡️🚌➡️🏠 +; + +reverseGraphemeClusters +required_capability: fn_reverse +ROW message = "áéíóúàèìòùâêîôû😊👍🏽🎉💖कंठाी" | EVAL message_reversed = REVERSE(message); + +message:keyword | message_reversed:keyword +áéíóúàèìòùâêîôû😊👍🏽🎉💖कंठाी | ठाीकं💖🎉👍🏽😊ûôîêâùòìèàúóíéá +; + +reverseMultiValue +required_capability: fn_reverse +FROM employees | SORT emp_no | EVAL jobs_reversed = REVERSE(job_positions) | KEEP job*, emp_no | LIMIT 5; + +warning:Line 1:53: evaluation of [REVERSE(job_positions)] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:53: java.lang.IllegalArgumentException: single-value function encountered multi-value + +job_positions:keyword | jobs_reversed:keyword | emp_no:integer +["Accountant", "Senior Python Developer"] | null | 10001 +Senior Team Lead | daeL maeT roineS | 10002 +null | null | 10003 +[Head Human Resources, Reporting Analyst, Support Engineer, Tech Lead] | null | 10004 +null | null | 10005 +; + +reverseNested +required_capability: fn_reverse +FROM employees | SORT emp_no | EVAL name_reversed = REVERSE(REVERSE(first_name)), eq = name_reversed == first_name | KEEP first_name, name_reversed, eq, emp_no | LIMIT 5; + +first_name:keyword | name_reversed:keyword | eq:boolean | emp_no:integer +Georgi | Georgi | true | 10001 +Bezalel | Bezalel | true | 10002 +Parto | Parto | true | 10003 +Chirstian | Chirstian | true | 10004 +Kyoichi | Kyoichi | true | 10005 +; + +reverseRowNull +required_capability: fn_reverse +ROW x = null | EVAL y = REVERSE(x); + +x:null | y:null +null | null +; + + +reverseRowInlineCastWithNull +required_capability: fn_reverse +ROW x = 1 | EVAL y = REVERSE((null + 1)::string); + +x:integer | y:string +1 | null +; + +reverseWithTextFields +required_capability: fn_reverse +FROM books +| EVAL title_reversed = REVERSE(title), author_reversed_twice = REVERSE(REVERSE(author)), eq = author_reversed_twice == author +| KEEP title, title_reversed, author, author_reversed_twice, eq, book_no +| SORT book_no +| WHERE book_no IN ("1211", "1463") +| LIMIT 2; + +title:text | title_reversed:text | author:text | author_reversed_twice:text | eq:boolean | book_no:keyword +The brothers Karamazov | vozamaraK srehtorb ehT | Fyodor Dostoevsky | Fyodor Dostoevsky | true | 1211 +Realms of Tolkien: Images of Middle-earth | htrae-elddiM fo segamI :neikloT fo smlaeR | J. R. R. Tolkien | J. R. R. Tolkien | true | 1463 +; + + values required_capability: agg_values diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java index 03757d44a9f58..4f4f3d112247e 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java @@ -43,6 +43,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { private static final String REMOTE_CLUSTER = "cluster-a"; @@ -339,6 +340,108 @@ public void testSearchesWhereNonExistentClusterIsSpecifiedWithWildcards() { } } + /** + * Searches with LIMIT 0 are used by Kibana to get a list of columns. After the initial planning + * (which involves cross-cluster field-caps calls), it is a coordinator only operation at query time + * which uses a different pathway compared to queries that require data node (and remote data node) operations + * at query time. + */ + public void testCCSExecutionOnSearchesWithLimit0() { + setupTwoClusters(); + + // Ensure non-cross cluster queries have overall took time + try (EsqlQueryResponse resp = runQuery("FROM logs* | LIMIT 0")) { + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertNotNull(executionInfo); + assertThat(executionInfo.isCrossClusterSearch(), is(false)); + assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); + } + + // ensure cross-cluster searches have overall took time and correct per-cluster details in EsqlExecutionInfo + try (EsqlQueryResponse resp = runQuery("FROM logs*,cluster-a:* | LIMIT 0")) { + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertNotNull(executionInfo); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + long overallTookMillis = executionInfo.overallTook().millis(); + assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); + + EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); + assertThat(remoteCluster.getIndexExpression(), equalTo("*")); + assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertNull(remoteCluster.getTotalShards()); + assertNull(remoteCluster.getSuccessfulShards()); + assertNull(remoteCluster.getSkippedShards()); + assertNull(remoteCluster.getFailedShards()); + + EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER); + assertThat(localCluster.getIndexExpression(), equalTo("logs*")); + assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertNull(localCluster.getTotalShards()); + assertNull(localCluster.getSuccessfulShards()); + assertNull(localCluster.getSkippedShards()); + assertNull(localCluster.getFailedShards()); + } + + try (EsqlQueryResponse resp = runQuery("FROM logs*,cluster-a:nomatch* | LIMIT 0")) { + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertNotNull(executionInfo); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + long overallTookMillis = executionInfo.overallTook().millis(); + assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); + + EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); + assertThat(remoteCluster.getIndexExpression(), equalTo("nomatch*")); + assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(remoteCluster.getTook().millis(), equalTo(0L)); + assertThat(remoteCluster.getTotalShards(), equalTo(0)); + assertThat(remoteCluster.getSuccessfulShards(), equalTo(0)); + assertThat(remoteCluster.getSkippedShards(), equalTo(0)); + assertThat(remoteCluster.getFailedShards(), equalTo(0)); + + EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER); + assertThat(localCluster.getIndexExpression(), equalTo("logs*")); + assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertNull(localCluster.getTotalShards()); + assertNull(localCluster.getSuccessfulShards()); + assertNull(localCluster.getSkippedShards()); + assertNull(localCluster.getFailedShards()); + } + + try (EsqlQueryResponse resp = runQuery("FROM nomatch*,cluster-a:* | LIMIT 0")) { + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertNotNull(executionInfo); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + long overallTookMillis = executionInfo.overallTook().millis(); + assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); + + EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); + assertThat(remoteCluster.getIndexExpression(), equalTo("*")); + assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertNull(remoteCluster.getTotalShards()); + assertNull(remoteCluster.getSuccessfulShards()); + assertNull(remoteCluster.getSkippedShards()); + assertNull(remoteCluster.getFailedShards()); + + EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER); + assertThat(localCluster.getIndexExpression(), equalTo("nomatch*")); + // TODO: in https://github.com/elastic/elasticsearch/issues/112886, this will be changed to be SKIPPED + assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + } + } + public void testMetadataIndex() { Map testClusterInfo = setupTwoClusters(); int localNumShards = (Integer) testClusterInfo.get("local.num_shards"); diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseEvaluator.java new file mode 100644 index 0000000000000..68ea53ad342e1 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseEvaluator.java @@ -0,0 +1,111 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Reverse}. + * This class is generated. Do not edit it. + */ +public final class ReverseEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator val; + + private final DriverContext driverContext; + + public ReverseEvaluator(Source source, EvalOperator.ExpressionEvaluator val, + DriverContext driverContext) { + this.val = val; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try (BytesRefBlock valBlock = (BytesRefBlock) val.eval(page)) { + BytesRefVector valVector = valBlock.asVector(); + if (valVector == null) { + return eval(page.getPositionCount(), valBlock); + } + return eval(page.getPositionCount(), valVector).asBlock(); + } + } + + public BytesRefBlock eval(int positionCount, BytesRefBlock valBlock) { + try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + BytesRef valScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (valBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (valBlock.getValueCount(p) != 1) { + if (valBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendBytesRef(Reverse.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch))); + } + return result.build(); + } + } + + public BytesRefVector eval(int positionCount, BytesRefVector valVector) { + try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { + BytesRef valScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + result.appendBytesRef(Reverse.process(valVector.getBytesRef(p, valScratch))); + } + return result.build(); + } + } + + @Override + public String toString() { + return "ReverseEvaluator[" + "val=" + val + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(val); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory val; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory val) { + this.source = source; + this.val = val; + } + + @Override + public ReverseEvaluator get(DriverContext context) { + return new ReverseEvaluator(source, val.get(context), context); + } + + @Override + public String toString() { + return "ReverseEvaluator[" + "val=" + val + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 8d209709b0ef8..17be7d37c1d74 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -27,6 +27,11 @@ */ public class EsqlCapabilities { public enum Cap { + /** + * Support for function {@code REVERSE}. + */ + FN_REVERSE, + /** * Support for function {@code CBRT}. Done in #108574. */ @@ -278,6 +283,11 @@ public enum Cap { */ TO_DATE_NANOS(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG), + /** + * Support for datetime in least and greatest functions + */ + LEAST_GREATEST_FOR_DATES, + /** * Support CIDRMatch in CombineDisjunctions rule. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java index 5888e30747557..d8692faef5290 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java @@ -14,8 +14,6 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.planner.Layout; -import java.util.function.Function; - import static org.elasticsearch.compute.data.BlockUtils.fromArrayRow; import static org.elasticsearch.compute.data.BlockUtils.toJavaObject; @@ -23,6 +21,10 @@ * Expressions that have a mapping to an {@link ExpressionEvaluator}. */ public interface EvaluatorMapper { + interface ToEvaluator { + ExpressionEvaluator.Factory apply(Expression expression); + } + /** *

* Note for implementors: @@ -50,7 +52,7 @@ public interface EvaluatorMapper { * garbage. Or return an evaluator that throws when run. *

*/ - ExpressionEvaluator.Factory toEvaluator(Function toEvaluator); + ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator); /** * Fold using {@link #toEvaluator} so you don't need a "by hand" diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 87f861041ca5a..704bf0d796dce 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -125,6 +125,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.RTrim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Repeat; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Replace; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Reverse; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Right; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Space; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split; @@ -301,22 +302,23 @@ private FunctionDefinition[][] functions() { def(Tau.class, Tau::new, "tau") }, // string new FunctionDefinition[] { - def(Length.class, Length::new, "length"), - def(Substring.class, Substring::new, "substring"), def(Concat.class, Concat::new, "concat"), + def(EndsWith.class, EndsWith::new, "ends_with"), def(LTrim.class, LTrim::new, "ltrim"), - def(RTrim.class, RTrim::new, "rtrim"), - def(Trim.class, Trim::new, "trim"), def(Left.class, Left::new, "left"), + def(Length.class, Length::new, "length"), + def(Locate.class, Locate::new, "locate"), + def(RTrim.class, RTrim::new, "rtrim"), + def(Repeat.class, Repeat::new, "repeat"), def(Replace.class, Replace::new, "replace"), + def(Reverse.class, Reverse::new, "reverse"), def(Right.class, Right::new, "right"), + def(Space.class, Space::new, "space"), def(StartsWith.class, StartsWith::new, "starts_with"), - def(EndsWith.class, EndsWith::new, "ends_with"), + def(Substring.class, Substring::new, "substring"), def(ToLower.class, ToLower::new, "to_lower"), def(ToUpper.class, ToUpper::new, "to_upper"), - def(Locate.class, Locate::new, "locate"), - def(Repeat.class, Repeat::new, "repeat"), - def(Space.class, Space::new, "space") }, + def(Trim.class, Trim::new, "trim") }, // date new FunctionDefinition[] { def(DateDiff.class, DateDiff::new, "date_diff"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java index 5fabfe0e03d89..3357b2abf0e0f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java @@ -39,7 +39,6 @@ import java.time.ZoneOffset; import java.util.ArrayList; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; @@ -241,7 +240,7 @@ public boolean foldable() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (field.dataType() == DataType.DATETIME) { Rounding.Prepared preparedRounding; if (buckets.dataType().isWholeNumber()) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java index 0865e070aecd4..75a9883a77102 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java @@ -36,7 +36,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -107,7 +106,7 @@ static int process( } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new CategorizeEvaluator.Factory( source(), toEvaluator.apply(field), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java index 14b0c872a3b86..afe9bf6e45eda 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java @@ -43,6 +43,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.Locate; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Repeat; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Replace; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Reverse; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Right; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; @@ -100,6 +101,7 @@ public static List getNamedWriteables() { entries.add(Right.ENTRY); entries.add(Repeat.ENTRY); entries.add(Replace.ENTRY); + entries.add(Reverse.ENTRY); entries.add(Round.ENTRY); entries.add(Split.ENTRY); entries.add(Substring.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java index 6acb8ea974ed0..62e5651d07dca 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java @@ -50,7 +50,7 @@ public final class Case extends EsqlScalarFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Case", Case::new); record Condition(Expression condition, Expression value) { - ConditionEvaluatorSupplier toEvaluator(Function toEvaluator) { + ConditionEvaluatorSupplier toEvaluator(ToEvaluator toEvaluator) { return new ConditionEvaluatorSupplier(condition.source(), toEvaluator.apply(condition), toEvaluator.apply(value)); } } @@ -311,7 +311,7 @@ private Expression finishPartialFold(List newChildren) { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { List conditionsFactories = conditions.stream().map(c -> c.toEvaluator(toEvaluator)).toList(); ExpressionEvaluator.Factory elseValueFactory = toEvaluator.apply(elseValue); ElementType resultType = PlannerUtils.toElementType(dataType()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java index 7c0427a95d478..9d815d15accdc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import java.util.stream.Stream; import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; @@ -44,7 +43,7 @@ public class Greatest extends EsqlScalarFunction implements OptionalArgument { private DataType dataType; @FunctionInfo( - returnType = { "boolean", "double", "integer", "ip", "keyword", "long", "text", "version" }, + returnType = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" }, description = "Returns the maximum value from multiple columns. This is similar to <>\n" + "except it is intended to run on multiple columns at once.", note = "When run on `keyword` or `text` fields, this returns the last string in alphabetical order. " @@ -55,12 +54,12 @@ public Greatest( Source source, @Param( name = "first", - type = { "boolean", "double", "integer", "ip", "keyword", "long", "text", "version" }, + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" }, description = "First of the columns to evaluate." ) Expression first, @Param( name = "rest", - type = { "boolean", "double", "integer", "ip", "keyword", "long", "text", "version" }, + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" }, description = "The rest of the columns to evaluate.", optional = true ) List rest @@ -138,7 +137,7 @@ public boolean foldable() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { // force datatype initialization var dataType = dataType(); ExpressionEvaluator.Factory[] factories = children().stream() @@ -153,7 +152,7 @@ public ExpressionEvaluator.Factory toEvaluator(Function> except it is intended to run on multiple columns at once.", examples = @Example(file = "math", tag = "least") @@ -53,12 +52,12 @@ public Least( Source source, @Param( name = "first", - type = { "boolean", "double", "integer", "ip", "keyword", "long", "text", "version" }, + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" }, description = "First of the columns to evaluate." ) Expression first, @Param( name = "rest", - type = { "boolean", "double", "integer", "ip", "keyword", "long", "text", "version" }, + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" }, description = "The rest of the columns to evaluate.", optional = true ) List rest @@ -136,7 +135,7 @@ public boolean foldable() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { // force datatype initialization var dataType = dataType(); @@ -152,7 +151,7 @@ public ExpressionEvaluator.Factory toEvaluator(Function factories(); @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return evaluator(toEvaluator.apply(field())); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64.java index 873d496bfc8fd..7f9d0d3f2e647 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.Base64; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; @@ -96,9 +95,7 @@ static BytesRef process(BytesRef field, @Fixed(includeInToString = false, build } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return switch (PlannerUtils.toElementType(field.dataType())) { case BYTES_REF -> new FromBase64Evaluator.Factory(source(), toEvaluator.apply(field), context -> new BytesRefBuilder()); case NULL -> EvalOperator.CONSTANT_NULL_FACTORY; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64.java index ab8287413c614..c23cef31f32f5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.Base64; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; @@ -90,9 +89,7 @@ static BytesRef process(BytesRef field, @Fixed(includeInToString = false, build } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return switch (PlannerUtils.toElementType(field.dataType())) { case BYTES_REF -> new ToBase64Evaluator.Factory(source(), toEvaluator.apply(field), context -> new BytesRefBuilder()); case NULL -> EvalOperator.CONSTANT_NULL_FACTORY; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java index f9039417e48a6..f6a23a5d5962e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java @@ -37,7 +37,6 @@ import java.util.Map; import java.util.Set; import java.util.function.BiFunction; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -227,7 +226,7 @@ static int process(BytesRef unit, long startTimestamp, long endTimestamp) throws } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { ExpressionEvaluator.Factory startTimestampEvaluator = toEvaluator.apply(startTimestamp); ExpressionEvaluator.Factory endTimestampEvaluator = toEvaluator.apply(endTimestamp); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java index bbb19ca0eecab..501dfd431f106 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java @@ -31,7 +31,6 @@ import java.time.ZoneId; import java.time.temporal.ChronoField; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isDate; import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isStringAndExact; @@ -108,7 +107,7 @@ public String getWriteableName() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var fieldEvaluator = toEvaluator.apply(children().get(1)); if (children().get(0).foldable()) { ChronoField chrono = chronoField(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java index bfca72a563c05..60bc014ccbeec 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.util.List; import java.util.Locale; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -139,7 +138,7 @@ static BytesRef process(long val, BytesRef formatter, @Fixed Locale locale) { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var fieldEvaluator = toEvaluator.apply(field); if (format == null) { return new DateFormatConstantEvaluator.Factory(source(), fieldEvaluator, DEFAULT_DATE_TIME_FORMATTER); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java index 27b613e0e6d5a..1aaa227c3846e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.time.ZoneId; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.common.time.DateFormatter.forPattern; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; @@ -136,7 +135,7 @@ static long process(BytesRef val, BytesRef formatter, @Fixed ZoneId zoneId) thro } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { ZoneId zone = UTC; // TODO session timezone? ExpressionEvaluator.Factory fieldEvaluator = toEvaluator.apply(field); if (format == null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java index d5ec3d1d96fae..35a705f418906 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java @@ -32,7 +32,6 @@ import java.time.ZoneOffset; import java.util.List; import java.util.concurrent.TimeUnit; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -199,7 +198,7 @@ private static Rounding.Prepared createRounding(final Duration duration, final Z } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var fieldEvaluator = toEvaluator.apply(timestampField); if (interval.foldable() == false) { throw new IllegalArgumentException("Function [" + sourceText() + "] has invalid interval [" + interval.sourceText() + "]."); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java index 0654aec3a0522..d259fc6ae57ce 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java @@ -25,7 +25,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; public class Now extends EsqlConfigurationFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Now", Now::new); @@ -90,7 +89,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return dvrCtx -> new NowEvaluator(source(), now, dvrCtx); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java index c141beeefb1ea..51430603a4077 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; @@ -113,12 +112,12 @@ public boolean foldable() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var ipEvaluatorSupplier = toEvaluator.apply(ipField); return new CIDRMatchEvaluator.Factory( source(), ipEvaluatorSupplier, - matches.stream().map(x -> toEvaluator.apply(x)).toArray(EvalOperator.ExpressionEvaluator.Factory[]::new) + matches.stream().map(toEvaluator::apply).toArray(EvalOperator.ExpressionEvaluator.Factory[]::new) ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java index 60b464b26750a..26e75e752f681 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -120,7 +119,7 @@ public boolean foldable() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var ipEvaluatorSupplier = toEvaluator.apply(ipField); var prefixLengthV4EvaluatorSupplier = toEvaluator.apply(prefixLengthV4Field); var prefixLengthV6EvaluatorSupplier = toEvaluator.apply(prefixLengthV6Field); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Abs.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Abs.java index 363b70ef5ed12..ba47fd15e9c9d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Abs.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Abs.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; public class Abs extends UnaryScalarFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Abs", Abs::new); @@ -69,7 +68,7 @@ static int process(int fieldVal) { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var field = toEvaluator.apply(field()); if (dataType() == DataType.DOUBLE) { return new AbsDoubleEvaluator.Factory(source(), field); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbstractTrigonometricFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbstractTrigonometricFunction.java index 8353fe24b3dd0..f44e7b029643c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbstractTrigonometricFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbstractTrigonometricFunction.java @@ -16,7 +16,6 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; import java.io.IOException; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric; @@ -39,7 +38,7 @@ protected AbstractTrigonometricFunction(StreamInput in) throws IOException { protected abstract EvalOperator.ExpressionEvaluator.Factory doubleEvaluator(EvalOperator.ExpressionEvaluator.Factory field); @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return doubleEvaluator(Cast.cast(source(), field().dataType(), DataType.DOUBLE, toEvaluator.apply(field()))); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2.java index f940cb6d68554..7dbc0001f4b3d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric; @@ -118,7 +117,7 @@ public boolean foldable() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var yEval = Cast.cast(source(), y.dataType(), DataType.DOUBLE, toEvaluator.apply(y)); var xEval = Cast.cast(source(), x.dataType(), DataType.DOUBLE, toEvaluator.apply(x)); return new Atan2Evaluator.Factory(source(), yEval, xEval); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Cbrt.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Cbrt.java index 364e91aad8b1b..dcc704318ca5f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Cbrt.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Cbrt.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric; @@ -56,7 +55,7 @@ public String getWriteableName() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var field = toEvaluator.apply(field()); var fieldType = field().dataType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Ceil.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Ceil.java index 909de387c62ff..f7295421de8aa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Ceil.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Ceil.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric; @@ -64,7 +63,7 @@ public String getWriteableName() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (dataType().isWholeNumber()) { return toEvaluator.apply(field()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Exp.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Exp.java index a0d9937fc87b8..7abef8bba711d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Exp.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Exp.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; /** * Returns the value of e raised to the power of tbe number specified as parameter @@ -58,9 +57,7 @@ public String getWriteableName() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var field = toEvaluator.apply(field()); var fieldType = field().dataType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Floor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Floor.java index 638770f2f079a..7e727c1c2cada 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Floor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Floor.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric; @@ -66,7 +65,7 @@ public String getWriteableName() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (dataType().isWholeNumber()) { return toEvaluator.apply(field()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log.java index da11d1e77885b..4528897c194ca 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -145,7 +144,7 @@ public DataType dataType() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var valueEval = Cast.cast(source(), value.dataType(), DataType.DOUBLE, toEvaluator.apply(value)); if (base != null) { var baseEval = Cast.cast(source(), base.dataType(), DataType.DOUBLE, toEvaluator.apply(base)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10.java index ae725f6ed6498..1e987651b686c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric; @@ -62,7 +61,7 @@ public String getWriteableName() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var field = toEvaluator.apply(field()); var fieldType = field().dataType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pow.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pow.java index 46d80635823ca..3f5249e3e8cb3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pow.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pow.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -128,7 +127,7 @@ public DataType dataType() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var baseEval = Cast.cast(source(), base.dataType(), DataType.DOUBLE, toEvaluator.apply(base)); var expEval = Cast.cast(source(), exponent.dataType(), DataType.DOUBLE, toEvaluator.apply(exponent)); return new PowEvaluator.Factory(source(), baseEval, expEval); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Round.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Round.java index 8fcb04d021e7a..b1baa6c55ce47 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Round.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Round.java @@ -31,7 +31,6 @@ import java.util.Arrays; import java.util.List; import java.util.function.BiFunction; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -167,7 +166,7 @@ public DataType dataType() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { DataType fieldType = dataType(); if (fieldType == DataType.DOUBLE) { return toEvaluator(toEvaluator, RoundDoubleNoDecimalsEvaluator.Factory::new, RoundDoubleEvaluator.Factory::new); @@ -185,7 +184,7 @@ public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator, + ToEvaluator toEvaluator, BiFunction noDecimals, TriFunction withDecimals ) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Signum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Signum.java index e78c2ce90e6c1..9c7a5fdcaa236 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Signum.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Signum.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; public class Signum extends UnaryScalarFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Signum", Signum::new); @@ -56,9 +55,7 @@ public String getWriteableName() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var field = toEvaluator.apply(field()); var fieldType = field().dataType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Sqrt.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Sqrt.java index d1af693d8aa7f..080c0448e082c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Sqrt.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Sqrt.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric; @@ -56,7 +55,7 @@ public String getWriteableName() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var field = toEvaluator.apply(field()); var fieldType = field().dataType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java index 998a1815cbada..6a3b58728b192 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java @@ -84,7 +84,7 @@ protected final TypeResolution resolveType() { protected abstract TypeResolution resolveFieldType(); @Override - public final ExpressionEvaluator.Factory toEvaluator(java.util.function.Function toEvaluator) { + public final ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return evaluator(toEvaluator.apply(field())); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java index deb170d9e569c..72d96a86d31eb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java @@ -35,7 +35,6 @@ import java.util.Arrays; import java.util.List; import java.util.Objects; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -149,9 +148,7 @@ public boolean foldable() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return switch (PlannerUtils.toElementType(dataType())) { case BOOLEAN -> new MvAppendBooleanEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2)); case BYTES_REF -> new MvAppendBytesRefEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java index fa9475055515f..1996744a76567 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java @@ -28,7 +28,6 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import java.io.IOException; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -87,7 +86,7 @@ public DataType dataType() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new EvaluatorFactory(toEvaluator.apply(left()), toEvaluator.apply(right())); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java index 212f626090789..cf49607893aae 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java @@ -14,7 +14,6 @@ import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.operator.EvalOperator; -import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.search.aggregations.metrics.CompensatedSum; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -33,7 +32,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -110,7 +108,7 @@ public boolean foldable() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return switch (PlannerUtils.toElementType(field.dataType())) { case DOUBLE -> new MvPSeriesWeightedSumDoubleEvaluator.Factory( source(), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java index 1eb0c70a7b08e..f3a63c835bd34 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java @@ -34,7 +34,6 @@ import java.math.BigDecimal; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -120,7 +119,7 @@ public DataType dataType() { } @Override - public final ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public final ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var fieldEval = toEvaluator.apply(field); var percentileEval = Cast.cast(source(), percentile.dataType(), DOUBLE, toEvaluator.apply(percentile)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java index c332e94b20049..9846ebe4111c0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java @@ -37,7 +37,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -178,9 +177,7 @@ public boolean foldable() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (start.foldable() && end.foldable()) { int startOffset = stringToInt(String.valueOf(start.fold())); int endOffset = stringToInt(String.valueOf(end.fold())); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java index 4ed9a01c29797..d9e41233952de 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java @@ -48,7 +48,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -146,9 +145,7 @@ public boolean foldable() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { boolean ordering = true; if (isValidOrder() == false) { throw new IllegalArgumentException( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java index fd3b9e7664dff..d6a30c6ca151c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java @@ -32,7 +32,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -130,9 +129,7 @@ public Nullability nullable() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new MvZipEvaluator.Factory( source(), toEvaluator.apply(mvLeft), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java index 30c6abc5398e3..575bb085c41f7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java @@ -35,7 +35,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -192,8 +191,8 @@ public boolean foldable() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { - List childEvaluators = children().stream().map(toEvaluator).toList(); + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + List childEvaluators = children().stream().map(toEvaluator::apply).toList(); return new ExpressionEvaluator.Factory() { @Override public ExpressionEvaluator get(DriverContext context) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java index f8b05aea324dc..46538b77edc74 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java @@ -151,6 +151,8 @@ *
  • {@code docs/reference/esql/functions/parameters/myfunction.asciidoc}
    • *
  • {@code docs/reference/esql/functions/signature/myfunction.svg}
    • *
  • {@code docs/reference/esql/functions/types/myfunction.asciidoc}
    • + *
  • {@code docs/reference/esql/functions/kibana/definition/myfunction.json}
    • + *
  • {@code docs/reference/esql/functions/kibana/docs/myfunction.asciidoc}
    • * * * Make sure to commit them. Add a reference to the @@ -194,6 +196,9 @@ * for your function. Now add something like {@code required_capability: my_function} * to all of your csv-spec tests. Run those csv-spec tests as integration tests to double * check that they run on the main branch. + *

    + * **Note:** you may notice tests gated based on Elasticsearch version. This was the old way + * of doing things. Now, we use specific capabilities for each function. * *
  • * Open the PR. The subject and description of the PR are important because those'll turn @@ -201,7 +206,7 @@ * happy. But functions don't need an essay. *
    • *
  • - * Add the {@code >enhancement} and {@code :Query Languages/ES|QL} tags if you are able. + * Add the {@code >enhancement} and {@code :Analytics/ES|QL} tags if you are able. * Request a review if you can, probably from one of the folks that github proposes to you. *
    • *
  • diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java index 6fd4f79125a21..1a51af8dfeeb4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java @@ -13,9 +13,9 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import java.util.Map; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asLuceneComponent2D; import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asLuceneComponent2Ds; @@ -34,15 +34,12 @@ abstract class SpatialEvaluatorFactory { this.factoryCreator = factoryCreator; } - public abstract EvalOperator.ExpressionEvaluator.Factory get( - SpatialSourceSupplier function, - Function toEvaluator - ); + public abstract EvalOperator.ExpressionEvaluator.Factory get(SpatialSourceSupplier function, EvaluatorMapper.ToEvaluator toEvaluator); static EvalOperator.ExpressionEvaluator.Factory makeSpatialEvaluator( SpatialSourceSupplier s, Map> evaluatorRules, - Function toEvaluator + EvaluatorMapper.ToEvaluator toEvaluator ) { var evaluatorKey = new SpatialEvaluatorKey( s.crsType(), @@ -149,10 +146,7 @@ protected static class SpatialEvaluatorFactoryWithFields extends SpatialEvaluato } @Override - public EvalOperator.ExpressionEvaluator.Factory get( - SpatialSourceSupplier s, - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory get(SpatialSourceSupplier s, EvaluatorMapper.ToEvaluator toEvaluator) { return factoryCreator.apply(s.source(), toEvaluator.apply(s.left()), toEvaluator.apply(s.right())); } } @@ -176,10 +170,7 @@ protected static class SpatialEvaluatorWithConstantFactory extends SpatialEvalua } @Override - public EvalOperator.ExpressionEvaluator.Factory get( - SpatialSourceSupplier s, - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory get(SpatialSourceSupplier s, EvaluatorMapper.ToEvaluator toEvaluator) { return factoryCreator.apply(s.source(), toEvaluator.apply(s.left()), asLuceneComponent2D(s.crsType(), s.right())); } } @@ -205,10 +196,7 @@ protected static class SpatialEvaluatorWithConstantArrayFactory extends SpatialE } @Override - public EvalOperator.ExpressionEvaluator.Factory get( - SpatialSourceSupplier s, - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory get(SpatialSourceSupplier s, EvaluatorMapper.ToEvaluator toEvaluator) { return factoryCreator.apply(s.source(), toEvaluator.apply(s.left()), asLuceneComponent2Ds(s.crsType(), s.right())); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java index 68ca793089499..ee2b4450a64ff 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java @@ -32,7 +32,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.Function; import java.util.function.Predicate; import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asGeometryDocValueReader; @@ -112,9 +111,7 @@ public SpatialRelatesFunction surrogate() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return SpatialEvaluatorFactory.makeSpatialEvaluator(this, evaluatorRules(), toEvaluator); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java index 14bded51aa55f..17bcc68004bff 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java @@ -30,7 +30,6 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import java.io.IOException; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.makeGeometryFromLiteral; @@ -177,9 +176,7 @@ public Object fold() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (right().foldable()) { return toEvaluator(toEvaluator, left(), makeGeometryFromLiteral(right()), leftDocValues); } else if (left().foldable()) { @@ -209,7 +206,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator( } private EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator, + ToEvaluator toEvaluator, Expression field, Geometry geometry, boolean docValues @@ -222,7 +219,7 @@ private EvalOperator.ExpressionEvaluator.Factory toEvaluator( } private EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator, + ToEvaluator toEvaluator, Expression field, Point point, boolean docValues diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StX.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StX.java index 18046135933b0..d1d85b03eb18a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StX.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StX.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; @@ -72,9 +71,7 @@ protected Expression.TypeResolution resolveType() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new StXFromWKBEvaluator.Factory(toEvaluator.apply(field()), source()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StY.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StY.java index bf97c3e2a3547..2056dcaed87a7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StY.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StY.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; @@ -72,9 +71,7 @@ protected TypeResolution resolveType() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new StYFromWKBEvaluator.Factory(toEvaluator.apply(field()), source()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java index 23ee942bcf53a..46ecc9e026d3d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import java.util.stream.Stream; import static org.elasticsearch.common.unit.ByteSizeUnit.MB; @@ -106,8 +105,8 @@ public boolean foldable() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { - var values = children().stream().map(toEvaluator).toArray(ExpressionEvaluator.Factory[]::new); + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + var values = children().stream().map(toEvaluator::apply).toArray(ExpressionEvaluator.Factory[]::new); return new ConcatEvaluator.Factory(source(), context -> new BreakingBytesRefBuilder(context.breaker(), "concat"), values); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java index 1d2b743fe5a7a..e97e65a3e60fc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -126,7 +125,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new EndsWithEvaluator.Factory(source(), toEvaluator.apply(str), toEvaluator.apply(suffix)); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrim.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrim.java index ece70da51ef19..8a4a5f4d841a5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrim.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrim.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -69,7 +68,7 @@ protected TypeResolution resolveType() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new LTrimEvaluator.Factory(source(), toEvaluator.apply(field())); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java index b0e5b41f971e1..e7572caafd8f5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -95,7 +94,7 @@ static BytesRef process( } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new LeftEvaluator.Factory( source(), context -> new BytesRef(), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java index 241eab6d5b904..f4bb7f35cb466 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -87,7 +86,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new LengthEvaluator.Factory(source(), toEvaluator.apply(field())); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Locate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Locate.java index f6eff2fcbd6b3..528baa613cc02 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Locate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Locate.java @@ -28,7 +28,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -161,7 +160,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { ExpressionEvaluator.Factory strExpr = toEvaluator.apply(str); ExpressionEvaluator.Factory substrExpr = toEvaluator.apply(substr); if (start == null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java index bb923ec924d31..b46c46c89deba 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java @@ -22,7 +22,6 @@ import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -79,9 +78,7 @@ protected TypeResolution resolveType() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return AutomataMatch.toEvaluator(source(), toEvaluator.apply(field()), pattern().createAutomaton()); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrim.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrim.java index 4c210607cfbe0..b79e1adf99a20 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrim.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrim.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -69,7 +68,7 @@ protected TypeResolution resolveType() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new RTrimEvaluator.Factory(source(), toEvaluator.apply(field())); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java index 3ff28e08f4ce1..2cc14399df2ae 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.common.unit.ByteSizeUnit.MB; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; @@ -143,7 +142,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { ExpressionEvaluator.Factory strExpr = toEvaluator.apply(str); if (number.foldable()) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java index 30c8793fe371a..4fa191244cb42 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -146,7 +145,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var strEval = toEvaluator.apply(str); var newStrEval = toEvaluator.apply(newStr); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java new file mode 100644 index 0000000000000..bf4e47d8d0de4 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; + +import java.io.IOException; +import java.text.BreakIterator; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import static org.elasticsearch.common.util.ArrayUtils.reverseArray; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; + +/** + * Function that reverses a string. + */ +public class Reverse extends UnaryScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Reverse", Reverse::new); + + @FunctionInfo( + returnType = { "keyword", "text" }, + description = "Returns a new string representing the input string in reverse order.", + examples = { + @Example(file = "string", tag = "reverse"), + @Example( + file = "string", + tag = "reverseEmoji", + description = "`REVERSE` works with unicode, too! It keeps unicode grapheme clusters together during reversal." + ) } + ) + public Reverse( + Source source, + @Param( + name = "str", + type = { "keyword", "text" }, + description = "String expression. If `null`, the function returns `null`." + ) Expression field + ) { + super(source, field); + } + + private Reverse(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + + return isString(field, sourceText(), DEFAULT); + } + + /** + * Reverses a unicode string, keeping grapheme clusters together + * @param str + * @return + */ + public static String reverseStringWithUnicodeCharacters(String str) { + BreakIterator boundary = BreakIterator.getCharacterInstance(Locale.ROOT); + boundary.setText(str); + + List characters = new ArrayList<>(); + int start = boundary.first(); + for (int end = boundary.next(); end != BreakIterator.DONE; start = end, end = boundary.next()) { + characters.add(str.substring(start, end)); + } + + StringBuilder reversed = new StringBuilder(str.length()); + for (int i = characters.size() - 1; i >= 0; i--) { + reversed.append(characters.get(i)); + } + + return reversed.toString(); + } + + private static boolean isOneByteUTF8(BytesRef ref) { + int end = ref.offset + ref.length; + for (int i = ref.offset; i < end; i++) { + if (ref.bytes[i] < 0) { + return false; + } + } + return true; + } + + @Evaluator + static BytesRef process(BytesRef val) { + if (isOneByteUTF8(val)) { + // this is the fast path. we know we can just reverse the bytes. + BytesRef reversed = BytesRef.deepCopyOf(val); + reverseArray(reversed.bytes, reversed.offset, reversed.length); + return reversed; + } + return BytesRefs.toBytesRef(reverseStringWithUnicodeCharacters(val.utf8ToString())); + } + + @Override + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + var fieldEvaluator = toEvaluator.apply(field); + return new ReverseEvaluator.Factory(source(), fieldEvaluator); + } + + @Override + public Expression replaceChildren(List newChildren) { + assert newChildren.size() == 1; + return new Reverse(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Reverse::new, field); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java index ab6d3bf6cef99..b069b984ea81e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -99,7 +98,7 @@ static BytesRef process( } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new RightEvaluator.Factory( source(), context -> new BytesRef(), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java index e6225a008fceb..6481ce5764e1f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.common.unit.ByteSizeUnit.MB; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; @@ -111,7 +110,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (field.foldable()) { Object folded = field.fold(); if (folded instanceof Integer num) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java index 79ff23ac6737a..b1f5da56d011b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java @@ -28,7 +28,6 @@ import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -158,7 +157,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var str = toEvaluator.apply(left()); if (right().foldable() == false) { return new SplitVariableEvaluator.Factory(source(), str, toEvaluator.apply(right()), context -> new BytesRef()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java index fc40a73471194..2256ec2179adf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -123,7 +122,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new StartsWithEvaluator.Factory(source(), toEvaluator.apply(str), toEvaluator.apply(prefix)); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java index 7c2ecd0c60e49..73ea409676fbd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -180,7 +179,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var strFactory = toEvaluator.apply(str); var startFactory = toEvaluator.apply(start); if (length == null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java index 62255a0a31ea6..c475469488d7b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.List; import java.util.Locale; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -96,7 +95,7 @@ static BytesRef process(BytesRef val, @Fixed Locale locale) { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var fieldEvaluator = toEvaluator.apply(field); return new ToLowerEvaluator.Factory(source(), fieldEvaluator, configuration().locale()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java index e6eba0d01e4da..1b5084a7916ef 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.List; import java.util.Locale; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -96,7 +95,7 @@ static BytesRef process(BytesRef val, @Fixed Locale locale) { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var fieldEvaluator = toEvaluator.apply(field); return new ToUpperEvaluator.Factory(source(), fieldEvaluator, configuration().locale()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Trim.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Trim.java index 36dc3d97992ab..1fe7529caa2da 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Trim.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Trim.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -69,7 +68,7 @@ protected TypeResolution resolveType() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var field = toEvaluator.apply(field()); return new TrimEvaluator.Factory(source(), field); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java index 15470bb56b29f..714c4ca04a862 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java @@ -23,7 +23,6 @@ import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -85,9 +84,7 @@ protected TypeResolution resolveType() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return AutomataMatch.toEvaluator( source(), toEvaluator.apply(field()), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java index 5b7cc74faed86..d407dd8bf7de1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java @@ -20,7 +20,6 @@ import java.time.Period; import java.time.temporal.TemporalAmount; import java.util.Collection; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD; @@ -159,7 +158,7 @@ public final Object fold() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (dataType() == DATETIME) { // One of the arguments has to be a datetime and the other a temporal amount. Expression datetimeArgument; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java index 400e70b641111..62201bcfa858d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; @@ -170,7 +169,7 @@ public static String formatIncompatibleTypesMessage(String symbol, DataType left } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var commonType = dataType(); var leftType = left().dataType(); if (leftType.isNumeric()) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java index 67b770d14339e..fb32282005f02 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java @@ -25,7 +25,6 @@ import java.time.Duration; import java.time.Period; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; @@ -61,7 +60,7 @@ public String getWriteableName() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { DataType type = dataType(); if (type.isNumeric()) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java index b50d70e69819d..db771a6354883 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java @@ -27,7 +27,6 @@ import java.time.ZoneId; import java.util.List; import java.util.Map; -import java.util.function.Function; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; @@ -168,9 +167,7 @@ public BinaryComparisonOperation getFunctionType() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { // Our type is always boolean, so figure out the evaluator type from the inputs DataType commonType = commonType(left().dataType(), right().dataType()); EvalOperator.ExpressionEvaluator.Factory lhs; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java index 333f32e82c579..eda6aadccc86a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java @@ -33,7 +33,6 @@ import java.util.BitSet; import java.util.Collections; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; @@ -213,9 +212,7 @@ protected Expression canonicalize() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var commonType = commonType(); EvalOperator.ExpressionEvaluator.Factory lhs; EvalOperator.ExpressionEvaluator.Factory[] factories; @@ -226,7 +223,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator( .toArray(EvalOperator.ExpressionEvaluator.Factory[]::new); } else { lhs = toEvaluator.apply(value); - factories = list.stream().map(e -> toEvaluator.apply(e)).toArray(EvalOperator.ExpressionEvaluator.Factory[]::new); + factories = list.stream().map(toEvaluator::apply).toArray(EvalOperator.ExpressionEvaluator.Factory[]::new); } if (commonType == BOOLEAN) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java index ef4417a1c7a02..2b09a395c4a3d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java @@ -209,7 +209,7 @@ private Attribute attributeFromCache(int id) throws IOException { private void cacheAttribute(int id, Attribute attr) { assert id >= 0; if (id >= attributesCache.length) { - attributesCache = ArrayUtil.grow(attributesCache); + attributesCache = ArrayUtil.grow(attributesCache, id + 1); } attributesCache[id] = attr; } @@ -252,7 +252,7 @@ private EsField esFieldFromCache(int id) throws IOException { private void cacheEsField(int id, EsField field) { assert id >= 0; if (id >= esFieldsCache.length) { - esFieldsCache = ArrayUtil.grow(esFieldsCache); + esFieldsCache = ArrayUtil.grow(esFieldsCache, id + 1); } esFieldsCache[id] = field; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index d1f2007af2757..3ec39d1b0ac4b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -171,17 +171,21 @@ public void execute( null, null ); + String local = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; try ( var computeListener = ComputeListener.create( - RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, + local, transportService, rootTask, execInfo, configuration.getQueryStartTimeNanos(), - listener.map(r -> new Result(physicalPlan.output(), collectedPages, r.getProfiles(), execInfo)) + listener.map(r -> { + updateExecutionInfoAfterCoordinatorOnlyQuery(configuration.getQueryStartTimeNanos(), execInfo); + return new Result(physicalPlan.output(), collectedPages, r.getProfiles(), execInfo); + }) ) ) { - runCompute(rootTask, computeContext, coordinatorPlan, computeListener.acquireCompute()); + runCompute(rootTask, computeContext, coordinatorPlan, computeListener.acquireCompute(local)); return; } } else { @@ -247,6 +251,27 @@ public void execute( } } + private static void updateExecutionInfoAfterCoordinatorOnlyQuery(long queryStartNanos, EsqlExecutionInfo execInfo) { + long tookTimeNanos = System.nanoTime() - queryStartNanos; + execInfo.overallTook(new TimeValue(tookTimeNanos, TimeUnit.NANOSECONDS)); + if (execInfo.isCrossClusterSearch()) { + for (String clusterAlias : execInfo.clusterAliases()) { + // The local cluster 'took' time gets updated as part of the acquireCompute(local) call in the coordinator, so + // here we only need to update status for remote clusters since there are no remote ComputeListeners in this case. + // This happens in cross cluster searches that use LIMIT 0, e.g, FROM logs*,remote*:logs* | LIMIT 0. + if (clusterAlias.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) == false) { + execInfo.swapCluster(clusterAlias, (k, v) -> { + if (v.getStatus() == EsqlExecutionInfo.Cluster.Status.RUNNING) { + return new EsqlExecutionInfo.Cluster.Builder(v).setStatus(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL).build(); + } else { + return v; + } + }); + } + } + } + } + private List getRemoteClusters( Map clusterToConcreteIndices, Map clusterToOriginalIndices diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 608e45bb2085b..96391c841856f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -72,6 +72,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Predicate; @@ -245,6 +246,7 @@ private void preAnalyze( if (indexResolution.isValid()) { updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); updateExecutionInfoWithUnavailableClusters(executionInfo, indexResolution.getUnavailableClusters()); + updateTookTimeForRemoteClusters(executionInfo); Set newClusters = enrichPolicyResolver.groupIndicesPerCluster( indexResolution.get().concreteIndices().toArray(String[]::new) ).keySet(); @@ -285,6 +287,7 @@ static void updateExecutionInfoWithClustersWithNoMatchingIndices(EsqlExecutionIn } Set clustersRequested = executionInfo.clusterAliases(); Set clustersWithNoMatchingIndices = Sets.difference(clustersRequested, clustersWithResolvedIndices); + clustersWithNoMatchingIndices.removeAll(indexResolution.getUnavailableClusters()); /* * These are clusters in the original request that are not present in the field-caps response. They were * specified with an index or indices that do not exist, so the search on that cluster is done. @@ -304,6 +307,28 @@ static void updateExecutionInfoWithClustersWithNoMatchingIndices(EsqlExecutionIn } } + private void updateTookTimeForRemoteClusters(EsqlExecutionInfo executionInfo) { + if (executionInfo.isCrossClusterSearch()) { + for (String clusterAlias : executionInfo.clusterAliases()) { + if (clusterAlias.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) == false) { + executionInfo.swapCluster(clusterAlias, (k, v) -> { + if (v.getTook() == null && v.getStatus() != EsqlExecutionInfo.Cluster.Status.SKIPPED) { + // set took time in case we are finished with the remote cluster (e.g., FROM foo | LIMIT 0). + // this will be overwritten later if ES|QL operations happen on the remote cluster (the typical scenario) + TimeValue took = new TimeValue( + System.nanoTime() - configuration.getQueryStartTimeNanos(), + TimeUnit.NANOSECONDS + ); + return new EsqlExecutionInfo.Cluster.Builder(v).setTook(took).build(); + } else { + return v; + } + }); + } + } + } + } + private void preAnalyzeIndices( LogicalPlan parsed, EsqlExecutionInfo executionInfo, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DeepCopy.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DeepCopy.java index d25305a9ea190..593a444eceec2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DeepCopy.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DeepCopy.java @@ -20,7 +20,6 @@ import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import java.io.IOException; -import java.util.function.Function; /** * Expression that makes a deep copy of the block it receives. @@ -41,9 +40,7 @@ public String getWriteableName() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { EvalOperator.ExpressionEvaluator.Factory childEval = toEvaluator.apply(child()); return ctx -> new EvalOperator.ExpressionEvaluator() { private final EvalOperator.ExpressionEvaluator child = childEval.get(ctx); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistryTests.java index 7f19419b21816..801bd8700d014 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistryTests.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static java.util.Collections.emptyList; import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration; @@ -258,9 +257,7 @@ protected NodeInfo info() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return null; } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java index 7cc03be7d6273..311e3e3d89149 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java @@ -100,6 +100,21 @@ public static Iterable parameters() { ) ) ); + suppliers.add( + new TestCaseSupplier( + "(a, b)", + List.of(DataType.DATETIME, DataType.DATETIME), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(1727877348000L, DataType.DATETIME, "a"), + new TestCaseSupplier.TypedData(1727790948000L, DataType.DATETIME, "b") + ), + "GreatestLongEvaluator[values=[MvMax[field=Attribute[channel=0]], MvMax[field=Attribute[channel=1]]]]", + DataType.DATETIME, + equalTo(1727877348000L) + ) + ) + ); return parameterSuppliersFromTypedData(anyNullIsNull(false, suppliers)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java index aa475f05ebe69..69842fde90312 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java @@ -99,6 +99,21 @@ public static Iterable parameters() { ) ) ); + suppliers.add( + new TestCaseSupplier( + "(a, b)", + List.of(DataType.DATETIME, DataType.DATETIME), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(1727877348000L, DataType.DATETIME, "a"), + new TestCaseSupplier.TypedData(1727790948000L, DataType.DATETIME, "b") + ), + "LeastLongEvaluator[values=[MvMin[field=Attribute[channel=0]], MvMin[field=Attribute[channel=1]]]]", + DataType.DATETIME, + equalTo(1727790948000L) + ) + ) + ); return parameterSuppliersFromTypedData(anyNullIsNull(false, suppliers)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java index f760694391ee4..c9b6de64e079d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.evaluator.EvalMapper; +import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.scalar.VaragsTestCaseBuilder; @@ -33,7 +34,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.function.Function; import java.util.function.Supplier; import static org.elasticsearch.compute.data.BlockUtils.toJavaObject; @@ -174,7 +174,7 @@ public void testCoalesceIsLazy() { Layout.Builder builder = new Layout.Builder(); buildLayout(builder, exp); Layout layout = builder.build(); - Function map = child -> { + EvaluatorMapper.ToEvaluator toEvaluator = child -> { if (child == evil) { return dvrCtx -> new EvalOperator.ExpressionEvaluator() { @Override @@ -189,7 +189,7 @@ public void close() {} return EvalMapper.toEvaluator(child, layout); }; try ( - EvalOperator.ExpressionEvaluator eval = exp.toEvaluator(map).get(driverContext()); + EvalOperator.ExpressionEvaluator eval = exp.toEvaluator(toEvaluator).get(driverContext()); Block block = eval.eval(row(testCase.getDataValues())) ) { assertThat(toJavaObject(block, 0), testCase.getMatcher()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseSerializationTests.java new file mode 100644 index 0000000000000..7b1ad8c9dffd0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseSerializationTests.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractUnaryScalarSerializationTests; + +public class ReverseSerializationTests extends AbstractUnaryScalarSerializationTests { + @Override + protected Reverse create(Source source, Expression child) { + return new Reverse(source, child); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java new file mode 100644 index 0000000000000..2873f18d53957 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; + +public class ReverseTests extends AbstractScalarFunctionTestCase { + public ReverseTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List suppliers = new ArrayList<>(); + + for (DataType stringType : new DataType[] { DataType.KEYWORD, DataType.TEXT }) { + for (var supplier : TestCaseSupplier.stringCases(stringType)) { + suppliers.add(makeSupplier(supplier)); + } + } + + return parameterSuppliersFromTypedData(suppliers); + } + + @Override + protected Expression build(Source source, List args) { + return new Reverse(source, args.get(0)); + } + + private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier) { + return new TestCaseSupplier(fieldSupplier.name(), List.of(fieldSupplier.type()), () -> { + var fieldTypedData = fieldSupplier.get(); + String expectedToString = "ReverseEvaluator[val=Attribute[channel=0]]"; + String value = BytesRefs.toString(fieldTypedData.data()); + String expectedValue = Reverse.reverseStringWithUnicodeCharacters(value); + + return new TestCaseSupplier.TestCase( + List.of(fieldTypedData), + expectedToString, + fieldSupplier.type(), + equalTo(new BytesRef(expectedValue)) + ); + }); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java index 8dcad2f354b26..326756ad0b5f4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java @@ -216,6 +216,7 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { randomMapping(), Map.of("logs-a", IndexMode.STANDARD) ); + // mark remote1 as unavailable IndexResolution indexResolution = IndexResolution.valid(esIndex, Set.of(remote1Alias)); EsqlSession.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); @@ -226,12 +227,10 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias); assertThat(remote1Cluster.getIndexExpression(), equalTo("*")); - assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); - assertThat(remote1Cluster.getTook().millis(), equalTo(0L)); - assertThat(remote1Cluster.getTotalShards(), equalTo(0)); - assertThat(remote1Cluster.getSuccessfulShards(), equalTo(0)); - assertThat(remote1Cluster.getSkippedShards(), equalTo(0)); - assertThat(remote1Cluster.getFailedShards(), equalTo(0)); + // remote1 is left as RUNNING, since another method (updateExecutionInfoWithUnavailableClusters) not under test changes status + assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.RUNNING)); + assertNull(remote1Cluster.getTook()); + assertNull(remote1Cluster.getTotalShards()); EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias); assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*")); diff --git a/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/HuggingFaceServiceMixedIT.java b/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/HuggingFaceServiceMixedIT.java index 59d3faf6489a6..457ae525b7f4b 100644 --- a/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/HuggingFaceServiceMixedIT.java +++ b/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/HuggingFaceServiceMixedIT.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Map; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.not; @@ -27,6 +28,7 @@ public class HuggingFaceServiceMixedIT extends BaseMixedTestCase { private static final String HF_EMBEDDINGS_ADDED = "8.12.0"; private static final String HF_ELSER_ADDED = "8.12.0"; + private static final String HF_EMBEDDINGS_CHUNKING_SETTINGS_ADDED = "8.16.0"; private static final String MINIMUM_SUPPORTED_VERSION = "8.15.0"; private static MockWebServer embeddingsServer; @@ -59,7 +61,24 @@ public void testHFEmbeddings() throws IOException { final String inferenceId = "mixed-cluster-embeddings"; embeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponse())); - put(inferenceId, embeddingConfig(getUrl(embeddingsServer)), TaskType.TEXT_EMBEDDING); + + try { + put(inferenceId, embeddingConfig(getUrl(embeddingsServer)), TaskType.TEXT_EMBEDDING); + } catch (Exception e) { + if (bwcVersion.before(Version.fromString(HF_EMBEDDINGS_CHUNKING_SETTINGS_ADDED))) { + // Chunking settings were added in 8.16.0. if the version is before that, an exception will be thrown if the index mapping + // was created based on a mapping from an old node + assertThat( + e.getMessage(), + containsString( + "One or more nodes in your cluster does not support chunking_settings. " + + "Please update all nodes in your cluster to the latest version to use chunking_settings." + ) + ); + return; + } + } + var configs = (List>) get(TaskType.TEXT_EMBEDDING, inferenceId).get("endpoints"); assertThat(configs, hasSize(1)); assertEquals("hugging_face", configs.get(0).get("service")); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java index 81ebebdb47e4f..3ae8dc0550391 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java @@ -16,10 +16,13 @@ import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; +import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingByteResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingByteResults; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults; import java.util.ArrayList; import java.util.List; @@ -42,7 +45,8 @@ public class EmbeddingRequestChunker { public enum EmbeddingType { FLOAT, - BYTE; + BYTE, + SPARSE; public static EmbeddingType fromDenseVectorElementType(DenseVectorFieldMapper.ElementType elementType) { return switch (elementType) { @@ -67,6 +71,7 @@ public static EmbeddingType fromDenseVectorElementType(DenseVectorFieldMapper.El private List> chunkedInputs; private List>> floatResults; private List>> byteResults; + private List>> sparseResults; private AtomicArray errors; private ActionListener> finalListener; @@ -117,6 +122,7 @@ private void splitIntoBatchedRequests(List inputs) { switch (embeddingType) { case FLOAT -> floatResults = new ArrayList<>(inputs.size()); case BYTE -> byteResults = new ArrayList<>(inputs.size()); + case SPARSE -> sparseResults = new ArrayList<>(inputs.size()); } errors = new AtomicArray<>(inputs.size()); @@ -127,6 +133,7 @@ private void splitIntoBatchedRequests(List inputs) { switch (embeddingType) { case FLOAT -> floatResults.add(new AtomicArray<>(numberOfSubBatches)); case BYTE -> byteResults.add(new AtomicArray<>(numberOfSubBatches)); + case SPARSE -> sparseResults.add(new AtomicArray<>(numberOfSubBatches)); } chunkedInputs.add(chunks); } @@ -217,6 +224,7 @@ public void onResponse(InferenceServiceResults inferenceServiceResults) { switch (embeddingType) { case FLOAT -> handleFloatResults(inferenceServiceResults); case BYTE -> handleByteResults(inferenceServiceResults); + case SPARSE -> handleSparseResults(inferenceServiceResults); } } @@ -266,6 +274,29 @@ private void handleByteResults(InferenceServiceResults inferenceServiceResults) } } + private void handleSparseResults(InferenceServiceResults inferenceServiceResults) { + if (inferenceServiceResults instanceof SparseEmbeddingResults sparseEmbeddings) { + if (failIfNumRequestsDoNotMatch(sparseEmbeddings.embeddings().size())) { + return; + } + + int start = 0; + for (var pos : positions) { + sparseResults.get(pos.inputIndex()) + .setOnce(pos.chunkIndex(), sparseEmbeddings.embeddings().subList(start, start + pos.embeddingCount())); + start += pos.embeddingCount(); + } + + if (resultCount.incrementAndGet() == totalNumberOfRequests) { + sendResponse(); + } + } else { + onFailure( + unexpectedResultTypeException(inferenceServiceResults.getWriteableName(), InferenceTextEmbeddingByteResults.NAME) + ); + } + } + private boolean failIfNumRequestsDoNotMatch(int numberOfResults) { int numberOfRequests = positions.stream().mapToInt(SubBatchPositionsAndCount::embeddingCount).sum(); if (numberOfRequests != numberOfResults) { @@ -319,6 +350,7 @@ private ChunkedInferenceServiceResults mergeResultsWithInputs(int resultIndex) { return switch (embeddingType) { case FLOAT -> mergeFloatResultsWithInputs(chunkedInputs.get(resultIndex), floatResults.get(resultIndex)); case BYTE -> mergeByteResultsWithInputs(chunkedInputs.get(resultIndex), byteResults.get(resultIndex)); + case SPARSE -> mergeSparseResultsWithInputs(chunkedInputs.get(resultIndex), sparseResults.get(resultIndex)); }; } @@ -366,6 +398,26 @@ private InferenceChunkedTextEmbeddingByteResults mergeByteResultsWithInputs( return new InferenceChunkedTextEmbeddingByteResults(embeddingChunks, false); } + private InferenceChunkedSparseEmbeddingResults mergeSparseResultsWithInputs( + List chunks, + AtomicArray> debatchedResults + ) { + var all = new ArrayList(); + for (int i = 0; i < debatchedResults.length(); i++) { + var subBatch = debatchedResults.get(i); + all.addAll(subBatch); + } + + assert chunks.size() == all.size(); + + var embeddingChunks = new ArrayList(); + for (int i = 0; i < chunks.size(); i++) { + embeddingChunks.add(new MlChunkedTextExpansionResults.ChunkedResult(chunks.get(i), all.get(i).tokens())); + } + + return new InferenceChunkedSparseEmbeddingResults(embeddingChunks); + } + public record BatchRequest(List subBatches) { public int size() { return subBatches.stream().mapToInt(SubBatch::size).sum(); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/DelegatingProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/DelegatingProcessor.java index 9af5668ecf75b..fc2d890dd89e6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/DelegatingProcessor.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/DelegatingProcessor.java @@ -21,7 +21,7 @@ public abstract class DelegatingProcessor implements Flow.Processor { private static final Logger log = LogManager.getLogger(DelegatingProcessor.class); private final AtomicLong pendingRequests = new AtomicLong(); - private final AtomicBoolean isClosed = new AtomicBoolean(false); + protected final AtomicBoolean isClosed = new AtomicBoolean(false); private Flow.Subscriber downstream; private Flow.Subscription upstream; @@ -49,7 +49,7 @@ private Flow.Subscription forwardingSubscription() { @Override public void request(long n) { if (isClosed.get()) { - downstream.onComplete(); // shouldn't happen, but reinforce that we're no longer listening + downstream.onComplete(); } else if (upstream != null) { upstream.request(n); } else { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandler.java index b5af0b474834f..3579cd4100bbb 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandler.java @@ -8,14 +8,19 @@ package org.elasticsearch.xpack.inference.external.cohere; import org.apache.logging.log4j.Logger; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler; import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser; import org.elasticsearch.xpack.inference.external.http.retry.RetryException; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.external.response.cohere.CohereErrorResponseEntity; +import org.elasticsearch.xpack.inference.external.response.streaming.NewlineDelimitedByteProcessor; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import java.util.concurrent.Flow; + import static org.elasticsearch.xpack.inference.external.http.HttpUtils.checkForEmptyBody; /** @@ -33,9 +38,11 @@ public class CohereResponseHandler extends BaseResponseHandler { static final String TEXTS_ARRAY_TOO_LARGE_MESSAGE_MATCHER = "invalid request: total number of texts must be at most"; static final String TEXTS_ARRAY_ERROR_MESSAGE = "Received a texts array too large response"; + private final boolean canHandleStreamingResponse; - public CohereResponseHandler(String requestType, ResponseParser parseFunction) { + public CohereResponseHandler(String requestType, ResponseParser parseFunction, boolean canHandleStreamingResponse) { super(requestType, parseFunction, CohereErrorResponseEntity::fromResponse); + this.canHandleStreamingResponse = canHandleStreamingResponse; } @Override @@ -45,6 +52,20 @@ public void validateResponse(ThrottlerManager throttlerManager, Logger logger, R checkForEmptyBody(throttlerManager, logger, request, result); } + @Override + public boolean canHandleStreamingResponses() { + return canHandleStreamingResponse; + } + + @Override + public InferenceServiceResults parseResult(Request request, Flow.Publisher flow) { + var ndProcessor = new NewlineDelimitedByteProcessor(); + var cohereProcessor = new CohereStreamingProcessor(); + flow.subscribe(ndProcessor); + ndProcessor.subscribe(cohereProcessor); + return new StreamingChatCompletionResults(cohereProcessor); + } + /** * Validates the status code throws an RetryException if not in the range [200, 300). * diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereStreamingProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereStreamingProcessor.java new file mode 100644 index 0000000000000..2516a647a91fd --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereStreamingProcessor.java @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.cohere; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; +import org.elasticsearch.xpack.inference.common.DelegatingProcessor; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Map; +import java.util.Optional; + +class CohereStreamingProcessor extends DelegatingProcessor, StreamingChatCompletionResults.Results> { + private static final Logger log = LogManager.getLogger(CohereStreamingProcessor.class); + + @Override + protected void next(Deque item) throws Exception { + if (item.isEmpty()) { + // discard empty result and go to the next + upstream().request(1); + return; + } + + var results = new ArrayDeque(item.size()); + for (String json : item) { + try (var jsonParser = jsonParser(json)) { + var responseMap = jsonParser.map(); + var eventType = (String) responseMap.get("event_type"); + switch (eventType) { + case "text-generation" -> parseText(responseMap).ifPresent(results::offer); + case "stream-end" -> validateResponse(responseMap); + case "stream-start", "search-queries-generation", "search-results", "citation-generation", "tool-calls-generation", + "tool-calls-chunk" -> { + log.debug("Skipping event type [{}] for line [{}].", eventType, item); + } + default -> throw new IOException("Unknown eventType found: " + eventType); + } + } catch (ElasticsearchStatusException e) { + throw e; + } catch (Exception e) { + log.warn("Failed to parse json from cohere: {}", json); + throw e; + } + } + + if (results.isEmpty()) { + upstream().request(1); + } else { + downstream().onNext(new StreamingChatCompletionResults.Results(results)); + } + } + + private static XContentParser jsonParser(String line) throws IOException { + return XContentFactory.xContent(XContentType.JSON).createParser(XContentParserConfiguration.EMPTY, line); + } + + private Optional parseText(Map responseMap) throws IOException { + var text = (String) responseMap.get("text"); + if (text != null) { + return Optional.of(new StreamingChatCompletionResults.Result(text)); + } else { + throw new IOException("Null text found in text-generation cohere event"); + } + } + + private void validateResponse(Map responseMap) { + var finishReason = (String) responseMap.get("finish_reason"); + switch (finishReason) { + case "ERROR", "ERROR_TOXIC" -> throw new ElasticsearchStatusException( + "Cohere stopped the stream due to an error: {}", + RestStatus.INTERNAL_SERVER_ERROR, + parseErrorMessage(responseMap) + ); + case "ERROR_LIMIT" -> throw new ElasticsearchStatusException( + "Cohere stopped the stream due to an error: {}", + RestStatus.TOO_MANY_REQUESTS, + parseErrorMessage(responseMap) + ); + } + } + + @SuppressWarnings("unchecked") + private String parseErrorMessage(Map responseMap) { + var innerResponseMap = (Map) responseMap.get("response"); + return (String) innerResponseMap.get("text"); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereCompletionRequestManager.java index 423093a14a9f0..ae46fbe0fef87 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereCompletionRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereCompletionRequestManager.java @@ -19,7 +19,6 @@ import org.elasticsearch.xpack.inference.external.response.cohere.CohereCompletionResponseEntity; import org.elasticsearch.xpack.inference.services.cohere.completion.CohereCompletionModel; -import java.util.List; import java.util.Objects; import java.util.function.Supplier; @@ -30,7 +29,7 @@ public class CohereCompletionRequestManager extends CohereRequestManager { private static final ResponseHandler HANDLER = createCompletionHandler(); private static ResponseHandler createCompletionHandler() { - return new CohereResponseHandler("cohere completion", CohereCompletionResponseEntity::fromResponse); + return new CohereResponseHandler("cohere completion", CohereCompletionResponseEntity::fromResponse, true); } public static CohereCompletionRequestManager of(CohereCompletionModel model, ThreadPool threadPool) { @@ -51,8 +50,10 @@ public void execute( Supplier hasRequestCompletedFunction, ActionListener listener ) { - List docsInput = DocumentsOnlyInput.of(inferenceInputs).getInputs(); - CohereCompletionRequest request = new CohereCompletionRequest(docsInput, model); + var docsOnly = DocumentsOnlyInput.of(inferenceInputs); + var docsInput = docsOnly.getInputs(); + var stream = docsOnly.stream(); + CohereCompletionRequest request = new CohereCompletionRequest(docsInput, model, stream); execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener)); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereEmbeddingsRequestManager.java index 402f91a0838dc..80617ea56e63c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereEmbeddingsRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereEmbeddingsRequestManager.java @@ -28,7 +28,7 @@ public class CohereEmbeddingsRequestManager extends CohereRequestManager { private static final ResponseHandler HANDLER = createEmbeddingsHandler(); private static ResponseHandler createEmbeddingsHandler() { - return new CohereResponseHandler("cohere text embedding", CohereEmbeddingsResponseEntity::fromResponse); + return new CohereResponseHandler("cohere text embedding", CohereEmbeddingsResponseEntity::fromResponse, false); } public static CohereEmbeddingsRequestManager of(CohereEmbeddingsModel model, ThreadPool threadPool) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRerankRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRerankRequestManager.java index 9d565e7124b03..d27812b17399b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRerankRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRerankRequestManager.java @@ -27,7 +27,7 @@ public class CohereRerankRequestManager extends CohereRequestManager { private static final ResponseHandler HANDLER = createCohereResponseHandler(); private static ResponseHandler createCohereResponseHandler() { - return new CohereResponseHandler("cohere rerank", (request, response) -> CohereRankedResponseEntity.fromResponse(response)); + return new CohereResponseHandler("cohere rerank", (request, response) -> CohereRankedResponseEntity.fromResponse(response), false); } public static CohereRerankRequestManager of(CohereRerankModel model, ThreadPool threadPool) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/completion/CohereCompletionRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/completion/CohereCompletionRequest.java index f68f919a7d85b..2172dcd4d791f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/completion/CohereCompletionRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/completion/CohereCompletionRequest.java @@ -25,22 +25,20 @@ import java.util.Objects; public class CohereCompletionRequest extends CohereRequest { - private final CohereAccount account; - private final List input; - private final String modelId; - private final String inferenceEntityId; + private final boolean stream; - public CohereCompletionRequest(List input, CohereCompletionModel model) { + public CohereCompletionRequest(List input, CohereCompletionModel model, boolean stream) { Objects.requireNonNull(model); this.account = CohereAccount.of(model, CohereCompletionRequest::buildDefaultUri); this.input = Objects.requireNonNull(input); this.modelId = model.getServiceSettings().modelId(); this.inferenceEntityId = model.getInferenceEntityId(); + this.stream = stream; } @Override @@ -48,7 +46,7 @@ public HttpRequest createHttpRequest() { HttpPost httpPost = new HttpPost(account.uri()); ByteArrayEntity byteEntity = new ByteArrayEntity( - Strings.toString(new CohereCompletionRequestEntity(input, modelId)).getBytes(StandardCharsets.UTF_8) + Strings.toString(new CohereCompletionRequestEntity(input, modelId, isStreaming())).getBytes(StandardCharsets.UTF_8) ); httpPost.setEntity(byteEntity); @@ -62,6 +60,11 @@ public String getInferenceEntityId() { return inferenceEntityId; } + @Override + public boolean isStreaming() { + return stream; + } + @Override public URI getURI() { return account.uri(); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/completion/CohereCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/completion/CohereCompletionRequestEntity.java index 8cb3dc6e3c8e8..b834e4335d73c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/completion/CohereCompletionRequestEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/completion/CohereCompletionRequestEntity.java @@ -15,11 +15,11 @@ import java.util.List; import java.util.Objects; -public record CohereCompletionRequestEntity(List input, @Nullable String model) implements ToXContentObject { +public record CohereCompletionRequestEntity(List input, @Nullable String model, boolean stream) implements ToXContentObject { private static final String MESSAGE_FIELD = "message"; - private static final String MODEL = "model"; + private static final String STREAM = "stream"; public CohereCompletionRequestEntity { Objects.requireNonNull(input); @@ -36,6 +36,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(MODEL, model); } + if (stream) { + builder.field(STREAM, true); + } + builder.endObject(); return builder; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/streaming/NewlineDelimitedByteProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/streaming/NewlineDelimitedByteProcessor.java new file mode 100644 index 0000000000000..7c44b202a816b --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/streaming/NewlineDelimitedByteProcessor.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.response.streaming; + +import org.elasticsearch.xpack.inference.common.DelegatingProcessor; +import org.elasticsearch.xpack.inference.external.http.HttpResult; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.regex.Pattern; + +/** + * Processes HttpResult bytes into lines separated by newlines, delimited by either line-feed or carriage-return line-feed. + * Downstream is responsible for validating the structure of the lines after they have been separated. + * Because Upstream (Apache) can send us a single line split between two HttpResults, this processor will aggregate bytes from the last + * HttpResult and append them to the front of the next HttpResult. + * When onComplete is called, the last batch is always flushed to the downstream onNext. + */ +public class NewlineDelimitedByteProcessor extends DelegatingProcessor> { + private static final Pattern END_OF_LINE_REGEX = Pattern.compile("\\n|\\r\\n"); + private volatile String previousTokens = ""; + + @Override + protected void next(HttpResult item) { + // discard empty result and go to the next + if (item.isBodyEmpty()) { + upstream().request(1); + return; + } + + var body = previousTokens + new String(item.body(), StandardCharsets.UTF_8); + var lines = END_OF_LINE_REGEX.split(body, -1); // -1 because we actually want trailing empty strings + + var results = new ArrayDeque(lines.length); + for (var i = 0; i < lines.length - 1; i++) { + var line = lines[i].trim(); + if (line.isBlank() == false) { + results.offer(line); + } + } + + previousTokens = lines[lines.length - 1].trim(); + + if (results.isEmpty()) { + upstream().request(1); + } else { + downstream().onNext(results); + } + } + + @Override + public void onComplete() { + if (previousTokens.isBlank()) { + super.onComplete(); + } else if (isClosed.compareAndSet(false, true)) { + var results = new ArrayDeque(1); + results.offer(previousTokens); + downstream().onNext(results); + } + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java index 00d44b8e7a0e2..a33a49cc52d94 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java @@ -50,6 +50,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; public class SemanticQueryBuilder extends AbstractQueryBuilder { + // **** THE semantic_text.inner_hits CLUSTER FEATURE IS DEFUNCT, NEVER USE IT **** public static final NodeFeature SEMANTIC_TEXT_INNER_HITS = new NodeFeature("semantic_text.inner_hits"); public static final String NAME = "semantic"; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java index eb36c445506a7..134f8af0e083d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java @@ -103,10 +103,7 @@ public int rankWindowSize() { @Override protected void doToXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(RETRIEVER_FIELD.getPreferredName()); - builder.startObject(); - builder.field(retrieverBuilder.getName(), retrieverBuilder); - builder.endObject(); + builder.field(RETRIEVER_FIELD.getPreferredName(), retrieverBuilder); builder.field(FIELD_FIELD.getPreferredName(), field); builder.field(RANK_WINDOW_SIZE_FIELD.getPreferredName(), rankWindowSize); if (seed != null) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java index ab013e0275a69..50d762e7b90aa 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java @@ -179,17 +179,11 @@ public int rankWindowSize() { @Override protected void doToXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(RETRIEVER_FIELD.getPreferredName()); - builder.startObject(); - builder.field(retrieverBuilder.getName(), retrieverBuilder); - builder.endObject(); + builder.field(RETRIEVER_FIELD.getPreferredName(), retrieverBuilder); builder.field(INFERENCE_ID_FIELD.getPreferredName(), inferenceId); builder.field(INFERENCE_TEXT_FIELD.getPreferredName(), inferenceText); builder.field(FIELD_FIELD.getPreferredName(), field); builder.field(RANK_WINDOW_SIZE_FIELD.getPreferredName(), rankWindowSize); - if (minScore != null) { - builder.field(MIN_SCORE_FIELD.getPreferredName(), minScore); - } } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java index 728a4ac137dff..3ba93dd8d1b66 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java @@ -39,6 +39,7 @@ import java.util.List; import java.util.Map; +import java.util.Set; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; @@ -288,4 +289,9 @@ static SimilarityMeasure defaultSimilarity() { public TransportVersion getMinimalSupportedVersion() { return TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED; } + + @Override + public Set supportedStreamingTasks() { + return COMPLETION_ONLY; + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java index 0dd41db2f016c..881e2e82b766a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java @@ -248,15 +248,14 @@ public static InferModelAction.Request buildInferenceRequest( InferenceConfigUpdate update, List inputs, InputType inputType, - TimeValue timeout, - boolean chunk + TimeValue timeout ) { var request = InferModelAction.Request.forTextInput(id, update, inputs, true, timeout); request.setPrefixType( InputType.SEARCH == inputType ? TrainedModelPrefixStrings.PrefixType.SEARCH : TrainedModelPrefixStrings.PrefixType.INGEST ); request.setHighPriority(InputType.SEARCH == inputType); - request.setChunked(chunk); + request.setChunked(false); return request; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java index 07d0cc14b2ac8..a593e1dfb6d9d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java @@ -58,6 +58,11 @@ public abstract ActionListener getC ActionListener listener ); + @Override + public ElasticsearchInternalServiceSettings getServiceSettings() { + return (ElasticsearchInternalServiceSettings) super.getServiceSettings(); + } + @Override public String toString() { return Strings.toString(this.getConfigurations()); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java index dd14e16412996..739f514bee1c9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java @@ -28,21 +28,19 @@ import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; -import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; -import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.RankedDocsResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction; import org.elasticsearch.xpack.core.ml.action.InferModelAction; import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; -import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextEmbeddingFloatResults; -import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults; +import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; +import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.EmptyConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextExpansionConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextSimilarityConfigUpdate; -import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate; +import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.ServiceUtils; @@ -74,6 +72,7 @@ public class ElasticsearchInternalService extends BaseElasticsearchInternalServi MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86 ); + public static final int EMBEDDING_MAX_BATCH_SIZE = 10; public static final String DEFAULT_ELSER_ID = ".elser-2"; private static final Logger logger = LogManager.getLogger(ElasticsearchInternalService.class); @@ -501,8 +500,7 @@ public void inferTextEmbedding( TextEmbeddingConfigUpdate.EMPTY_INSTANCE, inputs, inputType, - timeout, - false + timeout ); ActionListener mlResultsListener = listener.delegateFailureAndWrap( @@ -528,8 +526,7 @@ public void inferSparseEmbedding( TextExpansionConfigUpdate.EMPTY_UPDATE, inputs, inputType, - timeout, - false + timeout ); ActionListener mlResultsListener = listener.delegateFailureAndWrap( @@ -557,8 +554,7 @@ public void inferRerank( new TextSimilarityConfigUpdate(query), inputs, inputType, - timeout, - false + timeout ); var modelSettings = (CustomElandRerankTaskSettings) model.getTaskSettings(); @@ -608,47 +604,82 @@ public void chunkedInfer( return; } - var configUpdate = chunkingOptions != null - ? new TokenizationConfigUpdate(chunkingOptions.windowSize(), chunkingOptions.span()) - : new TokenizationConfigUpdate(null, null); + if (model instanceof ElasticsearchInternalModel esModel) { - var request = buildInferenceRequest( - model.getConfigurations().getInferenceEntityId(), - configUpdate, - input, - inputType, - timeout, - true - ); + var batchedRequests = new EmbeddingRequestChunker( + input, + EMBEDDING_MAX_BATCH_SIZE, + embeddingTypeFromTaskTypeAndSettings(model.getTaskType(), esModel.internalServiceSettings) + ).batchRequestsWithListeners(listener); + + for (var batch : batchedRequests) { + var inferenceRequest = buildInferenceRequest( + model.getConfigurations().getInferenceEntityId(), + EmptyConfigUpdate.INSTANCE, + batch.batch().inputs(), + inputType, + timeout + ); - client.execute( - InferModelAction.INSTANCE, - request, - listener.delegateFailureAndWrap( - (l, inferenceResult) -> l.onResponse(translateToChunkedResults(inferenceResult.getInferenceResults())) - ) - ); - } + ActionListener mlResultsListener = batch.listener() + .delegateFailureAndWrap( + (l, inferenceResult) -> translateToChunkedResult(model.getTaskType(), inferenceResult.getInferenceResults(), l) + ); - private static List translateToChunkedResults(List inferenceResults) { - var translated = new ArrayList(); + var maybeDeployListener = mlResultsListener.delegateResponse( + (l, exception) -> maybeStartDeployment(esModel, exception, inferenceRequest, mlResultsListener) + ); - for (var inferenceResult : inferenceResults) { - translated.add(translateToChunkedResult(inferenceResult)); + client.execute(InferModelAction.INSTANCE, inferenceRequest, maybeDeployListener); + } + } else { + listener.onFailure(notElasticsearchModelException(model)); } - - return translated; } - private static ChunkedInferenceServiceResults translateToChunkedResult(InferenceResults inferenceResult) { - if (inferenceResult instanceof MlChunkedTextEmbeddingFloatResults mlChunkedResult) { - return InferenceChunkedTextEmbeddingFloatResults.ofMlResults(mlChunkedResult); - } else if (inferenceResult instanceof MlChunkedTextExpansionResults mlChunkedResult) { - return InferenceChunkedSparseEmbeddingResults.ofMlResult(mlChunkedResult); - } else if (inferenceResult instanceof ErrorInferenceResults error) { - return new ErrorChunkedInferenceResults(error.getException()); - } else { - throw createInvalidChunkedResultException(MlChunkedTextEmbeddingFloatResults.NAME, inferenceResult.getWriteableName()); + private static void translateToChunkedResult( + TaskType taskType, + List inferenceResults, + ActionListener chunkPartListener + ) { + if (taskType == TaskType.TEXT_EMBEDDING) { + var translated = new ArrayList(); + + for (var inferenceResult : inferenceResults) { + if (inferenceResult instanceof MlTextEmbeddingResults mlTextEmbeddingResult) { + translated.add( + new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(mlTextEmbeddingResult.getInferenceAsFloat()) + ); + } else if (inferenceResult instanceof ErrorInferenceResults error) { + chunkPartListener.onFailure(error.getException()); + return; + } else { + chunkPartListener.onFailure( + createInvalidChunkedResultException(MlTextEmbeddingResults.NAME, inferenceResult.getWriteableName()) + ); + return; + } + } + chunkPartListener.onResponse(new InferenceTextEmbeddingFloatResults(translated)); + } else { // sparse + var translated = new ArrayList(); + + for (var inferenceResult : inferenceResults) { + if (inferenceResult instanceof TextExpansionResults textExpansionResult) { + translated.add( + new SparseEmbeddingResults.Embedding(textExpansionResult.getWeightedTokens(), textExpansionResult.isTruncated()) + ); + } else if (inferenceResult instanceof ErrorInferenceResults error) { + chunkPartListener.onFailure(error.getException()); + return; + } else { + chunkPartListener.onFailure( + createInvalidChunkedResultException(TextExpansionResults.NAME, inferenceResult.getWriteableName()) + ); + return; + } + } + chunkPartListener.onResponse(new SparseEmbeddingResults(translated)); } } @@ -731,4 +762,21 @@ public List defaultConfigs() { protected boolean isDefaultId(String inferenceId) { return DEFAULT_ELSER_ID.equals(inferenceId); } + + static EmbeddingRequestChunker.EmbeddingType embeddingTypeFromTaskTypeAndSettings( + TaskType taskType, + ElasticsearchInternalServiceSettings serviceSettings + ) { + return switch (taskType) { + case SPARSE_EMBEDDING -> EmbeddingRequestChunker.EmbeddingType.SPARSE; + case TEXT_EMBEDDING -> serviceSettings.elementType() == null + ? EmbeddingRequestChunker.EmbeddingType.FLOAT + : EmbeddingRequestChunker.EmbeddingType.fromDenseVectorElementType(serviceSettings.elementType()); + default -> throw new ElasticsearchStatusException( + "Chunking is not supported for task type [{}]", + RestStatus.BAD_REQUEST, + taskType + ); + }; + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java index cf862ee6fb4b8..c1be537a6b0a7 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java @@ -11,10 +11,13 @@ import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; +import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingByteResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingByteResults; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.ml.search.WeightedToken; import java.util.ArrayList; import java.util.List; @@ -357,6 +360,83 @@ public void testMergingListener_Byte() { } } + public void testMergingListener_Sparse() { + int batchSize = 4; + int chunkSize = 10; + int overlap = 0; + // passage will be chunked into 2.1 batches + // and spread over 3 batch requests + int numberOfWordsInPassage = (chunkSize * batchSize * 2) + 5; + + var passageBuilder = new StringBuilder(); + for (int i = 0; i < numberOfWordsInPassage; i++) { + passageBuilder.append("passage_input").append(i).append(" "); // chunk on whitespace + } + List inputs = List.of("1st small", "2nd small", "3rd small", passageBuilder.toString()); + + var finalListener = testListener(); + var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap, EmbeddingRequestChunker.EmbeddingType.SPARSE) + .batchRequestsWithListeners(finalListener); + assertThat(batches, hasSize(3)); + + // 4 inputs in 3 batches + { + var embeddings = new ArrayList(); + for (int i = 0; i < batchSize; i++) { + embeddings.add(new SparseEmbeddingResults.Embedding(List.of(new WeightedToken(randomAlphaOfLength(4), 1.0f)), false)); + } + batches.get(0).listener().onResponse(new SparseEmbeddingResults(embeddings)); + } + { + var embeddings = new ArrayList(); + for (int i = 0; i < batchSize; i++) { + embeddings.add(new SparseEmbeddingResults.Embedding(List.of(new WeightedToken(randomAlphaOfLength(4), 1.0f)), false)); + } + batches.get(1).listener().onResponse(new SparseEmbeddingResults(embeddings)); + } + { + var embeddings = new ArrayList(); + for (int i = 0; i < 4; i++) { // 4 chunks in the final batch + embeddings.add(new SparseEmbeddingResults.Embedding(List.of(new WeightedToken(randomAlphaOfLength(4), 1.0f)), false)); + } + batches.get(2).listener().onResponse(new SparseEmbeddingResults(embeddings)); + } + + assertNotNull(finalListener.results); + assertThat(finalListener.results, hasSize(4)); + { + var chunkedResult = finalListener.results.get(0); + assertThat(chunkedResult, instanceOf(InferenceChunkedSparseEmbeddingResults.class)); + var chunkedSparseResult = (InferenceChunkedSparseEmbeddingResults) chunkedResult; + assertThat(chunkedSparseResult.getChunkedResults(), hasSize(1)); + assertEquals("1st small", chunkedSparseResult.getChunkedResults().get(0).matchedText()); + } + { + var chunkedResult = finalListener.results.get(1); + assertThat(chunkedResult, instanceOf(InferenceChunkedSparseEmbeddingResults.class)); + var chunkedSparseResult = (InferenceChunkedSparseEmbeddingResults) chunkedResult; + assertThat(chunkedSparseResult.getChunkedResults(), hasSize(1)); + assertEquals("2nd small", chunkedSparseResult.getChunkedResults().get(0).matchedText()); + } + { + var chunkedResult = finalListener.results.get(2); + assertThat(chunkedResult, instanceOf(InferenceChunkedSparseEmbeddingResults.class)); + var chunkedSparseResult = (InferenceChunkedSparseEmbeddingResults) chunkedResult; + assertThat(chunkedSparseResult.getChunkedResults(), hasSize(1)); + assertEquals("3rd small", chunkedSparseResult.getChunkedResults().get(0).matchedText()); + } + { + // this is the large input split in multiple chunks + var chunkedResult = finalListener.results.get(3); + assertThat(chunkedResult, instanceOf(InferenceChunkedSparseEmbeddingResults.class)); + var chunkedSparseResult = (InferenceChunkedSparseEmbeddingResults) chunkedResult; + assertThat(chunkedSparseResult.getChunkedResults(), hasSize(9)); // passage is split into 9 chunks, 10 words each + assertThat(chunkedSparseResult.getChunkedResults().get(0).matchedText(), startsWith("passage_input0 ")); + assertThat(chunkedSparseResult.getChunkedResults().get(1).matchedText(), startsWith(" passage_input10 ")); + assertThat(chunkedSparseResult.getChunkedResults().get(8).matchedText(), startsWith(" passage_input80 ")); + } + } + public void testListenerErrorsWithWrongNumberOfResponses() { List inputs = List.of("1st small", "2nd small", "3rd small"); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandlerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandlerTests.java index d64ac495c8c99..444415dfc8e48 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandlerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandlerTests.java @@ -132,7 +132,7 @@ private static void callCheckForFailureStatusCode(int statusCode, @Nullable Stri var mockRequest = mock(Request.class); when(mockRequest.getInferenceEntityId()).thenReturn(modelId); var httpResult = new HttpResult(httpResponse, errorMessage == null ? new byte[] {} : responseJson.getBytes(StandardCharsets.UTF_8)); - var handler = new CohereResponseHandler("", (request, result) -> null); + var handler = new CohereResponseHandler("", (request, result) -> null, false); handler.checkForFailureStatusCode(mockRequest, httpResult); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/cohere/CohereStreamingProcessorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/cohere/CohereStreamingProcessorTests.java new file mode 100644 index 0000000000000..87d6d63bb8c51 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/cohere/CohereStreamingProcessorTests.java @@ -0,0 +1,189 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.cohere; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.common.xcontent.ChunkedToXContent; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentParseException; +import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.concurrent.Flow; +import java.util.function.Consumer; + +import static org.elasticsearch.xpack.inference.common.DelegatingProcessorTests.onError; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.isA; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.assertArg; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class CohereStreamingProcessorTests extends ESTestCase { + + public void testParseErrorCallsOnError() { + var item = new ArrayDeque(); + item.offer("this is not json"); + + var exception = onError(new CohereStreamingProcessor(), item); + assertThat(exception, instanceOf(XContentParseException.class)); + } + + public void testUnrecognizedEventCallsOnError() { + var item = new ArrayDeque(); + item.offer("{\"event_type\":\"test\"}"); + + var exception = onError(new CohereStreamingProcessor(), item); + assertThat(exception, instanceOf(IOException.class)); + assertThat(exception.getMessage(), equalTo("Unknown eventType found: test")); + } + + public void testMissingTextCallsOnError() { + var item = new ArrayDeque(); + item.offer("{\"event_type\":\"text-generation\"}"); + + var exception = onError(new CohereStreamingProcessor(), item); + assertThat(exception, instanceOf(IOException.class)); + assertThat(exception.getMessage(), equalTo("Null text found in text-generation cohere event")); + } + + public void testEmptyResultsRequestsMoreData() throws Exception { + var emptyDeque = new ArrayDeque(); + + var processor = new CohereStreamingProcessor(); + + Flow.Subscriber downstream = mock(); + processor.subscribe(downstream); + + Flow.Subscription upstream = mock(); + processor.onSubscribe(upstream); + + processor.next(emptyDeque); + + verify(upstream, times(1)).request(1); + verify(downstream, times(0)).onNext(any()); + } + + public void testNonDataEventsAreSkipped() throws Exception { + var item = new ArrayDeque(); + item.offer("{\"event_type\":\"stream-start\"}"); + item.offer("{\"event_type\":\"search-queries-generation\"}"); + item.offer("{\"event_type\":\"search-results\"}"); + item.offer("{\"event_type\":\"citation-generation\"}"); + item.offer("{\"event_type\":\"tool-calls-generation\"}"); + item.offer("{\"event_type\":\"tool-calls-chunk\"}"); + + var processor = new CohereStreamingProcessor(); + + Flow.Subscriber downstream = mock(); + processor.subscribe(downstream); + + Flow.Subscription upstream = mock(); + processor.onSubscribe(upstream); + + processor.next(item); + + verify(upstream, times(1)).request(1); + verify(downstream, times(0)).onNext(any()); + } + + public void testParseError() { + var json = "{\"event_type\":\"stream-end\", \"finish_reason\":\"ERROR\", \"response\":{ \"text\": \"a wild error appears\" }}"; + testError(json, e -> { + assertThat(e.status().getStatus(), equalTo(500)); + assertThat(e.getMessage(), containsString("a wild error appears")); + }); + } + + private void testError(String json, Consumer test) { + var item = new ArrayDeque(); + item.offer(json); + + var processor = new CohereStreamingProcessor(); + + Flow.Subscriber downstream = mock(); + processor.subscribe(downstream); + + Flow.Subscription upstream = mock(); + processor.onSubscribe(upstream); + + try { + processor.next(item); + fail("Expected an exception to be thrown"); + } catch (ElasticsearchStatusException e) { + test.accept(e); + } catch (Exception e) { + fail(e, "Expected an exception of type ElasticsearchStatusException to be thrown"); + } + } + + public void testParseToxic() { + var json = "{\"event_type\":\"stream-end\", \"finish_reason\":\"ERROR_TOXIC\", \"response\":{ \"text\": \"by britney spears\" }}"; + testError(json, e -> { + assertThat(e.status().getStatus(), equalTo(500)); + assertThat(e.getMessage(), containsString("by britney spears")); + }); + } + + public void testParseLimit() { + var json = "{\"event_type\":\"stream-end\", \"finish_reason\":\"ERROR_LIMIT\", \"response\":{ \"text\": \"over the limit\" }}"; + testError(json, e -> { + assertThat(e.status().getStatus(), equalTo(429)); + assertThat(e.getMessage(), containsString("over the limit")); + }); + } + + public void testNonErrorFinishesAreSkipped() throws Exception { + var item = new ArrayDeque(); + item.offer("{\"event_type\":\"stream-end\", \"finish_reason\":\"COMPLETE\"}"); + item.offer("{\"event_type\":\"stream-end\", \"finish_reason\":\"STOP_SEQUENCE\"}"); + item.offer("{\"event_type\":\"stream-end\", \"finish_reason\":\"USER_CANCEL\"}"); + item.offer("{\"event_type\":\"stream-end\", \"finish_reason\":\"MAX_TOKENS\"}"); + + var processor = new CohereStreamingProcessor(); + + Flow.Subscriber downstream = mock(); + processor.subscribe(downstream); + + Flow.Subscription upstream = mock(); + processor.onSubscribe(upstream); + + processor.next(item); + + verify(upstream, times(1)).request(1); + verify(downstream, times(0)).onNext(any()); + } + + public void testParseCohereData() throws Exception { + var item = new ArrayDeque(); + item.offer("{\"event_type\":\"text-generation\", \"text\":\"hello there\"}"); + + var processor = new CohereStreamingProcessor(); + + Flow.Subscriber downstream = mock(); + processor.subscribe(downstream); + + Flow.Subscription upstream = mock(); + processor.onSubscribe(upstream); + + processor.next(item); + + verify(upstream, times(0)).request(1); + verify(downstream, times(1)).onNext(assertArg(chunks -> { + assertThat(chunks, isA(StreamingChatCompletionResults.Results.class)); + var results = (StreamingChatCompletionResults.Results) chunks; + assertThat(results.results().size(), equalTo(1)); + assertThat(results.results().getFirst().delta(), equalTo("hello there")); + })); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereCompletionRequestEntityTests.java index dbe6a9438d884..c3b534f42e7ee 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereCompletionRequestEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereCompletionRequestEntityTests.java @@ -22,7 +22,7 @@ public class CohereCompletionRequestEntityTests extends ESTestCase { public void testXContent_WritesAllFields() throws IOException { - var entity = new CohereCompletionRequestEntity(List.of("some input"), "model"); + var entity = new CohereCompletionRequestEntity(List.of("some input"), "model", false); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); entity.toXContent(builder, null); @@ -33,7 +33,7 @@ public void testXContent_WritesAllFields() throws IOException { } public void testXContent_DoesNotWriteModelIfNotSpecified() throws IOException { - var entity = new CohereCompletionRequestEntity(List.of("some input"), null); + var entity = new CohereCompletionRequestEntity(List.of("some input"), null, false); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); entity.toXContent(builder, null); @@ -44,10 +44,10 @@ public void testXContent_DoesNotWriteModelIfNotSpecified() throws IOException { } public void testXContent_ThrowsIfInputIsNull() { - expectThrows(NullPointerException.class, () -> new CohereCompletionRequestEntity(null, null)); + expectThrows(NullPointerException.class, () -> new CohereCompletionRequestEntity(null, null, false)); } public void testXContent_ThrowsIfMessageInInputIsNull() { - expectThrows(NullPointerException.class, () -> new CohereCompletionRequestEntity(List.of((String) null), null)); + expectThrows(NullPointerException.class, () -> new CohereCompletionRequestEntity(List.of((String) null), null, false)); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereCompletionRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereCompletionRequestTests.java index d6d0d5c00eaf4..f2e6d4305f9e6 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereCompletionRequestTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereCompletionRequestTests.java @@ -26,7 +26,7 @@ public class CohereCompletionRequestTests extends ESTestCase { public void testCreateRequest_UrlDefined() throws IOException { - var request = new CohereCompletionRequest(List.of("abc"), CohereCompletionModelTests.createModel("url", "secret", null)); + var request = new CohereCompletionRequest(List.of("abc"), CohereCompletionModelTests.createModel("url", "secret", null), false); var httpRequest = request.createHttpRequest(); assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); @@ -43,7 +43,7 @@ public void testCreateRequest_UrlDefined() throws IOException { } public void testCreateRequest_ModelDefined() throws IOException { - var request = new CohereCompletionRequest(List.of("abc"), CohereCompletionModelTests.createModel("url", "secret", "model")); + var request = new CohereCompletionRequest(List.of("abc"), CohereCompletionModelTests.createModel("url", "secret", "model"), false); var httpRequest = request.createHttpRequest(); assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); @@ -60,14 +60,14 @@ public void testCreateRequest_ModelDefined() throws IOException { } public void testTruncate_ReturnsSameInstance() { - var request = new CohereCompletionRequest(List.of("abc"), CohereCompletionModelTests.createModel("url", "secret", "model")); + var request = new CohereCompletionRequest(List.of("abc"), CohereCompletionModelTests.createModel("url", "secret", "model"), false); var truncatedRequest = request.truncate(); assertThat(truncatedRequest, sameInstance(request)); } public void testTruncationInfo_ReturnsNull() { - var request = new CohereCompletionRequest(List.of("abc"), CohereCompletionModelTests.createModel("url", "secret", "model")); + var request = new CohereCompletionRequest(List.of("abc"), CohereCompletionModelTests.createModel("url", "secret", "model"), false); assertNull(request.getTruncationInfo()); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/streaming/NewlineDelimitedByteProcessorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/streaming/NewlineDelimitedByteProcessorTests.java new file mode 100644 index 0000000000000..488cbccd0e7c3 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/streaming/NewlineDelimitedByteProcessorTests.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.response.streaming; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.junit.Before; +import org.mockito.ArgumentCaptor; + +import java.nio.charset.StandardCharsets; +import java.util.Deque; +import java.util.concurrent.Flow; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.assertArg; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class NewlineDelimitedByteProcessorTests extends ESTestCase { + private Flow.Subscription upstream; + private Flow.Subscriber> downstream; + private NewlineDelimitedByteProcessor processor; + + @Before + public void setUp() throws Exception { + super.setUp(); + upstream = mock(); + downstream = mock(); + processor = new NewlineDelimitedByteProcessor(); + processor.onSubscribe(upstream); + processor.subscribe(downstream); + } + + public void testEmptyBody() { + processor.next(result(null)); + processor.onComplete(); + verify(upstream, times(1)).request(1); + verify(downstream, times(0)).onNext(any()); + } + + private HttpResult result(String response) { + return new HttpResult(mock(), response == null ? new byte[0] : response.getBytes(StandardCharsets.UTF_8)); + } + + public void testEmptyParseResponse() { + processor.next(result("")); + verify(upstream, times(1)).request(1); + verify(downstream, times(0)).onNext(any()); + } + + public void testValidResponse() { + processor.next(result("{\"hello\":\"there\"}\n")); + verify(downstream, times(1)).onNext(assertArg(deque -> { + assertThat(deque, notNullValue()); + assertThat(deque.size(), is(1)); + assertThat(deque.getFirst(), is("{\"hello\":\"there\"}")); + })); + } + + public void testMultipleValidResponse() { + processor.next(result(""" + {"value": 1} + {"value": 2} + {"value": 3} + """)); + verify(upstream, times(0)).request(1); + verify(downstream, times(1)).onNext(assertArg(deque -> { + assertThat(deque, notNullValue()); + assertThat(deque.size(), is(3)); + var items = deque.iterator(); + IntStream.range(1, 4).forEach(i -> { + assertThat(items.hasNext(), is(true)); + assertThat(items.next(), containsString(String.valueOf(i))); + }); + })); + } + + public void testOnCompleteFlushesResponse() { + processor.next(result(""" + {"value": 1}""")); + + // onNext should not be called with only one value + verify(downstream, times(0)).onNext(any()); + verify(downstream, times(0)).onComplete(); + + // onComplete should flush the value pending, and onNext should be called + processor.onComplete(); + verify(downstream, times(1)).onNext(assertArg(deque -> { + assertThat(deque, notNullValue()); + assertThat(deque.size(), is(1)); + var item = deque.getFirst(); + assertThat(item, containsString(String.valueOf(1))); + })); + verify(downstream, times(0)).onComplete(); + + // next time the downstream requests data, onComplete is called + var downstreamSubscription = ArgumentCaptor.forClass(Flow.Subscription.class); + verify(downstream).onSubscribe(downstreamSubscription.capture()); + downstreamSubscription.getValue().request(1); + verify(downstream, times(1)).onComplete(); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java index c33f30d461350..c0ef4e45f101f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java @@ -17,8 +17,6 @@ import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.json.JsonXContent; -import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankBuilder; -import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankRetrieverBuilder; import java.io.IOException; import java.util.ArrayList; @@ -48,8 +46,8 @@ protected RandomRankRetrieverBuilder createTestInstance() { } @Override - protected RandomRankRetrieverBuilder doParseInstance(XContentParser parser) { - return RandomRankRetrieverBuilder.PARSER.apply( + protected RandomRankRetrieverBuilder doParseInstance(XContentParser parser) throws IOException { + return (RandomRankRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( parser, new RetrieverParserContext( new SearchUsage(), @@ -77,8 +75,8 @@ protected NamedXContentRegistry xContentRegistry() { entries.add( new NamedXContentRegistry.Entry( RetrieverBuilder.class, - new ParseField(TextSimilarityRankBuilder.NAME), - (p, c) -> TextSimilarityRankRetrieverBuilder.PARSER.apply(p, (RetrieverParserContext) c) + new ParseField(RandomRankBuilder.NAME), + (p, c) -> RandomRankRetrieverBuilder.PARSER.apply(p, (RetrieverParserContext) c) ) ); return new NamedXContentRegistry(entries); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java index 1a72cb0da2899..140b181a42a0a 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.inference.rank.textsimilarity; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.common.Strings; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.MatchNoneQueryBuilder; @@ -25,6 +26,8 @@ import org.elasticsearch.test.AbstractXContentTestCase; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.usage.SearchUsage; +import org.elasticsearch.usage.SearchUsageHolder; +import org.elasticsearch.usage.UsageService; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentParser; @@ -72,8 +75,8 @@ protected TextSimilarityRankRetrieverBuilder createTestInstance() { } @Override - protected TextSimilarityRankRetrieverBuilder doParseInstance(XContentParser parser) { - return TextSimilarityRankRetrieverBuilder.PARSER.apply( + protected TextSimilarityRankRetrieverBuilder doParseInstance(XContentParser parser) throws IOException { + return (TextSimilarityRankRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( parser, new RetrieverParserContext( new SearchUsage(), @@ -208,6 +211,45 @@ public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder } } + public void testTextSimilarityRetrieverParsing() throws IOException { + String restContent = "{" + + " \"retriever\": {" + + " \"text_similarity_reranker\": {" + + " \"retriever\": {" + + " \"test\": {" + + " \"value\": \"my-test-retriever\"" + + " }" + + " }," + + " \"field\": \"my-field\"," + + " \"inference_id\": \"my-inference-id\"," + + " \"inference_text\": \"my-inference-text\"," + + " \"rank_window_size\": 100," + + " \"min_score\": 20.0," + + " \"_name\": \"foo_reranker\"" + + " }" + + " }" + + "}"; + SearchUsageHolder searchUsageHolder = new UsageService().getSearchUsageHolder(); + try (XContentParser jsonParser = createParser(JsonXContent.jsonXContent, restContent)) { + SearchSourceBuilder source = new SearchSourceBuilder().parseXContent(jsonParser, true, searchUsageHolder, nf -> true); + assertThat(source.retriever(), instanceOf(TextSimilarityRankRetrieverBuilder.class)); + TextSimilarityRankRetrieverBuilder parsed = (TextSimilarityRankRetrieverBuilder) source.retriever(); + assertThat(parsed.minScore(), equalTo(20f)); + assertThat(parsed.retrieverName(), equalTo("foo_reranker")); + try (XContentParser parseSerialized = createParser(JsonXContent.jsonXContent, Strings.toString(source))) { + SearchSourceBuilder deserializedSource = new SearchSourceBuilder().parseXContent( + parseSerialized, + true, + searchUsageHolder, + nf -> true + ); + assertThat(deserializedSource.retriever(), instanceOf(TextSimilarityRankRetrieverBuilder.class)); + TextSimilarityRankRetrieverBuilder deserialized = (TextSimilarityRankRetrieverBuilder) source.retriever(); + assertThat(parsed, equalTo(deserialized)); + } + } + } + public void testIsCompound() { RetrieverBuilder compoundInnerRetriever = new TestRetrieverBuilder(ESTestCase.randomAlphaOfLengthBetween(5, 10)) { @Override diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java index 22503108b5262..420a635963a29 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java @@ -38,6 +38,8 @@ import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import org.elasticsearch.xpack.inference.services.InferenceEventsAssertion; +import org.elasticsearch.xpack.inference.services.cohere.completion.CohereCompletionModelTests; import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingType; import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsModel; import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsModelTests; @@ -1349,6 +1351,54 @@ public void testDefaultSimilarity() { assertEquals(SimilarityMeasure.DOT_PRODUCT, CohereService.defaultSimilarity()); } + public void testInfer_StreamRequest() throws Exception { + String responseJson = """ + {"event_type":"text-generation", "text":"hello"} + {"event_type":"text-generation", "text":"there"} + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var result = streamChatCompletion(); + + InferenceEventsAssertion.assertThat(result).hasFinishedStream().hasNoErrors().hasEvent(""" + {"completion":[{"delta":"hello"},{"delta":"there"}]}"""); + } + + private InferenceServiceResults streamChatCompletion() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new CohereService(senderFactory, createWithEmptySettings(threadPool))) { + var model = CohereCompletionModelTests.createModel(getUrl(webServer), "secret", "model"); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + null, + List.of("abc"), + true, + new HashMap<>(), + InputType.INGEST, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + return listener.actionGet(TIMEOUT); + } + } + + public void testInfer_StreamRequest_ErrorResponse() throws Exception { + String responseJson = """ + { "event_type":"stream-end", "finish_reason":"ERROR", "response":{ "text": "how dare you" } } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var result = streamChatCompletion(); + + InferenceEventsAssertion.assertThat(result) + .hasFinishedStream() + .hasNoEvents() + .hasErrorWithStatusCode(500) + .hasErrorContaining("how dare you"); + } + private Map getRequestConfigMap( Map serviceSettings, Map taskSettings, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java index cd6da4c0ad8d8..db7189dc1af17 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.core.action.util.QueryPage; @@ -44,15 +43,14 @@ import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig; import org.elasticsearch.xpack.core.ml.inference.TrainedModelPrefixStrings; import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; -import org.elasticsearch.xpack.core.ml.inference.results.InferenceChunkedTextExpansionResultsTests; -import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextEmbeddingFloatResults; -import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextEmbeddingFloatResultsTests; -import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; +import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResultsTests; +import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; +import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResultsTests; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate; -import org.elasticsearch.xpack.core.utils.FloatConversionUtils; import org.elasticsearch.xpack.inference.InferencePlugin; +import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; import org.elasticsearch.xpack.inference.services.ServiceFields; import org.junit.After; import org.junit.Before; @@ -663,14 +661,12 @@ public void testParsePersistedConfig() { @SuppressWarnings("unchecked") public void testChunkInfer_e5() { var mlTrainedModelResults = new ArrayList(); - mlTrainedModelResults.add(MlChunkedTextEmbeddingFloatResultsTests.createRandomResults()); - mlTrainedModelResults.add(MlChunkedTextEmbeddingFloatResultsTests.createRandomResults()); - mlTrainedModelResults.add(new ErrorInferenceResults(new RuntimeException("boom"))); + mlTrainedModelResults.add(MlTextEmbeddingResultsTests.createRandomResults()); + mlTrainedModelResults.add(MlTextEmbeddingResultsTests.createRandomResults()); var response = new InferModelAction.Response(mlTrainedModelResults, "foo", true); - ThreadPool threadpool = new TestThreadPool("test"); Client client = mock(Client.class); - when(client.threadPool()).thenReturn(threadpool); + when(client.threadPool()).thenReturn(threadPool); doAnswer(invocationOnMock -> { var listener = (ActionListener) invocationOnMock.getArguments()[2]; listener.onResponse(response); @@ -687,47 +683,26 @@ public void testChunkInfer_e5() { var gotResults = new AtomicBoolean(); var resultsListener = ActionListener.>wrap(chunkedResponse -> { - assertThat(chunkedResponse, hasSize(3)); + assertThat(chunkedResponse, hasSize(2)); assertThat(chunkedResponse.get(0), instanceOf(InferenceChunkedTextEmbeddingFloatResults.class)); var result1 = (InferenceChunkedTextEmbeddingFloatResults) chunkedResponse.get(0); - assertEquals( - ((MlChunkedTextEmbeddingFloatResults) mlTrainedModelResults.get(0)).getChunks().size(), - result1.getChunks().size() - ); - assertEquals( - ((MlChunkedTextEmbeddingFloatResults) mlTrainedModelResults.get(0)).getChunks().get(0).matchedText(), - result1.getChunks().get(0).matchedText() - ); + assertThat(result1.chunks(), hasSize(1)); assertArrayEquals( - (FloatConversionUtils.floatArrayOf( - ((MlChunkedTextEmbeddingFloatResults) mlTrainedModelResults.get(0)).getChunks().get(0).embedding() - )), + ((MlTextEmbeddingResults) mlTrainedModelResults.get(0)).getInferenceAsFloat(), result1.getChunks().get(0).embedding(), 0.0001f ); + assertEquals("foo", result1.getChunks().get(0).matchedText()); assertThat(chunkedResponse.get(1), instanceOf(InferenceChunkedTextEmbeddingFloatResults.class)); var result2 = (InferenceChunkedTextEmbeddingFloatResults) chunkedResponse.get(1); - // assertEquals(((MlChunkedTextEmbeddingFloatResults) mlTrainedModelResults.get(1)).getChunks(), result2.getChunks()); - - assertEquals( - ((MlChunkedTextEmbeddingFloatResults) mlTrainedModelResults.get(1)).getChunks().size(), - result2.getChunks().size() - ); - assertEquals( - ((MlChunkedTextEmbeddingFloatResults) mlTrainedModelResults.get(1)).getChunks().get(0).matchedText(), - result2.getChunks().get(0).matchedText() - ); + assertThat(result2.chunks(), hasSize(1)); assertArrayEquals( - (FloatConversionUtils.floatArrayOf( - ((MlChunkedTextEmbeddingFloatResults) mlTrainedModelResults.get(1)).getChunks().get(0).embedding() - )), + ((MlTextEmbeddingResults) mlTrainedModelResults.get(1)).getInferenceAsFloat(), result2.getChunks().get(0).embedding(), 0.0001f ); + assertEquals("bar", result2.getChunks().get(0).matchedText()); - var result3 = (ErrorChunkedInferenceResults) chunkedResponse.get(2); - assertThat(result3.getException(), instanceOf(RuntimeException.class)); - assertThat(result3.getException().getMessage(), containsString("boom")); gotResults.set(true); }, ESTestCase::fail); @@ -739,26 +714,21 @@ public void testChunkInfer_e5() { InputType.SEARCH, new ChunkingOptions(null, null), InferenceAction.Request.DEFAULT_TIMEOUT, - ActionListener.runAfter(resultsListener, () -> terminate(threadpool)) + ActionListener.runAfter(resultsListener, () -> terminate(threadPool)) ); - if (gotResults.get() == false) { - terminate(threadpool); - } assertTrue("Listener not called", gotResults.get()); } @SuppressWarnings("unchecked") public void testChunkInfer_Sparse() { var mlTrainedModelResults = new ArrayList(); - mlTrainedModelResults.add(InferenceChunkedTextExpansionResultsTests.createRandomResults()); - mlTrainedModelResults.add(InferenceChunkedTextExpansionResultsTests.createRandomResults()); - mlTrainedModelResults.add(new ErrorInferenceResults(new RuntimeException("boom"))); + mlTrainedModelResults.add(TextExpansionResultsTests.createRandomResults()); + mlTrainedModelResults.add(TextExpansionResultsTests.createRandomResults()); var response = new InferModelAction.Response(mlTrainedModelResults, "foo", true); - ThreadPool threadpool = new TestThreadPool("test"); Client client = mock(Client.class); - when(client.threadPool()).thenReturn(threadpool); + when(client.threadPool()).thenReturn(threadPool); doAnswer(invocationOnMock -> { var listener = (ActionListener) invocationOnMock.getArguments()[2]; listener.onResponse(response); @@ -775,16 +745,21 @@ public void testChunkInfer_Sparse() { var gotResults = new AtomicBoolean(); var resultsListener = ActionListener.>wrap(chunkedResponse -> { - assertThat(chunkedResponse, hasSize(3)); + assertThat(chunkedResponse, hasSize(2)); assertThat(chunkedResponse.get(0), instanceOf(InferenceChunkedSparseEmbeddingResults.class)); var result1 = (InferenceChunkedSparseEmbeddingResults) chunkedResponse.get(0); - assertEquals(((MlChunkedTextExpansionResults) mlTrainedModelResults.get(0)).getChunks(), result1.getChunkedResults()); + assertEquals( + ((TextExpansionResults) mlTrainedModelResults.get(0)).getWeightedTokens(), + result1.getChunkedResults().get(0).weightedTokens() + ); + assertEquals("foo", result1.getChunkedResults().get(0).matchedText()); assertThat(chunkedResponse.get(1), instanceOf(InferenceChunkedSparseEmbeddingResults.class)); var result2 = (InferenceChunkedSparseEmbeddingResults) chunkedResponse.get(1); - assertEquals(((MlChunkedTextExpansionResults) mlTrainedModelResults.get(1)).getChunks(), result2.getChunkedResults()); - var result3 = (ErrorChunkedInferenceResults) chunkedResponse.get(2); - assertThat(result3.getException(), instanceOf(RuntimeException.class)); - assertThat(result3.getException().getMessage(), containsString("boom")); + assertEquals( + ((TextExpansionResults) mlTrainedModelResults.get(1)).getWeightedTokens(), + result2.getChunkedResults().get(0).weightedTokens() + ); + assertEquals("bar", result2.getChunkedResults().get(0).matchedText()); gotResults.set(true); }, ESTestCase::fail); @@ -796,12 +771,9 @@ public void testChunkInfer_Sparse() { InputType.SEARCH, new ChunkingOptions(null, null), InferenceAction.Request.DEFAULT_TIMEOUT, - ActionListener.runAfter(resultsListener, () -> terminate(threadpool)) + ActionListener.runAfter(resultsListener, () -> terminate(threadPool)) ); - if (gotResults.get() == false) { - terminate(threadpool); - } assertTrue("Listener not called", gotResults.get()); } @@ -811,57 +783,103 @@ public void testChunkInferSetsTokenization() { var expectedWindowSize = new AtomicReference(); Client client = mock(Client.class); - ThreadPool threadpool = new TestThreadPool("test"); - try { - when(client.threadPool()).thenReturn(threadpool); - doAnswer(invocationOnMock -> { - var request = (InferTrainedModelDeploymentAction.Request) invocationOnMock.getArguments()[1]; - assertThat(request.getUpdate(), instanceOf(TokenizationConfigUpdate.class)); - var update = (TokenizationConfigUpdate) request.getUpdate(); - assertEquals(update.getSpanSettings().span(), expectedSpan.get()); - assertEquals(update.getSpanSettings().maxSequenceLength(), expectedWindowSize.get()); - return null; - }).when(client) - .execute( - same(InferTrainedModelDeploymentAction.INSTANCE), - any(InferTrainedModelDeploymentAction.Request.class), - any(ActionListener.class) - ); - - var model = new MultilingualE5SmallModel( - "foo", - TaskType.TEXT_EMBEDDING, - "e5", - new MultilingualE5SmallInternalServiceSettings(1, 1, "cross-platform", null) + when(client.threadPool()).thenReturn(threadPool); + doAnswer(invocationOnMock -> { + var request = (InferTrainedModelDeploymentAction.Request) invocationOnMock.getArguments()[1]; + assertThat(request.getUpdate(), instanceOf(TokenizationConfigUpdate.class)); + var update = (TokenizationConfigUpdate) request.getUpdate(); + assertEquals(update.getSpanSettings().span(), expectedSpan.get()); + assertEquals(update.getSpanSettings().maxSequenceLength(), expectedWindowSize.get()); + return null; + }).when(client) + .execute( + same(InferTrainedModelDeploymentAction.INSTANCE), + any(InferTrainedModelDeploymentAction.Request.class), + any(ActionListener.class) ); - var service = createService(client); - expectedSpan.set(-1); - expectedWindowSize.set(null); - service.chunkedInfer( - model, - List.of("foo", "bar"), - Map.of(), - InputType.SEARCH, - null, - InferenceAction.Request.DEFAULT_TIMEOUT, - ActionListener.wrap(r -> fail("unexpected result"), e -> fail(e.getMessage())) - ); + var model = new MultilingualE5SmallModel( + "foo", + TaskType.TEXT_EMBEDDING, + "e5", + new MultilingualE5SmallInternalServiceSettings(1, 1, "cross-platform", null) + ); + var service = createService(client); + + expectedSpan.set(-1); + expectedWindowSize.set(null); + service.chunkedInfer( + model, + List.of("foo", "bar"), + Map.of(), + InputType.SEARCH, + null, + InferenceAction.Request.DEFAULT_TIMEOUT, + ActionListener.wrap(r -> fail("unexpected result"), e -> fail(e.getMessage())) + ); + + expectedSpan.set(-1); + expectedWindowSize.set(256); + service.chunkedInfer( + model, + List.of("foo", "bar"), + Map.of(), + InputType.SEARCH, + new ChunkingOptions(256, null), + InferenceAction.Request.DEFAULT_TIMEOUT, + ActionListener.wrap(r -> fail("unexpected result"), e -> fail(e.getMessage())) + ); - expectedSpan.set(-1); - expectedWindowSize.set(256); - service.chunkedInfer( - model, - List.of("foo", "bar"), - Map.of(), - InputType.SEARCH, - new ChunkingOptions(256, null), - InferenceAction.Request.DEFAULT_TIMEOUT, - ActionListener.wrap(r -> fail("unexpected result"), e -> fail(e.getMessage())) - ); - } finally { - terminate(threadpool); - } + } + + @SuppressWarnings("unchecked") + public void testChunkInfer_FailsBatch() { + var mlTrainedModelResults = new ArrayList(); + mlTrainedModelResults.add(MlTextEmbeddingResultsTests.createRandomResults()); + mlTrainedModelResults.add(MlTextEmbeddingResultsTests.createRandomResults()); + mlTrainedModelResults.add(new ErrorInferenceResults(new RuntimeException("boom"))); + var response = new InferModelAction.Response(mlTrainedModelResults, "foo", true); + + Client client = mock(Client.class); + when(client.threadPool()).thenReturn(threadPool); + doAnswer(invocationOnMock -> { + var listener = (ActionListener) invocationOnMock.getArguments()[2]; + listener.onResponse(response); + return null; + }).when(client).execute(same(InferModelAction.INSTANCE), any(InferModelAction.Request.class), any(ActionListener.class)); + + var model = new MultilingualE5SmallModel( + "foo", + TaskType.TEXT_EMBEDDING, + "e5", + new MultilingualE5SmallInternalServiceSettings(1, 1, "cross-platform", null) + ); + var service = createService(client); + + var gotResults = new AtomicBoolean(); + var resultsListener = ActionListener.>wrap(chunkedResponse -> { + assertThat(chunkedResponse, hasSize(3)); + // a single failure fails the batch + for (var er : chunkedResponse) { + assertThat(er, instanceOf(ErrorChunkedInferenceResults.class)); + assertEquals("boom", ((ErrorChunkedInferenceResults) er).getException().getMessage()); + } + + gotResults.set(true); + }, ESTestCase::fail); + + service.chunkedInfer( + model, + null, + List.of("foo", "bar", "baz"), + Map.of(), + InputType.SEARCH, + new ChunkingOptions(null, null), + InferenceAction.Request.DEFAULT_TIMEOUT, + ActionListener.runAfter(resultsListener, () -> terminate(threadPool)) + ); + + assertTrue("Listener not called", gotResults.get()); } public void testParsePersistedConfig_Rerank() { @@ -992,14 +1010,12 @@ public void testBuildInferenceRequest() { var inputs = randomList(1, 3, () -> randomAlphaOfLength(4)); var inputType = randomFrom(InputType.SEARCH, InputType.INGEST); var timeout = randomTimeValue(); - var chunk = randomBoolean(); var request = ElasticsearchInternalService.buildInferenceRequest( id, TextEmbeddingConfigUpdate.EMPTY_INSTANCE, inputs, inputType, - timeout, - chunk + timeout ); assertEquals(id, request.getId()); @@ -1009,7 +1025,7 @@ public void testBuildInferenceRequest() { request.getPrefixType() ); assertEquals(timeout, request.getInferenceTimeout()); - assertEquals(chunk, request.isChunked()); + assertEquals(false, request.isChunked()); } @SuppressWarnings("unchecked") @@ -1132,6 +1148,32 @@ public void testModelVariantDoesNotMatchArchitecturesAndIsNotPlatformAgnostic() } } + public void testEmbeddingTypeFromTaskTypeAndSettings() { + assertEquals( + EmbeddingRequestChunker.EmbeddingType.SPARSE, + ElasticsearchInternalService.embeddingTypeFromTaskTypeAndSettings( + TaskType.SPARSE_EMBEDDING, + new ElasticsearchInternalServiceSettings(1, 1, "foo", null) + ) + ); + assertEquals( + EmbeddingRequestChunker.EmbeddingType.FLOAT, + ElasticsearchInternalService.embeddingTypeFromTaskTypeAndSettings( + TaskType.TEXT_EMBEDDING, + new MultilingualE5SmallInternalServiceSettings(1, 1, "foo", null) + ) + ); + + var e = expectThrows( + ElasticsearchStatusException.class, + () -> ElasticsearchInternalService.embeddingTypeFromTaskTypeAndSettings( + TaskType.COMPLETION, + new ElasticsearchInternalServiceSettings(1, 1, "foo", null) + ) + ); + assertThat(e.getMessage(), containsString("Chunking is not supported for task type [completion]")); + } + private ElasticsearchInternalService createService(Client client) { var context = new InferenceServiceExtension.InferenceServiceFactoryContext(client, threadPool); return new ElasticsearchInternalService(context); diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml new file mode 100644 index 0000000000000..1c9d32a4289b9 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml @@ -0,0 +1,19 @@ +version: ${xpack.oteldata.template.version} +_meta: + description: | + Default mappings that can be changed by users for + the OpenTelemetry metrics index template installed by x-pack + managed: true +template: + mappings: + dynamic_templates: + - ecs_ip: + mapping: + type: ip + path_match: [ "ip", "*.ip", "*_ip" ] + match_mapping_type: string + - all_strings_to_keywords: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml index f350997de9e01..107901adb834f 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml @@ -8,7 +8,7 @@ template: index: mode: logsdb sort: - field: [ "resource.attributes.host.name" ] + field: [ "resource.attributes.host.name", "@timestamp" ] mappings: properties: attributes: diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml index 0e77bc208eed4..2b0d1ec536fa6 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml @@ -8,7 +8,7 @@ template: index: mode: logsdb sort: - field: [ "resource.attributes.host.name" ] + field: [ "resource.attributes.host.name", "@timestamp" ] mappings: _source: mode: synthetic diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml index c2a318f809b7d..3b4c3127bb71c 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml @@ -14,6 +14,7 @@ composed_of: - semconv-resource-to-ecs@mappings - metrics@custom - metrics-otel@custom + - ecs-tsdb@mappings ignore_missing_component_templates: - metrics@custom - metrics-otel@custom diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.10m@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.10m@template.yaml new file mode 100644 index 0000000000000..f5033135120bc --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.10m@template.yaml @@ -0,0 +1,40 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-service_destination.10m.otel-*"] +priority: 130 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-10m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-10m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 10m + name: + type: constant_keyword + value: service_destination diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.1m@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.1m@template.yaml new file mode 100644 index 0000000000000..9168062f30bfb --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.1m@template.yaml @@ -0,0 +1,39 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-service_destination.1m.otel-*"] +priority: 130 +data_stream: {} +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-1m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-1m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 1m + name: + type: constant_keyword + value: service_destination diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.60m@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.60m@template.yaml new file mode 100644 index 0000000000000..47c2d7d014322 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.60m@template.yaml @@ -0,0 +1,40 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-service_destination.60m.otel-*"] +priority: 130 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-60m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-60m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 60m + name: + type: constant_keyword + value: service_destination diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.10m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.10m.otel@template.yaml new file mode 100644 index 0000000000000..c9438e8c27402 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.10m.otel@template.yaml @@ -0,0 +1,40 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-service_summary.10m.otel-*"] +priority: 130 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-10m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-10m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 10m + name: + type: constant_keyword + value: service_summary diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.1m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.1m.otel@template.yaml new file mode 100644 index 0000000000000..b29caa3fe34a7 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.1m.otel@template.yaml @@ -0,0 +1,39 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-service_summary.1m.otel-*"] +priority: 130 +data_stream: {} +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-1m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-1m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 1m + name: + type: constant_keyword + value: service_summary diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.60m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.60m.otel@template.yaml new file mode 100644 index 0000000000000..4cab3e41a1dfa --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.60m.otel@template.yaml @@ -0,0 +1,40 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-service_summary.60m.otel-*"] +priority: 130 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-60m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-60m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 60m + name: + type: constant_keyword + value: service_summary diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.10m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.10m.otel@template.yaml new file mode 100644 index 0000000000000..037f3546205d6 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.10m.otel@template.yaml @@ -0,0 +1,40 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-service_transaction.10m.otel-*"] +priority: 130 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-10m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-10m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 10m + name: + type: constant_keyword + value: service_transaction diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.1m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.1m.otel@template.yaml new file mode 100644 index 0000000000000..303ac2c406fd0 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.1m.otel@template.yaml @@ -0,0 +1,39 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-service_transaction.1m.otel-*"] +priority: 130 +data_stream: {} +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-1m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-1m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 1m + name: + type: constant_keyword + value: service_transaction diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.60m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.60m.otel@template.yaml new file mode 100644 index 0000000000000..ea42079ced4dd --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.60m.otel@template.yaml @@ -0,0 +1,40 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-service_transaction.60m.otel-*"] +priority: 130 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-60m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-60m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 60m + name: + type: constant_keyword + value: service_transaction diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.10m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.10m.otel@template.yaml new file mode 100644 index 0000000000000..81e70cc3361fc --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.10m.otel@template.yaml @@ -0,0 +1,40 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-transaction.10m.otel-*"] +priority: 130 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-10m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-10m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 10m + name: + type: constant_keyword + value: transaction diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.1m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.1m.otel@template.yaml new file mode 100644 index 0000000000000..c54b90bf8b683 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.1m.otel@template.yaml @@ -0,0 +1,39 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-transaction.1m.otel-*"] +priority: 130 +data_stream: {} +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-1m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-1m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 1m + name: + type: constant_keyword + value: transaction diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.60m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.60m.otel@template.yaml new file mode 100644 index 0000000000000..8afe8b87951c0 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.60m.otel@template.yaml @@ -0,0 +1,40 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-transaction.60m.otel-*"] +priority: 130 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-60m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-60m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 60m + name: + type: constant_keyword + value: transaction diff --git a/x-pack/plugin/otel-data/src/main/resources/resources.yaml b/x-pack/plugin/otel-data/src/main/resources/resources.yaml index ba219b09388fb..e32037901a49c 100644 --- a/x-pack/plugin/otel-data/src/main/resources/resources.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/resources.yaml @@ -1,7 +1,7 @@ # "version" holds the version of the templates and ingest pipelines installed # by xpack-plugin otel-data. This must be increased whenever an existing template is # changed, in order for it to be updated on Elasticsearch upgrade. -version: 3 +version: 4 component-templates: - otel@mappings @@ -9,7 +9,20 @@ component-templates: - semconv-resource-to-ecs@mappings - metrics-otel@mappings - traces-otel@mappings + - ecs-tsdb@mappings index-templates: - logs-otel@template - metrics-otel@template - traces-otel@template + - metrics-transaction.60m.otel@template + - metrics-transaction.10m.otel@template + - metrics-transaction.1m.otel@template + - metrics-service_transaction.60m.otel@template + - metrics-service_transaction.10m.otel@template + - metrics-service_transaction.1m.otel@template + - metrics-service_summary.60m.otel@template + - metrics-service_summary.10m.otel@template + - metrics-service_summary.1m.otel@template + - metrics-service_destination.60m@template + - metrics-service_destination.10m@template + - metrics-service_destination.1m@template diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml index 657453bf4ae9f..fc162d0647d08 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml @@ -70,3 +70,23 @@ setup: - match: { hits.hits.0.fields.error\.exception\.type: ["MyException"] } - match: { hits.hits.0.fields.error\.exception\.message: ["foo"] } - match: { hits.hits.0.fields.error\.stack_trace: ["Exception in thread \"main\" java.lang.RuntimeException: Test exception\n at com.example.GenerateTrace.methodB(GenerateTrace.java:13)\n at com.example.GenerateTrace.methodA(GenerateTrace.java:9)\n at com.example.GenerateTrace.main(GenerateTrace.java:5)"] } +--- +"resource.attributes.host.name @timestamp should be used as sort fields": + - do: + bulk: + index: logs-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:49:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "body_text":"error1"}' + - is_false: errors + - do: + indices.get_data_stream: + name: logs-generic.otel-default + - set: { data_streams.0.indices.0.index_name: datastream-backing-index } + - do: + indices.get_settings: + index: $datastream-backing-index + - is_true: $datastream-backing-index + - match: { .$datastream-backing-index.settings.index.sort.field.0: "resource.attributes.host.name" } + - match: { .$datastream-backing-index.settings.index.sort.field.1: "@timestamp" } diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml index 6e51f3f91ddb5..d5b87c9b45116 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml @@ -76,23 +76,6 @@ setup: - match: { hits.hits.0._source.links.1.trace_id: "4aaa9f33312b3dbb8b2c2c62bb7abe1a1" } - match: { hits.hits.0._source.links.1.span_id: "b3b7d1f1f1b4e1e1" } --- -"Default data_stream.type must be traces": - - do: - bulk: - index: traces-generic.otel-default - refresh: true - body: - - create: {} - - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","type":"traces","namespace":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' - - is_false: errors - - do: - search: - index: traces-generic.otel-default - body: - fields: ["data_stream.type"] - - length: { hits.hits: 1 } - - match: { hits.hits.0.fields.data_stream\.type: ["traces"] } ---- Conflicting attribute types: - do: bulk: @@ -117,3 +100,23 @@ Conflicting attribute types: - length: { hits.hits: 2 } - match: { hits.hits.0.fields.attributes\.http\.status_code: [200] } - match: { hits.hits.1._ignored: ["attributes.http.status_code"] } +--- +"resource.attributes.host.name @timestamp should be used as sort fields": + - do: + bulk: + index: traces-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:49:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "span_id":"1"}' + - is_false: errors + - do: + indices.get_data_stream: + name: traces-generic.otel-default + - set: { data_streams.0.indices.0.index_name: datastream-backing-index } + - do: + indices.get_settings: + index: $datastream-backing-index + - is_true: $datastream-backing-index + - match: { .$datastream-backing-index.settings.index.sort.field.0: "resource.attributes.host.name" } + - match: { .$datastream-backing-index.settings.index.sort.field.1: "@timestamp" } diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_aggregated_metrics_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_aggregated_metrics_tests.yml new file mode 100644 index 0000000000000..c26a53d841f59 --- /dev/null +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_aggregated_metrics_tests.yml @@ -0,0 +1,428 @@ +--- +setup: + - do: + cluster.health: + wait_for_events: languid + - do: + cluster.put_component_template: + name: metrics-otel@custom + body: + template: + settings: + index: + routing_path: [unit, attributes.*, resource.attributes.*] + mode: time_series + time_series: + start_time: 2024-07-01T13:03:08.138Z +--- +"metrics-service_destination.10m must be hidden": + - do: + bulk: + index: metrics-service_destination.10m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_destination"},"resource":{"attributes":{ "metricset.interval": "10m" } } }' + - is_false: errors + - do: + search: + index: metrics-service_destination.10m.otel-default + body: + fields: ["metricset.name", "metricset.interval"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.metricset\.name: ["service_destination"] } + - match: { hits.hits.0.fields.metricset\.interval: ["10m"] } + - do: + indices.get_data_stream: + name: metrics-service_destination.10m.otel-default + - match: { data_streams.0.hidden: true } +--- +"metrics-service_destination.60m must be hidden": + - do: + bulk: + index: metrics-service_destination.60m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_destination" },"resource":{"attributes":{ "metricset.interval": "60m" } } }' + - is_false: errors + - do: + search: + index: metrics-service_destination.60m.otel-default + body: + fields: ["metricset.name", "metricset.interval"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.metricset\.name: ["service_destination"] } + - match: { hits.hits.0.fields.metricset\.interval: ["60m"] } + - do: + indices.get_data_stream: + name: metrics-service_destination.60m.otel-default + - match: { data_streams.0.hidden: true } +--- +"metrics-service_summary.10m must be hidden": + - do: + bulk: + index: metrics-service_summary.10m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_summary"},"resource":{"attributes":{ "metricset.interval": "10m" } } }' + - is_false: errors + - do: + search: + index: metrics-service_summary.10m.otel-default + body: + fields: ["metricset.name", "metricset.interval"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.metricset\.name: ["service_summary"] } + - match: { hits.hits.0.fields.metricset\.interval: ["10m"] } + - do: + indices.get_data_stream: + name: metrics-service_summary.10m.otel-default + - match: { data_streams.0.hidden: true } +--- +"metrics-service_summary.60m must be hidden": + - do: + bulk: + index: metrics-service_summary.60m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_summary" },"resource":{"attributes":{ "metricset.interval": "60m" } } }' + - is_false: errors + - do: + search: + index: metrics-service_summary.60m.otel-default + body: + fields: ["metricset.name", "metricset.interval"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.metricset\.name: ["service_summary"] } + - match: { hits.hits.0.fields.metricset\.interval: ["60m"] } + - do: + indices.get_data_stream: + name: metrics-service_summary.60m.otel-default + - match: { data_streams.0.hidden: true } +--- +"metrics-service_transaction.10m must be hidden": + - do: + bulk: + index: metrics-service_transaction.10m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_transaction" },"resource":{"attributes":{ "metricset.interval": "10m" } } }' + - is_false: errors + - do: + search: + index: metrics-service_transaction.10m.otel-default + body: + fields: ["metricset.name", "metricset.interval"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.metricset\.name: ["service_transaction"] } + - match: { hits.hits.0.fields.metricset\.interval: ["10m"] } + - do: + indices.get_data_stream: + name: metrics-service_transaction.10m.otel-default + - match: { data_streams.0.hidden: true } +--- +"metrics-service_transaction.60m must be hidden": + - do: + bulk: + index: metrics-service_transaction.60m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_transaction"},"resource":{"attributes":{ "metricset.interval": "60m" } } }' + - is_false: errors + - do: + search: + index: metrics-service_transaction.60m.otel-default + body: + fields: ["metricset.name", "metricset.interval"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.metricset\.name: ["service_transaction"] } + - match: { hits.hits.0.fields.metricset\.interval: ["60m"] } + - do: + indices.get_data_stream: + name: metrics-service_transaction.60m.otel-default + - match: { data_streams.0.hidden: true } +--- +"metrics-transaction.10m must be hidden": + - do: + bulk: + index: metrics-transaction.10m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "transaction"},"resource":{"attributes":{ "metricset.interval": "10m" } } }' + - is_false: errors + - do: + search: + index: metrics-transaction.10m.otel-default + body: + fields: ["metricset.name", "metricset.interval"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.metricset\.name: ["transaction"] } + - match: { hits.hits.0.fields.metricset\.interval: ["10m"] } + - do: + indices.get_data_stream: + name: metrics-transaction.10m.otel-default + - match: { data_streams.0.hidden: true } +--- +"metrics-transaction.60m must be hidden": + - do: + bulk: + index: metrics-transaction.60m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "transaction"},"resource":{"attributes":{ "metricset.interval": "60m" } } }' + - is_false: errors + - do: + search: + index: metrics-transaction.60m.otel-default + body: + fields: ["metricset.name", "metricset.interval"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.metricset\.name: ["transaction"] } + - match: { hits.hits.0.fields.metricset\.interval: ["60m"] } + - do: + indices.get_data_stream: + name: metrics-transaction.60m.otel-default + - match: { data_streams.0.hidden: true } +--- +"Terms aggregation on metricset.interval from metrics-transaction must by default only contain 1m": + - do: + bulk: + index: metrics-transaction.60m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "transaction"},"resource":{"attributes":{ "metricset.interval": "60m" } } }' + - do: + bulk: + index: metrics-transaction.10m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "transaction"},"resource":{"attributes":{ "metricset.interval": "10m" } } }' + - do: + bulk: + index: metrics-transaction.1m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "transaction"},"resource":{"attributes":{ "metricset.interval": "1m" } } }' + - is_false: errors + - do: + search: + index: metrics-*.otel-default + body: > + { + + "size": 0, + "aggs": { + "intervals": { + "terms": { + "field": "metricset.interval" + } + } + } + } + - length: { aggregations.intervals.buckets: 1 } + - match: { aggregations.intervals.buckets.0.key: "1m" } + - match: { aggregations.intervals.buckets.0.doc_count: 1 } + # With including hidden indices, 10m and 60m aggregation also show up + - do: + search: + index: metrics-*.otel-default + expand_wildcards: open,hidden + body: > + { + "size": 0, + "aggs": { + "intervals": { + "terms": { + "field": "metricset.interval" + } + } + } + } + - length: { aggregations.intervals.buckets: 3 } +--- +"Terms aggregation on metricset.interval from metrics-service_transaction must by default only contain 1m": + - do: + bulk: + index: metrics-service_transaction.60m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_transaction"},"resource":{"attributes":{ "metricset.interval": "60m" } } }' + - do: + bulk: + index: metrics-service_transaction.10m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_transaction"},"resource":{"attributes":{ "metricset.interval": "10m" } } }' + - do: + bulk: + index: metrics-service_transaction.1m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_transaction"},"resource":{"attributes":{ "metricset.interval": "1m" } } }' + - is_false: errors + - do: + search: + index: metrics-*.otel-default + body: > + { + + "size": 0, + "aggs": { + "intervals": { + "terms": { + "field": "metricset.interval" + } + } + } + } + - length: { aggregations.intervals.buckets: 1 } + - match: { aggregations.intervals.buckets.0.key: "1m" } + - match: { aggregations.intervals.buckets.0.doc_count: 1 } + # With including hidden indices, 10m and 60m aggregation also show up + - do: + search: + index: metrics-*.otel-default + expand_wildcards: open,hidden + body: > + { + "size": 0, + "aggs": { + "intervals": { + "terms": { + "field": "metricset.interval" + } + } + } + } + - length: { aggregations.intervals.buckets: 3 } +--- +"Terms aggregation on metricset.interval from metrics-service_summary must by default only contain 1m": + - do: + bulk: + index: metrics-service_summary.60m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_summary"},"resource":{"attributes":{ "metricset.interval": "60m" } } }' + - do: + bulk: + index: metrics-service_summary.10m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_summary"},"resource":{"attributes":{ "metricset.interval": "10m" } } }' + - do: + bulk: + index: metrics-service_summary.1m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_summary"},"resource":{"attributes":{ "metricset.interval": "1m" } } }' + - is_false: errors + - do: + search: + index: metrics-*.otel-default + body: > + { + + "size": 0, + "aggs": { + "intervals": { + "terms": { + "field": "metricset.interval" + } + } + } + } + - length: { aggregations.intervals.buckets: 1 } + - match: { aggregations.intervals.buckets.0.key: "1m" } + - match: { aggregations.intervals.buckets.0.doc_count: 1 } + # With including hidden indices, 10m and 60m aggregation also show up + - do: + search: + index: metrics-*.otel-default + expand_wildcards: open,hidden + body: > + { + "size": 0, + "aggs": { + "intervals": { + "terms": { + "field": "metricset.interval" + } + } + } + } + - length: { aggregations.intervals.buckets: 3 } +--- +"Terms aggregation on metricset.interval from metrics-service_destination must by default only contain 1m": + - do: + bulk: + index: metrics-service_destination.60m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_destination"},"resource":{"attributes":{ "metricset.interval": "60m" } } }' + - do: + bulk: + index: metrics-service_destination.10m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_destination"},"resource":{"attributes":{ "metricset.interval": "10m" } } }' + - do: + bulk: + index: metrics-service_destination.1m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_destination"},"resource":{"attributes":{ "metricset.interval": "1m" } } }' + - is_false: errors + - do: + search: + index: metrics-*.otel-default + body: > + { + + "size": 0, + "aggs": { + "intervals": { + "terms": { + "field": "metricset.interval" + } + } + } + } + - length: { aggregations.intervals.buckets: 1 } + - match: { aggregations.intervals.buckets.0.key: "1m" } + - match: { aggregations.intervals.buckets.0.doc_count: 1 } + # With including hidden indices, 10m and 60m aggregation also show up + - do: + search: + index: metrics-*.otel-default + expand_wildcards: open,hidden + body: > + { + "size": 0, + "aggs": { + "intervals": { + "terms": { + "field": "metricset.interval" + } + } + } + } + - length: { aggregations.intervals.buckets: 3 } diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java index 2e7bc44811bf6..be64d34dc8765 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java @@ -33,7 +33,6 @@ import java.util.Collection; import java.util.List; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -98,7 +97,7 @@ protected void setupIndex() { } } """; - createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).build()); + createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 5)).build()); admin().indices().preparePutMapping(INDEX).setSource(mapping, XContentType.JSON).get(); indexDoc(INDEX, "doc_1", DOC_FIELD, "doc_1", TOPIC_FIELD, "technology", TEXT_FIELD, "term"); indexDoc( @@ -167,8 +166,8 @@ public void testRRFPagination() { QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 3, 2, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + // this one retrieves docs 2, 3, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -214,8 +213,8 @@ public void testRRFWithAggs() { QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 3, 2, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + // this one retrieves docs 2, 3, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -266,8 +265,8 @@ public void testRRFWithCollapse() { QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 3, 2, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + // this one retrieves docs 2, 3, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -320,8 +319,8 @@ public void testRRFRetrieverWithCollapseAndAggs() { QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 3, 2, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + // this one retrieves docs 2, 3, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -383,8 +382,8 @@ public void testMultipleRRFRetrievers() { QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 3, 2, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + // this one retrieves docs 2, 3, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -446,8 +445,8 @@ public void testRRFExplainWithNamedRetrievers() { QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 3, 2, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + // this one retrieves docs 2, 3, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -474,13 +473,12 @@ public void testRRFExplainWithNamedRetrievers() { assertThat(resp.getHits().getAt(0).getExplanation().getDetails().length, equalTo(2)); var rrfDetails = resp.getHits().getAt(0).getExplanation().getDetails()[0]; assertThat(rrfDetails.getDetails().length, equalTo(3)); - assertThat(rrfDetails.getDescription(), containsString("computed for initial ranks [2, 1, 2]")); + assertThat(rrfDetails.getDescription(), containsString("computed for initial ranks [2, 1, 1]")); - assertThat(rrfDetails.getDetails()[0].getDescription(), containsString("for rank [2] in query at index [0]")); assertThat(rrfDetails.getDetails()[0].getDescription(), containsString("for rank [2] in query at index [0]")); assertThat(rrfDetails.getDetails()[0].getDescription(), containsString("[my_custom_retriever]")); assertThat(rrfDetails.getDetails()[1].getDescription(), containsString("for rank [1] in query at index [1]")); - assertThat(rrfDetails.getDetails()[2].getDescription(), containsString("for rank [2] in query at index [2]")); + assertThat(rrfDetails.getDetails()[2].getDescription(), containsString("for rank [1] in query at index [2]")); }); } @@ -503,8 +501,8 @@ public void testRRFExplainWithAnotherNestedRRF() { QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 3, 2, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + // this one retrieves docs 2, 3, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null); RRFRetrieverBuilder nestedRRF = new RRFRetrieverBuilder( Arrays.asList( diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java index 512874e5009f3..ea251917cfae2 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.retriever.CompoundRetrieverBuilder; import org.elasticsearch.search.retriever.KnnRetrieverBuilder; @@ -21,8 +22,9 @@ import java.util.Arrays; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; public class RRFRetrieverBuilderNestedDocsIT extends RRFRetrieverBuilderIT { @@ -68,7 +70,7 @@ protected void setupIndex() { } } """; - createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).build()); + createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 5)).build()); admin().indices().preparePutMapping(INDEX).setSource(mapping, XContentType.JSON).get(); indexDoc(INDEX, "doc_1", DOC_FIELD, "doc_1", TOPIC_FIELD, "technology", TEXT_FIELD, "term", LAST_30D_FIELD, 100); indexDoc( @@ -134,9 +136,9 @@ public void testRRFRetrieverWithNestedQuery() { final int rankWindowSize = 100; final int rankConstant = 10; SearchSourceBuilder source = new SearchSourceBuilder(); - // this one retrieves docs 1, 4 + // this one retrieves docs 1 StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder( - QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(LAST_30D_FIELD).gte(30L), ScoreMode.Avg) + QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(LAST_30D_FIELD).gte(50L), ScoreMode.Avg) ); // this one retrieves docs 2 and 6 due to prefilter StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( @@ -157,16 +159,21 @@ public void testRRFRetrieverWithNestedQuery() { ) ); source.fetchField(TOPIC_FIELD); + source.explain(true); SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); ElasticsearchAssertions.assertResponse(req, resp -> { assertNull(resp.pointInTimeId()); assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(4L)); + assertThat(resp.getHits().getTotalHits().value, equalTo(3L)); assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6")); - assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_2")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_4")); + assertThat((double) resp.getHits().getAt(0).getScore(), closeTo(0.1742, 1e-4)); + assertThat( + Arrays.stream(resp.getHits().getHits()).skip(1).map(SearchHit::getId).toList(), + containsInAnyOrder("doc_1", "doc_2") + ); + assertThat((double) resp.getHits().getAt(1).getScore(), closeTo(0.0909, 1e-4)); + assertThat((double) resp.getHits().getAt(2).getScore(), closeTo(0.0909, 1e-4)); }); } } diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java index 496af99574431..5f19e361d857d 100644 --- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java +++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java @@ -180,10 +180,7 @@ public void doToXContent(XContentBuilder builder, Params params) throws IOExcept builder.startArray(RETRIEVERS_FIELD.getPreferredName()); for (var entry : innerRetrievers) { - builder.startObject(); - builder.field(entry.retriever().getName()); entry.retriever().toXContent(builder, params); - builder.endObject(); } builder.endArray(); } diff --git a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java index e360237371a82..d324effe41c22 100644 --- a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java +++ b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java @@ -8,19 +8,27 @@ package org.elasticsearch.xpack.rank.rrf; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.retriever.RetrieverBuilder; import org.elasticsearch.search.retriever.RetrieverParserContext; import org.elasticsearch.search.retriever.TestRetrieverBuilder; import org.elasticsearch.test.AbstractXContentTestCase; import org.elasticsearch.usage.SearchUsage; +import org.elasticsearch.usage.SearchUsageHolder; +import org.elasticsearch.usage.UsageService; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + public class RRFRetrieverBuilderParsingTests extends AbstractXContentTestCase { /** @@ -53,7 +61,10 @@ protected RRFRetrieverBuilder createTestInstance() { @Override protected RRFRetrieverBuilder doParseInstance(XContentParser parser) throws IOException { - return RRFRetrieverBuilder.PARSER.apply(parser, new RetrieverParserContext(new SearchUsage(), nf -> true)); + return (RRFRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( + parser, + new RetrieverParserContext(new SearchUsage(), nf -> true) + ); } @Override @@ -81,4 +92,48 @@ protected NamedXContentRegistry xContentRegistry() { ); return new NamedXContentRegistry(entries); } + + public void testRRFRetrieverParsing() throws IOException { + String restContent = "{" + + " \"retriever\": {" + + " \"rrf\": {" + + " \"retrievers\": [" + + " {" + + " \"test\": {" + + " \"value\": \"foo\"" + + " }" + + " }," + + " {" + + " \"test\": {" + + " \"value\": \"bar\"" + + " }" + + " }" + + " ]," + + " \"rank_window_size\": 100," + + " \"rank_constant\": 10," + + " \"min_score\": 20.0," + + " \"_name\": \"foo_rrf\"" + + " }" + + " }" + + "}"; + SearchUsageHolder searchUsageHolder = new UsageService().getSearchUsageHolder(); + try (XContentParser jsonParser = createParser(JsonXContent.jsonXContent, restContent)) { + SearchSourceBuilder source = new SearchSourceBuilder().parseXContent(jsonParser, true, searchUsageHolder, nf -> true); + assertThat(source.retriever(), instanceOf(RRFRetrieverBuilder.class)); + RRFRetrieverBuilder parsed = (RRFRetrieverBuilder) source.retriever(); + assertThat(parsed.minScore(), equalTo(20f)); + assertThat(parsed.retrieverName(), equalTo("foo_rrf")); + try (XContentParser parseSerialized = createParser(JsonXContent.jsonXContent, Strings.toString(source))) { + SearchSourceBuilder deserializedSource = new SearchSourceBuilder().parseXContent( + parseSerialized, + true, + searchUsageHolder, + nf -> true + ); + assertThat(deserializedSource.retriever(), instanceOf(RRFRetrieverBuilder.class)); + RRFRetrieverBuilder deserialized = (RRFRetrieverBuilder) source.retriever(); + assertThat(parsed, equalTo(deserialized)); + } + } + } } diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/350_rrf_retriever_pagination.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/350_rrf_retriever_pagination.yml index 47ba3658bb38d..d5d7a5de1dc71 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/350_rrf_retriever_pagination.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/350_rrf_retriever_pagination.yml @@ -1,6 +1,8 @@ setup: - skip: - features: close_to + features: + - close_to + - contains - requires: cluster_features: 'rrf_retriever_composition_supported' @@ -10,8 +12,6 @@ setup: indices.create: index: test body: - settings: - number_of_shards: 1 mappings: properties: number_val: @@ -81,35 +81,49 @@ setup: bool: { should: [ { - term: { - number_val: { - value: "1", - boost: 10.0 - } - } - }, - { - term: { - number_val: { - value: "2", - boost: 9.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "1" + } + } + }, + boost: 10.0 } - }, - { - term: { - number_val: { - value: "3", - boost: 8.0 - } + },{ + constant_score: { + filter: { + term: { + number_val: { + value: "2" + } + } + }, + boost: 9.0 + } }, + { + constant_score: { + filter: { + term: { + number_val: { + value: "3" + } + } + }, + boost: 8.0 } }, { - term: { - number_val: { - value: "4", - boost: 7.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "4" + } + } + }, + boost: 7.0 } } ] @@ -124,35 +138,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "A", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "B", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "C", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "D", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 7.0 } } ] @@ -198,35 +228,49 @@ setup: bool: { should: [ { - term: { - number_val: { - value: "1", - boost: 10.0 - } - } - }, - { - term: { - number_val: { - value: "2", - boost: 9.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "1" + } + } + }, + boost: 10.0 } - }, - { - term: { - number_val: { - value: "3", - boost: 8.0 - } + },{ + constant_score: { + filter: { + term: { + number_val: { + value: "2" + } + } + }, + boost: 9.0 + } }, + { + constant_score: { + filter: { + term: { + number_val: { + value: "3" + } + } + }, + boost: 8.0 } }, { - term: { - number_val: { - value: "4", - boost: 7.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "4" + } + } + }, + boost: 7.0 } } ] @@ -241,35 +285,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "A", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "B", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "C", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "D", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 7.0 } } ] @@ -306,35 +366,49 @@ setup: bool: { should: [ { - term: { - number_val: { - value: "1", - boost: 10.0 - } - } - }, - { - term: { - number_val: { - value: "2", - boost: 9.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "1" + } + } + }, + boost: 10.0 } - }, - { - term: { - number_val: { - value: "3", - boost: 8.0 - } + },{ + constant_score: { + filter: { + term: { + number_val: { + value: "2" + } + } + }, + boost: 9.0 + } }, + { + constant_score: { + filter: { + term: { + number_val: { + value: "3" + } + } + }, + boost: 8.0 } }, { - term: { - number_val: { - value: "4", - boost: 7.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "4" + } + } + }, + boost: 7.0 } } ] @@ -349,35 +423,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "A", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "B", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "C", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "D", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 7.0 } } ] @@ -422,35 +512,49 @@ setup: bool: { should: [ { - term: { - number_val: { - value: "1", - boost: 10.0 - } - } - }, - { - term: { - number_val: { - value: "2", - boost: 9.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "1" + } + } + }, + boost: 10.0 } - }, - { - term: { - number_val: { - value: "3", - boost: 8.0 - } + },{ + constant_score: { + filter: { + term: { + number_val: { + value: "2" + } + } + }, + boost: 9.0 + } }, + { + constant_score: { + filter: { + term: { + number_val: { + value: "3" + } + } + }, + boost: 8.0 } }, { - term: { - number_val: { - value: "4", - boost: 7.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "4" + } + } + }, + boost: 7.0 } } ] @@ -465,35 +569,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "D", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "C", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "A", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "B", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 7.0 } } ] @@ -533,35 +653,49 @@ setup: bool: { should: [ { - term: { - number_val: { - value: "1", - boost: 10.0 - } - } - }, - { - term: { - number_val: { - value: "2", - boost: 9.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "1" + } + } + }, + boost: 10.0 } - }, - { - term: { - number_val: { - value: "3", - boost: 8.0 - } + },{ + constant_score: { + filter: { + term: { + number_val: { + value: "2" + } + } + }, + boost: 9.0 + } }, + { + constant_score: { + filter: { + term: { + number_val: { + value: "3" + } + } + }, + boost: 8.0 } }, { - term: { - number_val: { - value: "4", - boost: 7.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "4" + } + } + }, + boost: 7.0 } } ] @@ -576,35 +710,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "D", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "C", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "A", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "B", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 7.0 } } ] @@ -632,9 +782,9 @@ setup: "Pagination within interleaved results, different result set sizes, rank_window_size covering all results": # perform multiple searches with different "from" parameter, ensuring that results are consistent # rank_window_size covers the entire result set for both queries, so pagination should be consistent - # queryA has a result set of [5, 1] and + # queryA has a result set of [1] and # queryB has a result set of [4, 3, 1, 2] - # so for rank_constant=10, the expected order is [1, 4, 5, 3, 2] + # so for rank_constant=10, the expected order is [1, 4, 3, 2] - do: search: index: test @@ -645,19 +795,11 @@ setup: { retrievers: [ { - # this should clause would generate the result set [5, 1] + # this should clause would generate the result set [1] standard: { query: { bool: { should: [ - { - term: { - number_val: { - value: "5", - boost: 10.0 - } - } - }, { term: { number_val: { @@ -678,35 +820,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "D", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "C", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "A", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "B", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 7.0 } } ] @@ -721,11 +879,11 @@ setup: from : 0 size : 2 - - match: { hits.total.value : 5 } + - match: { hits.total.value : 4 } - length: { hits.hits : 2 } - match: { hits.hits.0._id: "1" } # score for doc 1 is (1/12 + 1/13) - - close_to: {hits.hits.0._score: {value: 0.1602, error: 0.001}} + - close_to: {hits.hits.0._score: {value: 0.1678, error: 0.001}} - match: { hits.hits.1._id: "4" } # score for doc 4 is (1/11) - close_to: {hits.hits.1._score: {value: 0.0909, error: 0.001}} @@ -740,19 +898,11 @@ setup: { retrievers: [ { - # this should clause would generate the result set [5, 1] + # this should clause would generate the result set [1] standard: { query: { bool: { should: [ - { - term: { - number_val: { - value: "5", - boost: 10.0 - } - } - }, { term: { number_val: { @@ -773,35 +923,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "D", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "C", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "A", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "B", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 7.0 } } ] @@ -816,14 +982,15 @@ setup: from : 2 size : 2 - - match: { hits.total.value : 5 } + - match: { hits.total.value : 4 } - length: { hits.hits : 2 } - - match: { hits.hits.0._id: "5" } - # score for doc 5 is (1/11) - - close_to: {hits.hits.0._score: {value: 0.0909, error: 0.001}} - - match: { hits.hits.1._id: "3" } + - match: { hits.hits.0._id: "3" } # score for doc 3 is (1/12) - - close_to: {hits.hits.1._score: {value: 0.0833, error: 0.001}} + - close_to: {hits.hits.0._score: {value: 0.0833, error: 0.001}} + - match: { hits.hits.1._id: "2" } + # score for doc 2 is (1/14) + - close_to: {hits.hits.1._score: {value: 0.0714, error: 0.001}} + - do: search: @@ -835,19 +1002,11 @@ setup: { retrievers: [ { - # this should clause would generate the result set [5, 1] + # this should clause would generate the result set [1] standard: { query: { bool: { should: [ - { - term: { - number_val: { - value: "5", - boost: 10.0 - } - } - }, { term: { number_val: { @@ -868,35 +1027,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "D", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "C", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "A", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "B", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 7.0 } } ] @@ -911,12 +1086,8 @@ setup: from: 4 size: 2 - - match: { hits.total.value: 5 } - - length: { hits.hits: 1 } - - match: { hits.hits.0._id: "2" } - # score for doc 2 is (1/14) - - close_to: {hits.hits.0._score: {value: 0.0714, error: 0.001}} - + - match: { hits.total.value: 4 } + - length: { hits.hits: 0 } --- "Pagination within interleaved results, different result set sizes, rank_window_size not covering all results": @@ -943,19 +1114,27 @@ setup: bool: { should: [ { - term: { - number_val: { - value: "5", - boost: 10.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "5" + } + } + }, + boost: 10.0 } }, { - term: { - number_val: { - value: "1", - boost: 9.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "1" + } + } + }, + boost: 9.0 } } ] @@ -970,35 +1149,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "D", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "C", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "A", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "B", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 7.0 } } ] @@ -1015,11 +1210,11 @@ setup: - match: { hits.total.value : 5 } - length: { hits.hits : 2 } - - match: { hits.hits.0._id: "4" } - # score for doc 4 is (1/11) + - contains: { hits.hits: { _id: "4" } } + - contains: { hits.hits: { _id: "5" } } + + # both docs have the same score (1/11) - close_to: {hits.hits.0._score: {value: 0.0909, error: 0.001}} - - match: { hits.hits.1._id: "5" } - # score for doc 5 is (1/11) - close_to: {hits.hits.1._score: {value: 0.0909, error: 0.001}} - do: @@ -1038,19 +1233,27 @@ setup: bool: { should: [ { - term: { - number_val: { - value: "5", - boost: 10.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "5" + } + } + }, + boost: 10.0 } }, { - term: { - number_val: { - value: "1", - boost: 9.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "1" + } + } + }, + boost: 9.0 } } ] @@ -1065,35 +1268,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "D", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "C", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "A", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "B", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 7.0 } } ] diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml index 1f7125377b892..517c162c33e95 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml @@ -1,7 +1,6 @@ setup: - skip: features: close_to - - requires: cluster_features: 'rrf_retriever_composition_supported' reason: 'test requires rrf retriever composition support' @@ -10,8 +9,6 @@ setup: indices.create: index: test body: - settings: - number_of_shards: 1 mappings: properties: text: @@ -42,7 +39,7 @@ setup: index: test id: "1" body: - text: "term term term term term term term term term" + text: "term1" vector: [1.0] - do: @@ -50,7 +47,7 @@ setup: index: test id: "2" body: - text: "term term term term term term term term" + text: "term2" text_to_highlight: "search for the truth" keyword: "biology" vector: [2.0] @@ -60,8 +57,8 @@ setup: index: test id: "3" body: - text: "term term term term term term term" - text_to_highlight: "nothing related but still a match" + text: "term3" + text_to_highlight: "nothing related" keyword: "technology" vector: [3.0] @@ -70,14 +67,14 @@ setup: index: test id: "4" body: - text: "term term term term term term" + text: "term4" vector: [4.0] - do: index: index: test id: "5" body: - text: "term term term term term" + text: "term5" text_to_highlight: "You know, for Search!" keyword: "technology" integer: 5 @@ -87,7 +84,7 @@ setup: index: test id: "6" body: - text: "term term term term" + text: "term6" keyword: "biology" integer: 6 vector: [6.0] @@ -96,27 +93,26 @@ setup: index: test id: "7" body: - text: "term term term" + text: "term7" keyword: "astronomy" - vector: [7.0] + vector: [77.0] nested: { views: 50} - do: index: index: test id: "8" body: - text: "term term" + text: "term8" keyword: "technology" - vector: [8.0] nested: { views: 100} - do: index: index: test id: "9" body: - text: "term" + text: "term9" + integer: 2 keyword: "technology" - vector: [9.0] nested: { views: 10} - do: indices.refresh: {} @@ -133,6 +129,7 @@ setup: rrf: retrievers: [ { + # this one retrieves docs 6, 5, 4 knn: { field: vector, query_vector: [ 6.0 ], @@ -141,10 +138,72 @@ setup: } }, { + # this one retrieves docs 4, 5, 1, 2, 6 standard: { query: { - term: { - text: term + bool: { + should: [ + { + constant_score: { + filter: { + term: { + text: term4 + } + }, + boost: 10.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term5 + } + }, + boost: 9.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term1 + } + }, + boost: 8.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term2 + } + }, + boost: 7.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term6 + } + }, + boost: 6.0 + } + }, + { + constant_score: { + filter: { + exists: { + field: text + } + }, + boost: 1 + } + } + ] } } } @@ -158,9 +217,13 @@ setup: terms: field: keyword - - match: { hits.hits.0._id: "5" } - - match: { hits.hits.1._id: "1" } + + - match: { hits.hits.0._id: "4" } + - close_to: { hits.hits.0._score: { value: 0.1678, error: 0.001 } } + - match: { hits.hits.1._id: "5" } + - close_to: { hits.hits.1._score: { value: 0.1666, error: 0.001 } } - match: { hits.hits.2._id: "6" } + - close_to: { hits.hits.2._score: { value: 0.1575, error: 0.001 } } - match: { aggregations.keyword_aggs.buckets.0.key: "technology" } - match: { aggregations.keyword_aggs.buckets.0.doc_count: 4 } @@ -181,6 +244,7 @@ setup: rrf: retrievers: [ { + # this one retrieves docs 6, 5, 4 knn: { field: vector, query_vector: [ 6.0 ], @@ -189,10 +253,72 @@ setup: } }, { + # this one retrieves docs 4, 5, 1, 2, 6 standard: { query: { - term: { - text: term + bool: { + should: [ + { + constant_score: { + filter: { + term: { + text: term4 + } + }, + boost: 10.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term5 + } + }, + boost: 9.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term1 + } + }, + boost: 8.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term2 + } + }, + boost: 7.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term6 + } + }, + boost: 6.0 + } + }, + { + constant_score: { + filter: { + exists: { + field: text + } + }, + boost: 1 + } + } + ] } } } @@ -208,12 +334,14 @@ setup: lang: painless source: "_score" - - - match: { hits.hits.0._id: "5" } - - match: { hits.hits.1._id: "1" } + - match: { hits.hits.0._id: "4" } + - close_to: { hits.hits.0._score: { value: 0.1678, error: 0.001 } } + - match: { hits.hits.1._id: "5" } + - close_to: { hits.hits.1._score: { value: 0.1666, error: 0.001 } } - match: { hits.hits.2._id: "6" } + - close_to: { hits.hits.2._score: { value: 0.1575, error: 0.001 } } - - close_to: { aggregations.max_score.value: { value: 0.15, error: 0.001 }} + - close_to: { aggregations.max_score.value: { value: 0.1678, error: 0.001 }} --- "rrf retriever with top-level collapse": @@ -228,6 +356,7 @@ setup: rrf: retrievers: [ { + # this one retrieves docs 6, 5, 4 knn: { field: vector, query_vector: [ 6.0 ], @@ -236,10 +365,72 @@ setup: } }, { + # this one retrieves docs 4, 5, 1, 2, 6 standard: { query: { - term: { - text: term + bool: { + should: [ + { + constant_score: { + filter: { + term: { + text: term4 + } + }, + boost: 10.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term5 + } + }, + boost: 9.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term1 + } + }, + boost: 8.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term2 + } + }, + boost: 7.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term6 + } + }, + boost: 6.0 + } + }, + { + constant_score: { + filter: { + exists: { + field: text + } + }, + boost: 1 + } + } + ] } } } @@ -250,18 +441,23 @@ setup: size: 3 collapse: { field: keyword, inner_hits: { name: sub_hits, size: 2 } } - - match: { hits.hits.0._id: "5" } - - match: { hits.hits.1._id: "1" } + - match: { hits.total : 9 } + + - match: { hits.hits.0._id: "4" } + - close_to: { hits.hits.0._score: { value: 0.1678, error: 0.001 } } + - match: { hits.hits.1._id: "5" } + - close_to: { hits.hits.1._score: { value: 0.1666, error: 0.001 } } - match: { hits.hits.2._id: "6" } + - close_to: { hits.hits.2._score: { value: 0.1575, error: 0.001 } } - - match: { hits.hits.0.inner_hits.sub_hits.hits.total : 4 } - length: { hits.hits.0.inner_hits.sub_hits.hits.hits : 2 } - - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "5" } - - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.1._id: "3" } + - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "4" } + - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.1._id: "1" } + - match: { hits.hits.1.inner_hits.sub_hits.hits.total : 4 } - length: { hits.hits.1.inner_hits.sub_hits.hits.hits : 2 } - - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "1" } - - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "4" } + - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "5" } + - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "3" } - length: { hits.hits.2.inner_hits.sub_hits.hits.hits: 2 } - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "6" } @@ -280,18 +476,132 @@ setup: rrf: retrievers: [ { - knn: { - field: vector, - query_vector: [ 6.0 ], - k: 3, - num_candidates: 10 + # this one retrieves docs 7, 3 + standard: { + query: { + bool: { + should: [ + { + constant_score: { + filter: { + term: { + text: term7 + } + }, + boost: 10.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term3 + } }, + boost: 9.0 + } + } + ] + } + } } }, { + # this one retrieves docs 1, 2, 3, 7 standard: { query: { - term: { - text: term + bool: { + should: [ + { + constant_score: { + filter: { + term: { + text: term1 + } + }, + boost: 10.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term2 + } + }, + boost: 9.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term3 + } + }, + boost: 8.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term4 + } + }, + boost: 7.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term5 + } + }, + boost: 6.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term6 + } + }, + boost: 5.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term7 + } + }, + boost: 4.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term8 + } + }, + boost: 3.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term9 + } + }, + boost: 2.0 + } + } + ] } }, collapse: { field: keyword, inner_hits: { name: sub_hits, size: 1 } } @@ -303,8 +613,9 @@ setup: size: 3 - match: { hits.hits.0._id: "7" } - - match: { hits.hits.1._id: "1" } - - match: { hits.hits.2._id: "6" } + - close_to: { hits.hits.0._score: { value: 0.1623, error: 0.001 } } + - match: { hits.hits.1._id: "3" } + - close_to: { hits.hits.1._score: { value: 0.1602, error: 0.001 } } --- "rrf retriever highlighting results": @@ -331,7 +642,7 @@ setup: standard: { query: { term: { - keyword: technology + text: term5 } } } @@ -349,7 +660,7 @@ setup: } } - - match: { hits.total : 5 } + - match: { hits.total : 2 } - match: { hits.hits.0._id: "5" } - match: { hits.hits.0.highlight.text_to_highlight.0: "You know, for Search!" } @@ -357,9 +668,6 @@ setup: - match: { hits.hits.1._id: "2" } - match: { hits.hits.1.highlight.text_to_highlight.0: "search for the truth" } - - match: { hits.hits.2._id: "3" } - - not_exists: hits.hits.2.highlight - --- "rrf retriever with custom nested sort": @@ -374,12 +682,103 @@ setup: retrievers: [ { # this one retrievers docs 1, 2, 3, .., 9 - # but due to sorting, it will revert the order to 6, 5, .., 9 which due to + # but due to sorting, it will revert the order to 6, 5, 9, ... which due to # rank_window_size: 2 will only return 6 and 5 standard: { query: { - term: { - text: term + bool: { + should: [ + { + constant_score: { + filter: { + term: { + text: term1 + } + }, + boost: 10.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term2 + } + }, + boost: 9.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term3 + } + }, + boost: 8.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term4 + } + }, + boost: 7.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term5 + } + }, + boost: 6.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term6 + } + }, + boost: 5.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term7 + } + }, + boost: 4.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term8 + } + }, + boost: 3.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term9 + } + }, + boost: 2.0 + } + } + ] } }, sort: [ @@ -410,7 +809,6 @@ setup: - length: {hits.hits: 2 } - match: { hits.hits.0._id: "6" } - - match: { hits.hits.1._id: "2" } --- "rrf retriever with nested query": @@ -427,7 +825,7 @@ setup: { knn: { field: vector, - query_vector: [ 7.0 ], + query_vector: [ 77.0 ], k: 1, num_candidates: 3 } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java index 38dd7116acce4..19c18bf855b4e 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java @@ -34,6 +34,7 @@ import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingAction; import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest; import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequestBuilder; +import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper; import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; @@ -58,12 +59,11 @@ import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING; import static org.elasticsearch.xcontent.XContentType.JSON; import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7; +import static org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper.RESERVED_ROLE_MAPPING_SUFFIX; import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.notNullValue; @@ -270,21 +270,28 @@ private void assertRoleMappingsSaveOK(CountDownLatch savedClusterState, AtomicLo assertThat(resolveRolesFuture.get(), containsInAnyOrder("kibana_user", "fleet_user")); } - // the role mappings are not retrievable by the role mapping action (which only accesses "native" i.e. index-based role mappings) - var request = new GetRoleMappingsRequest(); - request.setNames("everyone_kibana", "everyone_fleet"); - var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get(); - assertFalse(response.hasMappings()); - assertThat(response.mappings(), emptyArray()); - - // role mappings (with the same names) can also be stored in the "native" store - var putRoleMappingResponse = client().execute(PutRoleMappingAction.INSTANCE, sampleRestRequest("everyone_kibana")).actionGet(); - assertTrue(putRoleMappingResponse.isCreated()); - putRoleMappingResponse = client().execute(PutRoleMappingAction.INSTANCE, sampleRestRequest("everyone_fleet")).actionGet(); - assertTrue(putRoleMappingResponse.isCreated()); + // the role mappings are retrievable by the role mapping action for BWC + assertGetResponseHasMappings(true, "everyone_kibana", "everyone_fleet"); + + // role mappings (with the same names) can be stored in the "native" store + { + PutRoleMappingResponse response = client().execute(PutRoleMappingAction.INSTANCE, sampleRestRequest("everyone_kibana")) + .actionGet(); + assertTrue(response.isCreated()); + response = client().execute(PutRoleMappingAction.INSTANCE, sampleRestRequest("everyone_fleet")).actionGet(); + assertTrue(response.isCreated()); + } + { + // deleting role mappings that exist in the native store and in cluster-state should result in success + var response = client().execute(DeleteRoleMappingAction.INSTANCE, deleteRequest("everyone_kibana")).actionGet(); + assertTrue(response.isFound()); + response = client().execute(DeleteRoleMappingAction.INSTANCE, deleteRequest("everyone_fleet")).actionGet(); + assertTrue(response.isFound()); + } + } - public void testRoleMappingsApplied() throws Exception { + public void testClusterStateRoleMappingsAddedThenDeleted() throws Exception { ensureGreen(); var savedClusterState = setupClusterStateListener(internalCluster().getMasterName(), "everyone_kibana"); @@ -293,6 +300,12 @@ public void testRoleMappingsApplied() throws Exception { assertRoleMappingsSaveOK(savedClusterState.v1(), savedClusterState.v2()); logger.info("---> cleanup cluster settings..."); + { + // Deleting non-existent native role mappings returns not found even if they exist in config file + var response = client().execute(DeleteRoleMappingAction.INSTANCE, deleteRequest("everyone_kibana")).get(); + assertFalse(response.isFound()); + } + savedClusterState = setupClusterStateListenerForCleanup(internalCluster().getMasterName()); writeJSONFile(internalCluster().getMasterName(), emptyJSON, logger, versionCounter); @@ -307,48 +320,95 @@ public void testRoleMappingsApplied() throws Exception { clusterStateResponse.getState().metadata().persistentSettings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()) ); - // native role mappings are not affected by the removal of the cluster-state based ones + // cluster-state role mapping was removed and is not returned in the API anymore { var request = new GetRoleMappingsRequest(); request.setNames("everyone_kibana", "everyone_fleet"); var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get(); - assertTrue(response.hasMappings()); - assertThat( - Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(), - containsInAnyOrder("everyone_kibana", "everyone_fleet") - ); + assertFalse(response.hasMappings()); } - // and roles are resolved based on the native role mappings + // no role mappings means no roles are resolved for (UserRoleMapper userRoleMapper : internalCluster().getInstances(UserRoleMapper.class)) { PlainActionFuture> resolveRolesFuture = new PlainActionFuture<>(); userRoleMapper.resolveRoles( new UserRoleMapper.UserData("anyUsername", null, List.of(), Map.of(), mock(RealmConfig.class)), resolveRolesFuture ); - assertThat(resolveRolesFuture.get(), contains("kibana_user_native")); + assertThat(resolveRolesFuture.get(), empty()); } + } - { - var request = new DeleteRoleMappingRequest(); - request.setName("everyone_kibana"); - var response = client().execute(DeleteRoleMappingAction.INSTANCE, request).get(); - assertTrue(response.isFound()); - request = new DeleteRoleMappingRequest(); - request.setName("everyone_fleet"); - response = client().execute(DeleteRoleMappingAction.INSTANCE, request).get(); - assertTrue(response.isFound()); + public void testGetRoleMappings() throws Exception { + ensureGreen(); + + final List nativeMappings = List.of("everyone_kibana", "_everyone_kibana", "zzz_mapping", "123_mapping"); + for (var mapping : nativeMappings) { + client().execute(PutRoleMappingAction.INSTANCE, sampleRestRequest(mapping)).actionGet(); } - // no roles are resolved now, because both native and cluster-state based stores have been cleared - for (UserRoleMapper userRoleMapper : internalCluster().getInstances(UserRoleMapper.class)) { - PlainActionFuture> resolveRolesFuture = new PlainActionFuture<>(); - userRoleMapper.resolveRoles( - new UserRoleMapper.UserData("anyUsername", null, List.of(), Map.of(), mock(RealmConfig.class)), - resolveRolesFuture - ); - assertThat(resolveRolesFuture.get(), empty()); + var savedClusterState = setupClusterStateListener(internalCluster().getMasterName(), "everyone_kibana"); + writeJSONFile(internalCluster().getMasterName(), testJSON, logger, versionCounter); + boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS); + assertTrue(awaitSuccessful); + + var request = new GetRoleMappingsRequest(); + var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get(); + assertTrue(response.hasMappings()); + assertThat( + Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(), + containsInAnyOrder( + "everyone_kibana", + "everyone_kibana" + RESERVED_ROLE_MAPPING_SUFFIX, + "_everyone_kibana", + "everyone_fleet" + RESERVED_ROLE_MAPPING_SUFFIX, + "zzz_mapping", + "123_mapping" + ) + ); + + int readOnlyCount = 0; + // assert that cluster-state role mappings come last + for (ExpressionRoleMapping mapping : response.mappings()) { + readOnlyCount = mapping.getName().endsWith(RESERVED_ROLE_MAPPING_SUFFIX) ? readOnlyCount + 1 : readOnlyCount; } + // Two sourced from cluster-state + assertEquals(readOnlyCount, 2); + + // it's possible to delete overlapping native role mapping + assertTrue(client().execute(DeleteRoleMappingAction.INSTANCE, deleteRequest("everyone_kibana")).actionGet().isFound()); + + // Fetch a specific file based role + request = new GetRoleMappingsRequest(); + request.setNames("everyone_kibana" + RESERVED_ROLE_MAPPING_SUFFIX); + response = client().execute(GetRoleMappingsAction.INSTANCE, request).get(); + assertTrue(response.hasMappings()); + assertThat( + Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(), + containsInAnyOrder("everyone_kibana" + RESERVED_ROLE_MAPPING_SUFFIX) + ); + + savedClusterState = setupClusterStateListenerForCleanup(internalCluster().getMasterName()); + writeJSONFile(internalCluster().getMasterName(), emptyJSON, logger, versionCounter); + awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS); + assertTrue(awaitSuccessful); + + final ClusterStateResponse clusterStateResponse = clusterAdmin().state( + new ClusterStateRequest(TEST_REQUEST_TIMEOUT).waitForMetadataVersion(savedClusterState.v2().get()) + ).get(); + + assertNull( + clusterStateResponse.getState().metadata().persistentSettings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()) + ); + + // Make sure remaining native mappings can still be fetched + request = new GetRoleMappingsRequest(); + response = client().execute(GetRoleMappingsAction.INSTANCE, request).get(); + assertTrue(response.hasMappings()); + assertThat( + Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(), + containsInAnyOrder("_everyone_kibana", "zzz_mapping", "123_mapping") + ); } public static Tuple setupClusterStateListenerForError( @@ -433,11 +493,8 @@ public void testRoleMappingApplyWithSecurityIndexClosed() throws Exception { boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS); assertTrue(awaitSuccessful); - // no native role mappings exist - var request = new GetRoleMappingsRequest(); - request.setNames("everyone_kibana", "everyone_fleet"); - var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get(); - assertFalse(response.hasMappings()); + // even if index is closed, cluster-state role mappings are still returned + assertGetResponseHasMappings(true, "everyone_kibana", "everyone_fleet"); // cluster state settings are also applied var clusterStateResponse = clusterAdmin().state( @@ -476,6 +533,12 @@ public void testRoleMappingApplyWithSecurityIndexClosed() throws Exception { } } + private DeleteRoleMappingRequest deleteRequest(String name) { + var request = new DeleteRoleMappingRequest(); + request.setName(name); + return request; + } + private PutRoleMappingRequest sampleRestRequest(String name) throws Exception { var json = """ { @@ -494,4 +557,17 @@ private PutRoleMappingRequest sampleRestRequest(String name) throws Exception { return new PutRoleMappingRequestBuilder(null).source(name, parser).request(); } } + + private static void assertGetResponseHasMappings(boolean readOnly, String... mappings) throws InterruptedException, ExecutionException { + var request = new GetRoleMappingsRequest(); + request.setNames(mappings); + var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get(); + assertTrue(response.hasMappings()); + assertThat( + Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(), + containsInAnyOrder( + Arrays.stream(mappings).map(mapping -> mapping + (readOnly ? RESERVED_ROLE_MAPPING_SUFFIX : "")).toArray(String[]::new) + ) + ); + } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 79a00fa1293bd..f4d9360d1ed84 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -897,7 +897,8 @@ Collection createComponents( reservedRealm ); components.add(nativeUsersStore); - components.add(new PluginComponentBinding<>(NativeRoleMappingStore.class, nativeRoleMappingStore)); + components.add(clusterStateRoleMapper); + components.add(nativeRoleMappingStore); components.add(new PluginComponentBinding<>(UserRoleMapper.class, userRoleMapper)); components.add(reservedRealm); components.add(realms); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleAction.java index e8d248233415c..569cdc1a79fd9 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleAction.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; @@ -17,6 +18,7 @@ import org.elasticsearch.xpack.core.security.action.role.DeleteRoleAction; import org.elasticsearch.xpack.core.security.action.role.DeleteRoleRequest; import org.elasticsearch.xpack.core.security.action.role.DeleteRoleResponse; +import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper; import org.elasticsearch.xpack.security.authz.ReservedRoleNameChecker; import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; @@ -25,16 +27,20 @@ public class TransportDeleteRoleAction extends TransportAction { + if (clusterStateRoleMapper.hasMapping(request.name())) { + // Allow to delete a mapping with the same name in the native role mapping store as the file_settings namespace, but + // add a warning header to signal to the caller that this could be a problem. + HeaderWarning.addWarning( + "A read only role mapping with the same name [" + + request.name() + + "] has been previously been defined in a configuration file. " + + "The read only role mapping will still be active." + ); + } + return new DeleteRoleResponse(found); + })); } catch (Exception e) { logger.error((Supplier) () -> "failed to delete role [" + request.name() + "]", e); listener.onFailure(e); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportDeleteRoleMappingAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportDeleteRoleMappingAction.java index 74129facae70a..467cc1c8a9027 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportDeleteRoleMappingAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportDeleteRoleMappingAction.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; @@ -16,17 +17,20 @@ import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingAction; import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequest; import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingResponse; +import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper; import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore; public class TransportDeleteRoleMappingAction extends HandledTransportAction { private final NativeRoleMappingStore roleMappingStore; + private final ClusterStateRoleMapper clusterStateRoleMapper; @Inject public TransportDeleteRoleMappingAction( ActionFilters actionFilters, TransportService transportService, - NativeRoleMappingStore roleMappingStore + NativeRoleMappingStore roleMappingStore, + ClusterStateRoleMapper clusterStateRoleMapper ) { super( DeleteRoleMappingAction.NAME, @@ -36,10 +40,22 @@ public TransportDeleteRoleMappingAction( EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.roleMappingStore = roleMappingStore; + this.clusterStateRoleMapper = clusterStateRoleMapper; } @Override protected void doExecute(Task task, DeleteRoleMappingRequest request, ActionListener listener) { + if (clusterStateRoleMapper.hasMapping(request.getName())) { + // Since it's allowed to add a mapping with the same name in the native role mapping store as the file_settings namespace, + // a warning header is added to signal to the caller that this could be a problem. + HeaderWarning.addWarning( + "A read only role mapping with the same name [" + + request.getName() + + "] has been previously been defined in a configuration file. The role mapping [" + + request.getName() + + "] defined in the configuration file is read only, will not be deleted, and will remain active." + ); + } roleMappingStore.deleteRoleMapping(request, listener.safeMap(DeleteRoleMappingResponse::new)); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java index ac0d3177cca09..db0ee01af70e4 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java @@ -17,21 +17,30 @@ import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequest; import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsResponse; import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; +import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper; import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore; import java.util.Arrays; +import java.util.Comparator; import java.util.HashSet; +import java.util.List; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper.RESERVED_ROLE_MAPPING_SUFFIX; public class TransportGetRoleMappingsAction extends HandledTransportAction { private final NativeRoleMappingStore roleMappingStore; + private final ClusterStateRoleMapper clusterStateRoleMapper; @Inject public TransportGetRoleMappingsAction( ActionFilters actionFilters, TransportService transportService, - NativeRoleMappingStore nativeRoleMappingStore + NativeRoleMappingStore nativeRoleMappingStore, + ClusterStateRoleMapper clusterStateRoleMapper ) { super( GetRoleMappingsAction.NAME, @@ -41,6 +50,7 @@ public TransportGetRoleMappingsAction( EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.roleMappingStore = nativeRoleMappingStore; + this.clusterStateRoleMapper = clusterStateRoleMapper; } @Override @@ -51,9 +61,32 @@ protected void doExecute(Task task, final GetRoleMappingsRequest request, final } else { names = new HashSet<>(Arrays.asList(request.getNames())); } - this.roleMappingStore.getRoleMappings(names, ActionListener.wrap(mappings -> { - ExpressionRoleMapping[] array = mappings.toArray(new ExpressionRoleMapping[mappings.size()]); - listener.onResponse(new GetRoleMappingsResponse(array)); + roleMappingStore.getRoleMappings(names, ActionListener.wrap(mappings -> { + List combinedRoleMappings = Stream.concat( + mappings.stream(), + clusterStateRoleMapper.getMappings(names == null ? null : names.stream().map(name -> { + // If a read-only role is fetched by name including suffix, remove suffix + return name.endsWith(RESERVED_ROLE_MAPPING_SUFFIX) + ? name.substring(0, name.length() - RESERVED_ROLE_MAPPING_SUFFIX.length()) + : name; + }).collect(Collectors.toSet())) + .stream() + .map(this::cloneAndMarkAsReadOnly) + .sorted(Comparator.comparing(ExpressionRoleMapping::getName)) + ).toList(); + listener.onResponse(new GetRoleMappingsResponse(combinedRoleMappings)); }, listener::onFailure)); } + + private ExpressionRoleMapping cloneAndMarkAsReadOnly(ExpressionRoleMapping mapping) { + // Mark role mappings from cluster state as "read only" by adding a suffix to their name + return new ExpressionRoleMapping( + mapping.getName() + RESERVED_ROLE_MAPPING_SUFFIX, + mapping.getExpression(), + mapping.getRoles(), + mapping.getRoleTemplates(), + mapping.getMetadata(), + mapping.isEnabled() + ); + } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingAction.java index 82a3b4f000064..76f520bed517e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingAction.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; @@ -16,27 +17,52 @@ import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingAction; import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest; import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse; +import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper; import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore; +import static org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper.RESERVED_ROLE_MAPPING_SUFFIX; + public class TransportPutRoleMappingAction extends HandledTransportAction { private final NativeRoleMappingStore roleMappingStore; + private final ClusterStateRoleMapper clusterStateRoleMapper; @Inject public TransportPutRoleMappingAction( ActionFilters actionFilters, TransportService transportService, - NativeRoleMappingStore roleMappingStore + NativeRoleMappingStore roleMappingStore, + ClusterStateRoleMapper clusterStateRoleMapper ) { super(PutRoleMappingAction.NAME, transportService, actionFilters, PutRoleMappingRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); this.roleMappingStore = roleMappingStore; + this.clusterStateRoleMapper = clusterStateRoleMapper; } @Override protected void doExecute(Task task, final PutRoleMappingRequest request, final ActionListener listener) { + validateMappingName(request.getName()); + if (clusterStateRoleMapper.hasMapping(request.getName())) { + // Allow to define a mapping with the same name in the native role mapping store as the file_settings namespace, but add a + // warning header to signal to the caller that this could be a problem. + HeaderWarning.addWarning( + "A read only role mapping with the same name [" + + request.getName() + + "] has been previously been defined in a configuration file. " + + "Both role mappings will be used to determine role assignments." + ); + } roleMappingStore.putRoleMapping( request, ActionListener.wrap(created -> listener.onResponse(new PutRoleMappingResponse(created)), listener::onFailure) ); } + + private static void validateMappingName(String mappingName) { + if (mappingName.endsWith(RESERVED_ROLE_MAPPING_SUFFIX)) { + throw new IllegalArgumentException( + "Invalid mapping name [" + mappingName + "]. [" + RESERVED_ROLE_MAPPING_SUFFIX + "] is not an allowed suffix" + ); + } + } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapper.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapper.java index 9a6e9e75c4685..baea5970b4637 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapper.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapper.java @@ -14,13 +14,16 @@ import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Nullable; import org.elasticsearch.script.ScriptService; import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper; import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata; +import java.util.Arrays; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import static org.elasticsearch.xpack.core.security.SecurityExtension.SecurityComponents; @@ -28,8 +31,7 @@ * A role mapper the reads the role mapping rules (i.e. {@link ExpressionRoleMapping}s) from the cluster state * (i.e. {@link RoleMappingMetadata}). This is not enabled by default. */ -public final class ClusterStateRoleMapper extends AbstractRoleMapperClearRealmCache implements ClusterStateListener { - +public class ClusterStateRoleMapper extends AbstractRoleMapperClearRealmCache implements ClusterStateListener { /** * This setting is never registered by the xpack security plugin - in order to enable the * cluster-state based role mapper another plugin must register it as a boolean setting @@ -45,6 +47,7 @@ public final class ClusterStateRoleMapper extends AbstractRoleMapperClearRealmCa * */ public static final String CLUSTER_STATE_ROLE_MAPPINGS_ENABLED = "xpack.security.authc.cluster_state_role_mappings.enabled"; + public static final String RESERVED_ROLE_MAPPING_SUFFIX = "-read-only-operator-config"; private static final Logger logger = LogManager.getLogger(ClusterStateRoleMapper.class); private final ScriptService scriptService; @@ -54,8 +57,8 @@ public final class ClusterStateRoleMapper extends AbstractRoleMapperClearRealmCa public ClusterStateRoleMapper(Settings settings, ScriptService scriptService, ClusterService clusterService) { this.scriptService = scriptService; this.clusterService = clusterService; - // this role mapper is disabled by default and only code in other plugins can enable it - this.enabled = settings.getAsBoolean(CLUSTER_STATE_ROLE_MAPPINGS_ENABLED, false); + // this role mapper is enabled by default and only code in other plugins can disable it + this.enabled = settings.getAsBoolean(CLUSTER_STATE_ROLE_MAPPINGS_ENABLED, true); if (this.enabled) { clusterService.addListener(this); } @@ -81,12 +84,30 @@ public void clusterChanged(ClusterChangedEvent event) { } } + public boolean hasMapping(String name) { + return getMappings().stream().map(ExpressionRoleMapping::getName).anyMatch(name::equals); + } + + public Set getMappings(@Nullable Set names) { + if (enabled == false) { + return Set.of(); + } + final Set mappings = getMappings(); + if (names == null || names.isEmpty()) { + return mappings; + } + return mappings.stream().filter(it -> names.contains(it.getName())).collect(Collectors.toSet()); + } + private Set getMappings() { if (enabled == false) { return Set.of(); } else { final Set mappings = RoleMappingMetadata.getFromClusterState(clusterService.state()).getRoleMappings(); - logger.trace("Retrieved [{}] mapping(s) from cluster state", mappings.size()); + logger.trace( + "Retrieved mapping(s) {} from cluster state", + Arrays.toString(mappings.stream().map(ExpressionRoleMapping::getName).toArray(String[]::new)) + ); return mappings; } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java index 6d7817db8ec05..ce5aaacdb92b9 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java @@ -403,7 +403,7 @@ public static class UnregisteredSecuritySettingsPlugin extends Plugin { ); public static final Setting CLUSTER_STATE_ROLE_MAPPINGS_ENABLED = Setting.boolSetting( "xpack.security.authc.cluster_state_role_mappings.enabled", - false, + true, Setting.Property.NodeScope ); public static final Setting NATIVE_ROLES_ENABLED = Setting.boolSetting( diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleActionTests.java index 84e4dc402c767..d647088017dc1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.core.security.action.role.DeleteRoleResponse; import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; +import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper; import org.elasticsearch.xpack.security.authz.ReservedRoleNameChecker; import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; import org.junit.BeforeClass; @@ -66,7 +67,8 @@ public void testReservedRole() { mock(ActionFilters.class), rolesStore, transportService, - new ReservedRoleNameChecker.Default() + new ReservedRoleNameChecker.Default(), + mock(ClusterStateRoleMapper.class) ); DeleteRoleRequest request = new DeleteRoleRequest(); @@ -115,7 +117,8 @@ private void testValidRole(String roleName) { mock(ActionFilters.class), rolesStore, transportService, - new ReservedRoleNameChecker.Default() + new ReservedRoleNameChecker.Default(), + mock(ClusterStateRoleMapper.class) ); DeleteRoleRequest request = new DeleteRoleRequest(); @@ -168,7 +171,8 @@ public void testException() { mock(ActionFilters.class), rolesStore, transportService, - new ReservedRoleNameChecker.Default() + new ReservedRoleNameChecker.Default(), + mock(ClusterStateRoleMapper.class) ); DeleteRoleRequest request = new DeleteRoleRequest(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsActionTests.java index 6e8698f095d32..799e0c334172c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequest; import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsResponse; import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; +import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper; import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore; import org.hamcrest.Matchers; import org.junit.Before; @@ -34,13 +35,16 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.notNullValue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TransportGetRoleMappingsActionTests extends ESTestCase { private NativeRoleMappingStore store; + private ClusterStateRoleMapper clusterStateRoleMapper; private TransportGetRoleMappingsAction action; private AtomicReference> namesRef; private List result; @@ -49,6 +53,8 @@ public class TransportGetRoleMappingsActionTests extends ESTestCase { @Before public void setupMocks() { store = mock(NativeRoleMappingStore.class); + clusterStateRoleMapper = mock(ClusterStateRoleMapper.class); + when(clusterStateRoleMapper.getMappings(anySet())).thenReturn(Set.of()); TransportService transportService = new TransportService( Settings.EMPTY, mock(Transport.class), @@ -58,7 +64,7 @@ public void setupMocks() { null, Collections.emptySet() ); - action = new TransportGetRoleMappingsAction(mock(ActionFilters.class), transportService, store); + action = new TransportGetRoleMappingsAction(mock(ActionFilters.class), transportService, store, clusterStateRoleMapper); namesRef = new AtomicReference<>(null); result = Collections.emptyList(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingActionTests.java index 6f789a10a3a6c..0bb3e7dd4ac3e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingActionTests.java @@ -19,26 +19,32 @@ import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse; import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression; +import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper; import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore; import org.junit.Before; import java.util.Arrays; import java.util.Collections; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import static org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper.RESERVED_ROLE_MAPPING_SUFFIX; import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.iterableWithSize; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TransportPutRoleMappingActionTests extends ESTestCase { private NativeRoleMappingStore store; + private ClusterStateRoleMapper clusterStateRoleMapper; private TransportPutRoleMappingAction action; private AtomicReference requestRef; @@ -46,6 +52,9 @@ public class TransportPutRoleMappingActionTests extends ESTestCase { @Before public void setupMocks() { store = mock(NativeRoleMappingStore.class); + clusterStateRoleMapper = mock(ClusterStateRoleMapper.class); + when(clusterStateRoleMapper.getMappings(anySet())).thenReturn(Set.of()); + when(clusterStateRoleMapper.hasMapping(any())).thenReturn(false); TransportService transportService = new TransportService( Settings.EMPTY, mock(Transport.class), @@ -55,7 +64,7 @@ public void setupMocks() { null, Collections.emptySet() ); - action = new TransportPutRoleMappingAction(mock(ActionFilters.class), transportService, store); + action = new TransportPutRoleMappingAction(mock(ActionFilters.class), transportService, store, clusterStateRoleMapper); requestRef = new AtomicReference<>(null); @@ -85,6 +94,25 @@ public void testPutValidMapping() throws Exception { assertThat(mapping.getMetadata().get("dumb"), equalTo(true)); } + public void testPutMappingWithInvalidName() { + final FieldExpression expression = new FieldExpression("username", Collections.singletonList(new FieldExpression.FieldValue("*"))); + IllegalArgumentException illegalArgumentException = expectThrows( + IllegalArgumentException.class, + () -> put("anarchy" + RESERVED_ROLE_MAPPING_SUFFIX, expression, "superuser", Collections.singletonMap("dumb", true)) + ); + + assertThat( + illegalArgumentException.getMessage(), + equalTo( + "Invalid mapping name [anarchy" + + RESERVED_ROLE_MAPPING_SUFFIX + + "]. [" + + RESERVED_ROLE_MAPPING_SUFFIX + + "] is not an allowed suffix" + ) + ); + } + private PutRoleMappingResponse put(String name, FieldExpression expression, String role, Map metadata) throws Exception { final PutRoleMappingRequest request = new PutRoleMappingRequest(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapperTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapperTests.java index 7a9dd65f84c67..515b5ef741a00 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapperTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapperTests.java @@ -56,12 +56,12 @@ public void setup() { () -> 1L ); clusterService = mock(ClusterService.class); - enabledSettings = Settings.builder().put("xpack.security.authc.cluster_state_role_mappings.enabled", true).build(); + disabledSettings = Settings.builder().put("xpack.security.authc.cluster_state_role_mappings.enabled", false).build(); if (randomBoolean()) { - disabledSettings = Settings.builder().put("xpack.security.authc.cluster_state_role_mappings.enabled", false).build(); + enabledSettings = Settings.builder().put("xpack.security.authc.cluster_state_role_mappings.enabled", true).build(); } else { - // the cluster state role mapper is disabled by default - disabledSettings = Settings.EMPTY; + // the cluster state role mapper is enabled by default + enabledSettings = Settings.EMPTY; } } @@ -95,6 +95,9 @@ public void testRoleResolving() throws Exception { verify(mapping1).isEnabled(); verify(mapping2).isEnabled(); verify(mapping3).isEnabled(); + verify(mapping1).getName(); + verify(mapping2).getName(); + verify(mapping3).getName(); verify(mapping2).getExpression(); verify(mapping3).getExpression(); verify(mapping3).getRoleNames(same(scriptService), same(expressionModel)); diff --git a/x-pack/plugin/shutdown/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/plugin/shutdown/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index 8125c8d9d52ad..fa6a908891400 100644 --- a/x-pack/plugin/shutdown/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/plugin/shutdown/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -14,18 +14,15 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.FeatureFlag; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.cluster.util.resource.Resource; import org.elasticsearch.test.rest.ESRestTestCase; -import org.elasticsearch.test.rest.RestTestLegacyFeatures; import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; import org.elasticsearch.upgrades.ParameterizedFullClusterRestartTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; -import org.junit.Before; import org.junit.ClassRule; import java.io.IOException; @@ -88,13 +85,6 @@ protected Settings restClientSettings() { .build(); } - @Before - public void checkClusterVersion() { - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // always true - var originalClusterSupportsShutdown = oldClusterHasFeature(RestTestLegacyFeatures.SHUTDOWN_SUPPORTED); - assumeTrue("no shutdown in versions before 7.15", originalClusterSupportsShutdown); - } - @SuppressWarnings("unchecked") public void testNodeShutdown() throws Exception { if (isRunningAgainstOldCluster()) { diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml index 9607b64385721..939f153b8b0ea 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml @@ -3,7 +3,7 @@ setup: - requires: cluster_features: ["gte_v8.11.0"] reason: "ESQL is available in 8.11+" - test_runner_features: allowed_warnings_regex + test_runner_features: [allowed_warnings_regex, capabilities] - do: indices.create: @@ -385,8 +385,31 @@ setup: - length: { values: 2 } - match: { values.0: [ [ "foo", "bar" ] ] } - match: { values.1: [ "baz" ] } +--- +"reverse text": + - requires: + capabilities: + - method: POST + path: /_query + parameters: [method, path, parameters, capabilities] + capabilities: [fn_reverse] + reason: "reverse not yet added" + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM test | SORT name | EVAL job_reversed = REVERSE(job), tag_reversed = REVERSE(tag) | KEEP job_reversed, tag_reversed' + + - match: { columns.0.name: "job_reversed" } + - match: { columns.0.type: "text" } + - match: { columns.1.name: "tag_reversed" } + - match: { columns.1.type: "text" } + - length: { values: 2 } + - match: { values.0: [ "rotceriD TI", "rab oof" ] } + - match: { values.1: [ "tsilaicepS lloryaP", "zab" ] } --- "stats text with raw": - do: diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index 303f799e0d9cd..c57e5653d1279 100644 --- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -361,29 +361,10 @@ public void testApiKeySuperuser() throws IOException { ) ) ); - if (clusterHasFeature(RestTestLegacyFeatures.SECURITY_ROLE_DESCRIPTORS_OPTIONAL)) { - createApiKeyRequest.setJsonEntity(""" - { - "name": "super_legacy_key" - }"""); - } else { - createApiKeyRequest.setJsonEntity(""" - { - "name": "super_legacy_key", - "role_descriptors": { - "super": { - "cluster": [ "all" ], - "indices": [ - { - "names": [ "*" ], - "privileges": [ "all" ], - "allow_restricted_indices": true - } - ] - } - } - }"""); - } + createApiKeyRequest.setJsonEntity(""" + { + "name": "super_legacy_key" + }"""); final Map createApiKeyResponse = entityAsMap(client().performRequest(createApiKeyRequest)); final byte[] keyBytes = (createApiKeyResponse.get("id") + ":" + createApiKeyResponse.get("api_key")).getBytes( StandardCharsets.UTF_8 @@ -393,20 +374,6 @@ public void testApiKeySuperuser() throws IOException { final Request saveApiKeyRequest = new Request("PUT", "/api_keys/_doc/super_legacy_key"); saveApiKeyRequest.setJsonEntity("{\"auth_header\":\"" + apiKeyAuthHeader + "\"}"); assertOK(client().performRequest(saveApiKeyRequest)); - - if (clusterHasFeature(RestTestLegacyFeatures.SYSTEM_INDICES_REST_ACCESS_ENFORCED) == false) { - final Request indexRequest = new Request("POST", ".security/_doc"); - indexRequest.setJsonEntity(""" - { - "doc_type": "foo" - }"""); - if (clusterHasFeature(RestTestLegacyFeatures.SYSTEM_INDICES_REST_ACCESS_DEPRECATED)) { - indexRequest.setOptions(systemIndexWarningHandlerOptions(".security-7").addHeader("Authorization", apiKeyAuthHeader)); - } else { - indexRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", apiKeyAuthHeader)); - } - assertOK(client().performRequest(indexRequest)); - } } else { final Request getRequest = new Request("GET", "/api_keys/_doc/super_legacy_key"); final Map getResponseMap = responseAsMap(client().performRequest(getRequest)); @@ -472,15 +439,7 @@ public void testRollupAfterRestart() throws Exception { // create the rollup job final Request createRollupJobRequest = new Request("PUT", "/_rollup/job/rollup-job-test"); - - String intervalType; - if (clusterHasFeature(RestTestLegacyFeatures.SEARCH_AGGREGATIONS_FORCE_INTERVAL_SELECTION_DATE_HISTOGRAM)) { - intervalType = "fixed_interval"; - } else { - intervalType = "interval"; - } - - createRollupJobRequest.setJsonEntity(Strings.format(""" + createRollupJobRequest.setJsonEntity(""" { "index_pattern": "rollup-*", "rollup_index": "results-rollup", @@ -489,7 +448,7 @@ public void testRollupAfterRestart() throws Exception { "groups": { "date_histogram": { "field": "timestamp", - "%s": "5m" + "fixed_interval": "5m" } }, "metrics": [ @@ -498,7 +457,7 @@ public void testRollupAfterRestart() throws Exception { "metrics": [ "min", "max", "sum" ] } ] - }""", intervalType)); + }"""); Map createRollupJobResponse = entityAsMap(client().performRequest(createRollupJobRequest)); assertThat(createRollupJobResponse.get("acknowledged"), equalTo(Boolean.TRUE)); @@ -550,11 +509,7 @@ public void testTransformLegacyTemplateCleanup() throws Exception { assertThat(createIndexResponse.get("acknowledged"), equalTo(Boolean.TRUE)); // create a transform - String endpoint = clusterHasFeature(RestTestLegacyFeatures.TRANSFORM_NEW_API_ENDPOINT) - ? "_transform/transform-full-cluster-restart-test" - : "_data_frame/transforms/transform-full-cluster-restart-test"; - final Request createTransformRequest = new Request("PUT", endpoint); - + final Request createTransformRequest = new Request("PUT", "_transform/transform-full-cluster-restart-test"); createTransformRequest.setJsonEntity(""" { "source": { diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java index 3674f811ebb0a..c825de31a7f6e 100644 --- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java @@ -28,8 +28,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractXpackFullClusterRestartTestCase { @@ -62,13 +60,8 @@ public void testMlConfigIndexMappingsAfterMigration() throws Exception { if (isRunningAgainstOldCluster()) { // trigger .ml-config index creation createAnomalyDetectorJob(OLD_CLUSTER_JOB_ID); - if (clusterHasFeature(RestTestLegacyFeatures.ML_ANALYTICS_MAPPINGS)) { - // .ml-config has mappings for analytics as the feature was introduced in 7.3.0 - assertThat(getDataFrameAnalysisMappings().keySet(), hasItem("outlier_detection")); - } else { - // .ml-config does not yet have correct mappings, it will need an update after cluster is upgraded - assertThat(getDataFrameAnalysisMappings(), is(nullValue())); - } + // .ml-config has mappings for analytics as the feature was introduced in 7.3.0 + assertThat(getDataFrameAnalysisMappings().keySet(), hasItem("outlier_detection")); } else { // trigger .ml-config index mappings update createAnomalyDetectorJob(NEW_CLUSTER_JOB_ID); diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index 16345a19fc950..7dc0a2f48bbc9 100644 --- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -38,7 +38,6 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; public class MlHiddenIndicesFullClusterRestartIT extends AbstractXpackFullClusterRestartTestCase { @@ -78,36 +77,6 @@ public void testMlIndicesBecomeHidden() throws Exception { // trigger ML indices creation createAnomalyDetectorJob(JOB_ID); openAnomalyDetectorJob(JOB_ID); - - if (clusterHasFeature(RestTestLegacyFeatures.ML_INDICES_HIDDEN) == false) { - Map indexSettingsMap = contentAsMap(getMlIndicesSettings()); - Map aliasesMap = contentAsMap(getMlAliases()); - - assertThat("Index settings map was: " + indexSettingsMap, indexSettingsMap, is(aMapWithSize(greaterThanOrEqualTo(4)))); - for (Map.Entry e : indexSettingsMap.entrySet()) { - String indexName = e.getKey(); - @SuppressWarnings("unchecked") - Map settings = (Map) e.getValue(); - assertThat(settings, is(notNullValue())); - assertThat( - "Index " + indexName + " expected not to be hidden but was, settings = " + settings, - XContentMapValues.extractValue(settings, "settings", "index", "hidden"), - is(nullValue()) - ); - } - - for (Tuple, String> indexAndAlias : EXPECTED_INDEX_ALIAS_PAIRS) { - List indices = indexAndAlias.v1(); - String alias = indexAndAlias.v2(); - for (String index : indices) { - assertThat( - indexAndAlias + " expected not be hidden but was, aliasesMap = " + aliasesMap, - XContentMapValues.extractValue(aliasesMap, index, "aliases", alias, "is_hidden"), - is(nullValue()) - ); - } - } - } } else { // The 5 operations in MlInitializationService.makeMlInternalIndicesHidden() run sequentially, so might // not all be finished when this test runs. The desired state should exist within a few seconds of startup, diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java index 767f27d4e4f93..fee6910fcf6c0 100644 --- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java @@ -76,12 +76,7 @@ public void testMappingsAreUpdated() throws Exception { """); client().performRequest(putWatchRequest); - if (clusterHasFeature(RestTestLegacyFeatures.WATCHES_VERSION_IN_META)) { - assertMappingVersion(".watches", getOldClusterVersion()); - } else { - // watches indices from before 7.10 do not have mapping versions in _meta - assertNoMappingVersion(".watches"); - } + assertMappingVersion(".watches", getOldClusterVersion()); } else { assertMappingVersion(".watches", Build.current().version()); } @@ -101,9 +96,7 @@ private void assertNoMappingVersion(String index) throws Exception { assertBusy(() -> { Request mappingRequest = new Request("GET", index + "/_mappings"); assert isRunningAgainstOldCluster(); - if (clusterHasFeature(RestTestLegacyFeatures.SYSTEM_INDICES_REST_ACCESS_DEPRECATED)) { - mappingRequest.setOptions(getWarningHandlerOptions(index)); - } + mappingRequest.setOptions(getWarningHandlerOptions(index)); Response response = client().performRequest(mappingRequest); String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); assertThat(responseBody, not(containsString("\"version\":\""))); diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java index af67ab5751e96..4324aed5fee18 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java @@ -7,7 +7,6 @@ package org.elasticsearch.upgrades; import org.elasticsearch.Build; -import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.io.Streams; @@ -15,7 +14,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Booleans; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.test.SecuritySettingsSourceField; import org.junit.Before; @@ -49,16 +47,6 @@ protected static boolean isOriginalClusterCurrent() { return UPGRADE_FROM_VERSION.equals(Build.current().version()); } - @Deprecated(forRemoval = true) - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - // Tests should be reworked to rely on features from the current cluster (old, mixed or upgraded). - // Version test against the original cluster will be removed - protected static boolean isOriginalClusterVersionAtLeast(Version supportedVersion) { - // Always assume non-semantic versions are OK: this method will be removed in V9, we are testing the pre-upgrade cluster version, - // and non-semantic versions are always V8+ - return parseLegacyVersion(UPGRADE_FROM_VERSION).map(x -> x.onOrAfter(supportedVersion)).orElse(true); - } - @Override protected boolean resetFeatureStates() { return false; diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java index 4de2c610e5c48..8c051d03d5f04 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java @@ -8,13 +8,11 @@ package org.elasticsearch.upgrades; import org.apache.http.util.EntityUtils; -import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.Strings; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.xcontent.XContentType; import org.junit.After; import org.junit.Before; @@ -101,11 +99,6 @@ public void removeLogging() throws IOException { } public void testTrainedModelDeployment() throws Exception { - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // upgrade will always be from v8, condition can be removed - var originalClusterAtLeastV8 = isOriginalClusterVersionAtLeast(Version.V_8_0_0); - // These tests assume the original cluster is v8 - testing for features on the _current_ cluster will break for NEW - assumeTrue("NLP model deployments added in 8.0", originalClusterAtLeastV8); - final String modelId = "upgrade-deployment-test"; switch (CLUSTER_TYPE) { @@ -140,11 +133,6 @@ public void testTrainedModelDeployment() throws Exception { } public void testTrainedModelDeploymentStopOnMixedCluster() throws Exception { - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // upgrade will always be from v8, condition can be removed - var originalClusterAtLeastV8 = isOriginalClusterVersionAtLeast(Version.V_8_0_0); - // These tests assume the original cluster is v8 - testing for features on the _current_ cluster will break for NEW - assumeTrue("NLP model deployments added in 8.0", originalClusterAtLeastV8); - final String modelId = "upgrade-deployment-test-stop-mixed-cluster"; switch (CLUSTER_TYPE) { diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlAssignmentPlannerUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlAssignmentPlannerUpgradeIT.java index 7cefaa2edb388..74165eeb07b8a 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlAssignmentPlannerUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlAssignmentPlannerUpgradeIT.java @@ -7,14 +7,12 @@ package org.elasticsearch.upgrades; -import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.Strings; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.test.rest.RestTestLegacyFeatures; @@ -71,10 +69,6 @@ public class MlAssignmentPlannerUpgradeIT extends AbstractUpgradeTestCase { @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/101926") public void testMlAssignmentPlannerUpgrade() throws Exception { - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // upgrade will always be from v8, condition can be removed - var originalClusterAtLeastV8 = isOriginalClusterVersionAtLeast(Version.V_8_0_0); - // These tests assume the original cluster is v8 - testing for features on the _current_ cluster will break for NEW - assumeTrue("NLP model deployments added in 8.0", originalClusterAtLeastV8); assumeFalse("This test deploys multiple models which cannot be accommodated on a single processor", IS_SINGLE_PROCESSOR_TEST); logger.info("Starting testMlAssignmentPlannerUpgrade, model size {}", RAW_MODEL_SIZE);