diff --git a/.github/workflows/sync-main-to-jdk-branch.yml b/.github/workflows/sync-main-to-jdk-branch.yml deleted file mode 100644 index 4e2dcfb0eed76..0000000000000 --- a/.github/workflows/sync-main-to-jdk-branch.yml +++ /dev/null @@ -1,21 +0,0 @@ -# Daily update of JDK update branch with changes from main -name: "Merge main to openjdk23-bundle branch" -on: - schedule: - - cron: "30 17 * * *" - workflow_dispatch: {} - -jobs: - merge-branch: - if: github.repository == 'elastic/elasticsearch' - runs-on: ubuntu-latest - steps: - - name: checkout - uses: actions/checkout@master - - - name: merge - uses: devmasx/merge-branch@1.4.0 - with: - type: "now" - target_branch: openjdk23-bundle - github_token: ${{ secrets.ELASTICSEARCHMACHINE_TOKEN }} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesPlugin.java index 4724c576066f1..c62089a42b159 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesPlugin.java @@ -39,7 +39,7 @@ * } * } * - * Will copy the entire core Rest API specifications (assuming the project has tests) and any of the the X-pack specs starting with enrich*. + * Will copy the entire core Rest API specifications (assuming the project has tests) and any of the X-pack specs starting with enrich*. * It is recommended (but not required) to also explicitly declare which core specs your project depends on to help optimize the caching * behavior. * For example: @@ -66,7 +66,7 @@ * } * } * - * Will copy any of the the x-pack tests that start with graph, and will copy the X-pack graph specification, as well as the full core + * Will copy any of the x-pack tests that start with graph, and will copy the X-pack graph specification, as well as the full core * Rest API specification. *

* Additionally you can specify which sourceSetName resources should be copied to. The default is the yamlRestTest source set. diff --git a/build.gradle b/build.gradle index 746f964cb6158..2ef0511b2be88 100644 --- a/build.gradle +++ b/build.gradle @@ -409,6 +409,10 @@ gradle.projectsEvaluated { } } +tasks.named("validateChangelogs") { + onlyIf { project.gradle.startParameter.taskNames.any { it.startsWith("checkPart") || it == 'functionalTests' } == false } +} + tasks.named("precommit") { dependsOn gradle.includedBuild('build-tools').task(':precommit') dependsOn gradle.includedBuild('build-tools-internal').task(':precommit') diff --git a/docs/changelog/115938.yaml b/docs/changelog/115938.yaml new file mode 100644 index 0000000000000..e096d8821a1d7 --- /dev/null +++ b/docs/changelog/115938.yaml @@ -0,0 +1,9 @@ +pr: 115938 +summary: Wait for up to 2 seconds for yellow status before starting search +area: Machine Learning +type: bug +issues: + - 107777 + - 105955 + - 107815 + - 112191 diff --git a/docs/reference/connector/docs/connectors-hosted-tutorial-mongo.asciidoc b/docs/reference/connector/docs/connectors-hosted-tutorial-mongo.asciidoc index a1f7048705555..12050ff17e279 100644 --- a/docs/reference/connector/docs/connectors-hosted-tutorial-mongo.asciidoc +++ b/docs/reference/connector/docs/connectors-hosted-tutorial-mongo.asciidoc @@ -90,7 +90,7 @@ Find this by https://www.mongodb.com/docs/atlas/tutorial/connect-to-your-cluster In this example, we'll use the `sample_mflix` database. * *Collection*: The name of the collection you want to sync. In this example, we'll use the `comments` collection of the `sample_mflix` database. -* *Username*: The username you created earlier, in the the setup phase. +* *Username*: The username you created earlier, in the setup phase. * *Password*: The password you created earlier. Keep these details handy! diff --git a/docs/reference/esql/esql-query-api.asciidoc b/docs/reference/esql/esql-query-api.asciidoc index b1582721ad0e0..63b8738266132 100644 --- a/docs/reference/esql/esql-query-api.asciidoc +++ b/docs/reference/esql/esql-query-api.asciidoc @@ -46,7 +46,7 @@ supports this parameter for CSV responses. `drop_null_columns`:: (Optional, boolean) Should columns that are entirely `null` be removed from the `columns` and `values` portion of the results? Defaults to `false`. If -`true` the the response will include an extra section under the name +`true` the response will include an extra section under the name `all_columns` which has the name of all columns. `format`:: diff --git a/docs/reference/migration/index.asciidoc b/docs/reference/migration/index.asciidoc index 719588cb4b0d0..11aca45b003fa 100644 --- a/docs/reference/migration/index.asciidoc +++ b/docs/reference/migration/index.asciidoc @@ -1,40 +1,6 @@ include::migration_intro.asciidoc[] -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> +* <> -include::migrate_8_17.asciidoc[] -include::migrate_8_16.asciidoc[] -include::migrate_8_15.asciidoc[] -include::migrate_8_14.asciidoc[] -include::migrate_8_13.asciidoc[] -include::migrate_8_12.asciidoc[] -include::migrate_8_11.asciidoc[] -include::migrate_8_10.asciidoc[] -include::migrate_8_9.asciidoc[] -include::migrate_8_8.asciidoc[] -include::migrate_8_7.asciidoc[] -include::migrate_8_6.asciidoc[] -include::migrate_8_5.asciidoc[] -include::migrate_8_4.asciidoc[] -include::migrate_8_3.asciidoc[] -include::migrate_8_2.asciidoc[] -include::migrate_8_1.asciidoc[] -include::migrate_8_0.asciidoc[] +include::migrate_9_0.asciidoc[] diff --git a/docs/reference/migration/migrate_8_0.asciidoc b/docs/reference/migration/migrate_8_0.asciidoc deleted file mode 100644 index 09433904f2ea8..0000000000000 --- a/docs/reference/migration/migrate_8_0.asciidoc +++ /dev/null @@ -1,94 +0,0 @@ -[[migrating-8.0]] -== Migrating to 8.0 -++++ -8.0 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.0. - -See also <> and <>. - -[discrete] -[[breaking-changes-8.0]] -=== Breaking changes - -The following changes in {es} 8.0 might affect your applications -and prevent them from operating normally. -Before upgrading to 8.0, review these changes and take the described steps -to mitigate the impact. - -include::migrate_8_0/cluster-node-setting-changes.asciidoc[] -include::migrate_8_0/command-line-tool-changes.asciidoc[] -include::migrate_8_0/index-setting-changes.asciidoc[] -include::migrate_8_0/java-api-changes.asciidoc[] -include::migrate_8_0/jvm-option-changes.asciidoc[] -include::migrate_8_0/logging-changes.asciidoc[] -include::migrate_8_0/mapping-changes.asciidoc[] -include::migrate_8_0/packaging-changes.asciidoc[] -include::migrate_8_0/painless-changes.asciidoc[] -include::migrate_8_0/plugin-changes.asciidoc[] -include::migrate_8_0/rest-api-changes.asciidoc[] -include::migrate_8_0/sql-jdbc-changes.asciidoc[] -include::migrate_8_0/system-req-changes.asciidoc[] -include::migrate_8_0/transform.asciidoc[] - -[discrete] -[[deprecated-8.0]] -=== Deprecations - -The following functionality has been deprecated in {es} 8.0 -and will be removed in a future version. -While this won't have an immediate impact on your applications, -we strongly encourage you take the described steps to update your code -after upgrading to 8.0. - -To find out if you are using any deprecated functionality, -enable <>. - -[discrete] -[[breaking_80_cluster_node_setting_deprecations]] -==== Cluster and node setting deprecations - -[[deprecate-transient-cluster-settings]] -.We no longer recommend using transient cluster settings. -[%collapsible] -==== -*Details* + -We no longer recommend using transient cluster settings. Use persistent cluster -settings instead. If a cluster becomes unstable, transient settings can clear -unexpectedly, resulting in an undesired cluster configuration. - -*Impact* + -Transient cluster settings are not yet deprecated, but we plan to deprecate them -in a future release. For migration steps, see the -{ref}/transient-settings-migration-guide.html[Transient settings migration -guide]. -==== - -[discrete] -[[breaking_80_command_line_tool_deprecations]] -==== Command line tool deprecations - -TIP: {ess-skip-section} - -[[deprecate-elasticsearch-setup-passwords]] -.The `elasticsearch-setup-passwords` tool is deprecated. -[%collapsible] -==== -*Details* + -The `elasticsearch-setup-passwords` tool is deprecated in 8.0. To -manually reset the password for built-in users (including the `elastic` user), use -the {ref}/reset-password.html[`elasticsearch-reset-password`] tool, the {es} -{ref}/security-api-change-password.html[change passwords API], or the -User Management features in {kib}. -`elasticsearch-setup-passwords` will be removed in a future release. - -*Impact* + -Passwords are generated automatically for the `elastic` user when you start {es} for the first time. If you run `elasticsearch-setup-passwords` after -starting {es}, it will fail because the `elastic` -user password is already configured. -==== - -include::migrate_8_0/migrate_to_java_time.asciidoc[] -include::transient-settings-migration-guide.asciidoc[] diff --git a/docs/reference/migration/migrate_8_0/cluster-node-setting-changes.asciidoc b/docs/reference/migration/migrate_8_0/cluster-node-setting-changes.asciidoc deleted file mode 100644 index bad4ab93676bb..0000000000000 --- a/docs/reference/migration/migrate_8_0/cluster-node-setting-changes.asciidoc +++ /dev/null @@ -1,931 +0,0 @@ -[discrete] -[[breaking_80_cluster_node_setting_changes]] -==== Cluster and node setting changes - -TIP: {ess-setting-change} - -.`action.destructive_requires_name` now defaults to `true`. {ess-icon} -[%collapsible] -==== -*Details* + -The default for the `action.destructive_requires_name` setting changes from `false` -to `true` in {es} 8.0.0. - -Previously, defaulting to `false` allowed users to use wildcard -patterns to delete, close, or change index blocks on indices. -To prevent the accidental deletion of indices that happen to match a -wildcard pattern, we now default to requiring that destructive -operations explicitly name the indices to be modified. - -*Impact* + -To use wildcard patterns for destructive actions, set -`action.destructive_requires_name` to `false` using the -{ref}/cluster-update-settings.html[] cluster settings API]. -==== - -.You can no longer set `xpack.searchable.snapshot.shared_cache.size` on non-frozen nodes. -[%collapsible] -==== -*Details* + -You can no longer set -{ref}/searchable-snapshots.html#searchable-snapshots-shared-cache[`xpack.searchable.snapshot.shared_cache.size`] -on a node that doesn't have the `data_frozen` node role. This setting reserves -disk space for the shared cache of partially mounted indices. {es} only -allocates partially mounted indices to nodes with the `data_frozen` role. - -*Impact* + -Remove `xpack.searchable.snapshot.shared_cache.size` from `elasticsearch.yml` -for nodes that don't have the `data_frozen` role. Specifying the setting on a -non-frozen node will result in an error on startup. -==== - -[[max_clause_count_change]] -.`indices.query.bool.max_clause_count` is deprecated and has no effect. -[%collapsible] -==== -*Details* + -Elasticsearch will now dynamically set the maximum number of allowed clauses -in a query, using a heuristic based on the size of the search thread pool and -the size of the heap allocated to the JVM. This limit has a minimum value of -1024 and will in most cases be larger (for example, a node with 30Gb RAM and -48 CPUs will have a maximum clause count of around 27,000). Larger heaps lead -to higher values, and larger thread pools result in lower values. - -*Impact* + -Queries with many clauses should be avoided whenever possible. -If you previously bumped this setting to accommodate heavy queries, -you might need to increase the amount of memory available to Elasticsearch, -or to reduce the size of your search thread pool so that more memory is -available to each concurrent search. - -In previous versions of Lucene you could get around this limit by nesting -boolean queries within each other, but the limit is now based on the total -number of leaf queries within the query as a whole and this workaround will -no longer help. - -Specifying `indices.query.bool.max_clause_count` will have no effect -but will generate deprecation warnings. To avoid these warnings, remove the -setting from `elasticsearch.yml` during an upgrade or node restart. -==== - -[[ilm-poll-interval-limit]] -.`indices.lifecycle.poll_interval` must be greater than `1s`. -[%collapsible] -==== -*Details* + -Setting `indices.lifecycle.poll_interval` too low can cause -excessive load on a cluster. The poll interval must now be at least `1s` (one second). - -*Impact* + -Set `indices.lifecycle.poll_interval` setting to `1s` or -greater in `elasticsearch.yml` or through the -{ref}/cluster-update-settings.html[cluster update settings API]. - -Setting `indices.lifecycle.poll_interval` to less than `1s` in -`elasticsearch.yml` will result in an error on startup. -{ref}/cluster-update-settings.html[Cluster update settings API] requests that -set `indices.lifecycle.poll_interval` to less than `1s` will return an error. -==== - -.The file and native realms are now enabled unless explicitly disabled. -[%collapsible] -==== -*Details* + -The file and native realms are now enabled unless explicitly disabled. If -explicitly disabled, the file and native realms remain disabled at all times. - -Previously, the file and native realms had the following implicit behaviors: - -* If the file and native realms were not configured, they were implicitly disabled -if any other realm was configured. - -* If no other realm was available because realms were either not configured, -not permitted by license, or explicitly disabled, the file and native realms -were enabled, even if explicitly disabled. - -*Impact* + -To explicitly disable the file or native realm, set the respective -`file..enabled` or `native..enabled` setting to `false` -under the `xpack.security.authc.realms` namespace in `elasticsearch.yml`. - -The following configuration example disables the native realm and the file realm. - -[source,yaml] ----- -xpack.security.authc.realms: - - native.realm1.enabled: false - file.realm2.enabled: false - - ... ----- -==== - -.The realm `order` setting is now required. -[%collapsible] -==== -*Details* + -The `xpack.security.authc.realms.{type}.{name}.order` setting is now required and must be -specified for each explicitly configured realm. Each value must be unique. - -*Impact* + -The cluster will fail to start if the requirements are not met. - -For example, the following configuration is invalid: -[source,yaml] --------------------------------------------------- -xpack.security.authc.realms.kerberos.kerb1: - keytab.path: es.keytab - remove_realm_name: false --------------------------------------------------- - -And must be configured as: -[source,yaml] --------------------------------------------------- -xpack.security.authc.realms.kerberos.kerb1: - order: 0 - keytab.path: es.keytab - remove_realm_name: false --------------------------------------------------- -==== - -[[breaking_80_allocation_change_include_relocations_removed]] -.`cluster.routing.allocation.disk.include_relocations` has been removed. -[%collapsible] -==== -*Details* + -{es} now always accounts for the sizes of relocating shards when making -allocation decisions based on the disk usage of the nodes in the cluster. In -earlier versions, you could disable this by setting `cluster.routing.allocation.disk.include_relocations` to `false`. -That could result in poor allocation decisions that could overshoot watermarks and require significant -extra work to correct. The `cluster.routing.allocation.disk.include_relocations` setting has been removed. - -*Impact* + -Remove the `cluster.routing.allocation.disk.include_relocations` -setting. Specifying this setting in `elasticsearch.yml` will result in an error -on startup. -==== - -.`cluster.join.timeout` has been removed. -[%collapsible] -==== -*Details* + -The `cluster.join.timeout` setting has been removed. Join attempts no longer -time out. - -*Impact* + -Remove `cluster.join.timeout` from `elasticsearch.yml`. -==== - -.`discovery.zen` settings have been removed. -[%collapsible] -==== -*Details* + -All settings under the `discovery.zen` namespace are no longer supported. They existed only only for BWC reasons in 7.x. This includes: - -- `discovery.zen.minimum_master_nodes` -- `discovery.zen.no_master_block` -- `discovery.zen.hosts_provider` -- `discovery.zen.publish_timeout` -- `discovery.zen.commit_timeout` -- `discovery.zen.publish_diff.enable` -- `discovery.zen.ping.unicast.concurrent_connects` -- `discovery.zen.ping.unicast.hosts.resolve_timeout` -- `discovery.zen.ping.unicast.hosts` -- `discovery.zen.ping_timeout` -- `discovery.zen.unsafe_rolling_upgrades_enabled` -- `discovery.zen.fd.connect_on_network_disconnect` -- `discovery.zen.fd.ping_interval` -- `discovery.zen.fd.ping_timeout` -- `discovery.zen.fd.ping_retries` -- `discovery.zen.fd.register_connection_listener` -- `discovery.zen.join_retry_attempts` -- `discovery.zen.join_retry_delay` -- `discovery.zen.join_timeout` -- `discovery.zen.max_pings_from_another_master` -- `discovery.zen.send_leave_request` -- `discovery.zen.master_election.wait_for_joins_timeout` -- `discovery.zen.master_election.ignore_non_master_pings` -- `discovery.zen.publish.max_pending_cluster_states` -- `discovery.zen.bwc_ping_timeout` - -*Impact* + -Remove the `discovery.zen` settings from `elasticsearch.yml`. Specifying these settings will result in an error on startup. -==== - -.`http.content_type.required` has been removed. -[%collapsible] -==== -*Details* + -The `http.content_type.required` setting was deprecated in Elasticsearch 6.0 -and has been removed in Elasticsearch 8.0. The setting was introduced in -Elasticsearch 5.3 to prepare users for Elasticsearch 6.0, where content type -auto detection was removed for HTTP requests. - -*Impact* + -Remove the `http.content_type.required` setting from `elasticsearch.yml`. Specifying this setting will result in an error on startup. -==== - -.`http.tcp_no_delay` has been removed. -[%collapsible] -==== -*Details* + -The `http.tcp_no_delay` setting was deprecated in 7.x and has been removed in 8.0. Use `http.tcp.no_delay` instead. - -*Impact* + -Replace the `http.tcp_no_delay` setting with `http.tcp.no_delay`. -Specifying `http.tcp_no_delay` in `elasticsearch.yml` will -result in an error on startup. -==== - -.`network.tcp.connect_timeout` has been removed. -[%collapsible] -==== -*Details* + -The `network.tcp.connect_timeout` setting was deprecated in 7.x and has been removed in 8.0. This setting -was a fallback setting for `transport.connect_timeout`. - -*Impact* + -Remove the `network.tcp.connect_timeout` setting. -Use the `transport.connect_timeout` setting to change the default connection -timeout for client connections. Specifying -`network.tcp.connect_timeout` in `elasticsearch.yml` will result in an -error on startup. -==== - -.`node.max_local_storage_nodes` has been removed. -[%collapsible] -==== -*Details* + -The `node.max_local_storage_nodes` setting was deprecated in 7.x and -has been removed in 8.0. Nodes should be run on separate data paths -to ensure that each node is consistently assigned to the same data path. - -*Impact* + -Remove the `node.max_local_storage_nodes` setting. Specifying this -setting in `elasticsearch.yml` will result in an error on startup. -==== - -[[accept-default-password-removed]] -.The `accept_default_password` setting has been removed. -[%collapsible] -==== -*Details* + -The `xpack.security.authc.accept_default_password` setting has not had any affect -since the 6.0 release of {es} and is no longer allowed. - -*Impact* + -Remove the `xpack.security.authc.accept_default_password` setting from `elasticsearch.yml`. -Specifying this setting will result in an error on startup. -==== - -[[roles-index-cache-removed]] -.The `roles.index.cache.*` settings have been removed. -[%collapsible] -==== -*Details* + -The `xpack.security.authz.store.roles.index.cache.max_size` and -`xpack.security.authz.store.roles.index.cache.ttl` settings have -been removed. These settings have been redundant and deprecated -since the 5.2 release of {es}. - -*Impact* + -Remove the `xpack.security.authz.store.roles.index.cache.max_size` -and `xpack.security.authz.store.roles.index.cache.ttl` settings from `elasticsearch.yml` . -Specifying these settings will result in an error on startup. -==== - -[[separating-node-and-client-traffic]] -.The `transport.profiles.*.xpack.security.type` setting has been removed. -[%collapsible] -==== -*Details* + -The `transport.profiles.*.xpack.security.type` setting is no longer supported. -The Transport Client has been removed and all client traffic now uses -the HTTP transport. Transport profiles using this setting should be removed. - -*Impact* + -Remove the `transport.profiles.*.xpack.security.type` setting from `elasticsearch.yml`. -Specifying this setting in a transport profile will result in an error on startup. -==== - -[discrete] -[[saml-realm-nameid-changes]] -.The `nameid_format` SAML realm setting no longer has a default value. -[%collapsible] -==== -*Details* + -In SAML, Identity Providers (IdPs) can either be explicitly configured to -release a `NameID` with a specific format, or configured to attempt to conform -with the requirements of a Service Provider (SP). The SP declares its -requirements in the `NameIDPolicy` element of a SAML Authentication Request. -In {es}, the `nameid_format` SAML realm setting controls the `NameIDPolicy` -value. - -Previously, the default value for `nameid_format` was -`urn:oasis:names:tc:SAML:2.0:nameid-format:transient`. This setting created -authentication requests that required the IdP to release `NameID` with a -`transient` format. - -The default value has been removed, which means that {es} will create SAML Authentication Requests by default that don't put this requirement on the -IdP. If you want to retain the previous behavior, set `nameid_format` to -`urn:oasis:names:tc:SAML:2.0:nameid-format:transient`. - -*Impact* + -If you currently don't configure `nameid_format` explicitly, it's possible -that your IdP will reject authentication requests from {es} because the requests -do not specify a `NameID` format (and your IdP is configured to expect one). -This mismatch can result in a broken SAML configuration. If you're unsure whether -your IdP is explicitly configured to use a certain `NameID` format and you want to retain current behavior -, try setting `nameid_format` to `urn:oasis:names:tc:SAML:2.0:nameid-format:transient` explicitly. -==== - -.The `xpack.security.transport.ssl.enabled` setting is now required to configure `xpack.security.transport.ssl` settings. -[%collapsible] -==== -*Details* + -It is now an error to configure any SSL settings for -`xpack.security.transport.ssl` without also configuring -`xpack.security.transport.ssl.enabled`. - -*Impact* + -If using other `xpack.security.transport.ssl` settings, you must explicitly -specify the `xpack.security.transport.ssl.enabled` setting. - -If you do not want to enable SSL and are currently using other -`xpack.security.transport.ssl` settings, do one of the following: - -* Explicitly specify `xpack.security.transport.ssl.enabled` as `false` -* Discontinue use of other `xpack.security.transport.ssl` settings - -If you want to enable SSL, follow the instructions in -{ref}/configuring-tls.html#tls-transport[Encrypting communications between nodes -in a cluster]. As part of this configuration, explicitly specify -`xpack.security.transport.ssl.enabled` as `true`. - -For example, the following configuration is invalid: -[source,yaml] --------------------------------------------------- -xpack.security.transport.ssl.keystore.path: elastic-certificates.p12 -xpack.security.transport.ssl.truststore.path: elastic-certificates.p12 --------------------------------------------------- - -And must be configured as: -[source,yaml] --------------------------------------------------- -xpack.security.transport.ssl.enabled: true <1> -xpack.security.transport.ssl.keystore.path: elastic-certificates.p12 -xpack.security.transport.ssl.truststore.path: elastic-certificates.p12 --------------------------------------------------- -<1> or `false`. -==== - -.The `xpack.security.http.ssl.enabled` setting is now required to configure `xpack.security.http.ssl` settings. -[%collapsible] -==== -*Details* + -It is now an error to configure any SSL settings for -`xpack.security.http.ssl` without also configuring -`xpack.security.http.ssl.enabled`. - -*Impact* + -If using other `xpack.security.http.ssl` settings, you must explicitly -specify the `xpack.security.http.ssl.enabled` setting. - -If you do not want to enable SSL and are currently using other -`xpack.security.http.ssl` settings, do one of the following: - -* Explicitly specify `xpack.security.http.ssl.enabled` as `false` -* Discontinue use of other `xpack.security.http.ssl` settings - -If you want to enable SSL, follow the instructions in -{ref}/security-basic-setup-https.html#encrypt-http-communication[Encrypting HTTP client communications]. As part -of this configuration, explicitly specify `xpack.security.http.ssl.enabled` -as `true`. - -For example, the following configuration is invalid: -[source,yaml] --------------------------------------------------- -xpack.security.http.ssl.certificate: elasticsearch.crt -xpack.security.http.ssl.key: elasticsearch.key -xpack.security.http.ssl.certificate_authorities: [ "corporate-ca.crt" ] --------------------------------------------------- - -And must be configured as either: -[source,yaml] --------------------------------------------------- -xpack.security.http.ssl.enabled: true <1> -xpack.security.http.ssl.certificate: elasticsearch.crt -xpack.security.http.ssl.key: elasticsearch.key -xpack.security.http.ssl.certificate_authorities: [ "corporate-ca.crt" ] --------------------------------------------------- -<1> or `false`. -==== - -.A `xpack.security.transport.ssl` certificate and key are now required to enable SSL for the transport interface. -[%collapsible] -==== -*Details* + -It is now an error to enable SSL for the transport interface without also configuring -a certificate and key through use of the `xpack.security.transport.ssl.keystore.path` -setting or the `xpack.security.transport.ssl.certificate` and -`xpack.security.transport.ssl.key` settings. - -*Impact* + -If `xpack.security.transport.ssl.enabled` is set to `true`, provide a -certificate and key using the `xpack.security.transport.ssl.keystore.path` -setting or the `xpack.security.transport.ssl.certificate` and -`xpack.security.transport.ssl.key` settings. If a certificate and key is not -provided, {es} will return in an error on startup. -==== - -.A `xpack.security.http.ssl` certificate and key are now required to enable SSL for the HTTP server. -[%collapsible] -==== -*Details* + -It is now an error to enable SSL for the HTTP (Rest) server without also configuring -a certificate and key through use of the `xpack.security.http.ssl.keystore.path` -setting or the `xpack.security.http.ssl.certificate` and -`xpack.security.http.ssl.key` settings. - -*Impact* + -If `xpack.security.http.ssl.enabled` is set to `true`, provide a certificate and -key using the `xpack.security.http.ssl.keystore.path` setting or the -`xpack.security.http.ssl.certificate` and `xpack.security.http.ssl.key` -settings. If certificate and key is not provided, {es} will return in an error -on startup. -==== - -.PKCS#11 keystores and trustores cannot be configured in `elasticsearch.yml` -[%collapsible] -==== -*Details* + -The settings `*.ssl.keystore.type` and `*.ssl.truststore.type` no longer accept "PKCS11" as a valid type. -This applies to all SSL settings in Elasticsearch, including - -- `xpack.security.http.keystore.type` -- `xpack.security.transport.keystore.type` -- `xpack.security.http.truststore.type` -- `xpack.security.transport.truststore.type` - -As well as SSL settings for security realms, watcher and monitoring. - -Use of a PKCS#11 keystore or truststore as the JRE's default store is not affected. - -*Impact* + -If you have a PKCS#11 keystore configured within your `elasticsearch.yml` file, you must remove that -configuration and switch to a supported keystore type, or configure your PKCS#11 keystore as the -JRE default store. -==== - -.The `kibana` user has been replaced by `kibana_system`. -[%collapsible] -==== -*Details* + -The `kibana` user was historically used to authenticate {kib} to {es}. -The name of this user was confusing, and was often mistakenly used to login to {kib}. -This has been renamed to `kibana_system` in order to reduce confusion, and to better -align with other built-in system accounts. - -*Impact* + -Replace any use of the `kibana` user with the `kibana_system` user. Specifying -the `kibana` user in `kibana.yml` will result in an error on startup. - -If your `kibana.yml` used to contain: -[source,yaml] --------------------------------------------------- -elasticsearch.username: kibana --------------------------------------------------- - -then you should update to use the new `kibana_system` user instead: -[source,yaml] --------------------------------------------------- -elasticsearch.username: kibana_system --------------------------------------------------- - -IMPORTANT: The new `kibana_system` user does not preserve the previous `kibana` -user password. You must explicitly set a password for the `kibana_system` user. -==== - -[[search-remote-settings-removed]] -.The `search.remote.*` settings have been removed. -[%collapsible] -==== -*Details* + -In 6.5 these settings were deprecated in favor of `cluster.remote`. In 7.x we -provided automatic upgrading of these settings to their `cluster.remote` -counterparts. In 8.0.0, these settings have been removed. Elasticsearch will -refuse to start if you have these settings in your configuration or cluster -state. - -*Impact* + -Use the replacement `cluster.remote` settings. Discontinue use of the -`search.remote.*` settings. Specifying these settings in `elasticsearch.yml` -will result in an error on startup. -==== - -[[remove-pidfile]] -.The `pidfile` setting has been replaced by `node.pidfile`. -[%collapsible] -==== -*Details* + -To ensure that all settings are in a proper namespace, the `pidfile` setting was -previously deprecated in version 7.4.0 of Elasticsearch, and is removed in -version 8.0.0. Instead, use `node.pidfile`. - -*Impact* + -Use the `node.pidfile` setting. Discontinue use of the `pidfile` setting. -Specifying the `pidfile` setting in `elasticsearch.yml` will result in an error -on startup. -==== - -[[remove-processors]] -.The `processors` setting has been replaced by `node.processors`. -[%collapsible] -==== -*Details* + -To ensure that all settings are in a proper namespace, the `processors` setting -was previously deprecated in version 7.4.0 of Elasticsearch, and is removed in -version 8.0.0. Instead, use `node.processors`. - -*Impact* + -Use the `node.processors` setting. Discontinue use of the `processors` setting. -Specifying the `processors` setting in `elasticsearch.yml` will result in an -error on startup. -==== - -.The `node.processors` setting can no longer exceed the available number of processors. -[%collapsible] -==== -*Details* + -Previously it was possible to set the number of processors used to set the -default sizes for the thread pools to be more than the number of available -processors. As this leads to more context switches and more threads but without -an increase in the number of physical CPUs on which to schedule these additional -threads, the `node.processors` setting is now bounded by the number of available -processors. - -*Impact* + -If specified, ensure the value of `node.processors` setting does not exceed the -number of available processors. Setting the `node.processors` value greater than -the number of available processors in `elasticsearch.yml` will result in an -error on startup. -==== - -.The `cluster.remote.connect` setting has been removed. -[%collapsible] -==== -*Details* + -In Elasticsearch 7.7.0, the setting `cluster.remote.connect` was deprecated in -favor of setting `node.remote_cluster_client`. In Elasticsearch 8.0.0, the -setting `cluster.remote.connect` is removed. - -*Impact* + -Use the `node.remote_cluster_client` setting. Discontinue use of the -`cluster.remote.connect` setting. Specifying the `cluster.remote.connect` -setting in `elasticsearch.yml` will result in an error on startup. -==== - -.The `node.local_storage` setting has been removed. -[%collapsible] -==== -*Details* + -In Elasticsearch 7.8.0, the setting `node.local_storage` was deprecated and -beginning in Elasticsearch 8.0.0 all nodes will require local storage. Therefore, -the `node.local_storage` setting has been removed. - -*Impact* + -Discontinue use of the `node.local_storage` setting. Specifying this setting in -`elasticsearch.yml` will result in an error on startup. -==== - -.The `auth.password` setting for HTTP monitoring has been removed. -[%collapsible] -==== -*Details* + -In Elasticsearch 7.7.0, the setting `xpack.monitoring.exporters..auth.password` -was deprecated in favor of setting `xpack.monitoring.exporters..auth.secure_password`. -In Elasticsearch 8.0.0, the setting `xpack.monitoring.exporters..auth.password` is -removed. - -*Impact* + -Use the `xpack.monitoring.exporters..auth.secure_password` -setting. Discontinue use of the -`xpack.monitoring.exporters..auth.password` setting. Specifying -the `xpack.monitoring.exporters..auth.password` setting in -`elasticsearch.yml` will result in an error on startup. -==== - -.Settings used to disable basic license features have been removed. -[%collapsible] -==== -*Details* + -The following settings were deprecated in {es} 7.8.0 and have been removed -in {es} 8.0.0: - -* `xpack.enrich.enabled` -* `xpack.flattened.enabled` -* `xpack.ilm.enabled` -* `xpack.monitoring.enabled` -* `xpack.rollup.enabled` -* `xpack.slm.enabled` -* `xpack.sql.enabled` -* `xpack.transform.enabled` -* `xpack.vectors.enabled` - -These basic license features are now always enabled. - -If you have disabled ILM so that you can use another tool to manage Watcher -indices, the newly introduced `xpack.watcher.use_ilm_index_management` setting -may be set to false. - -*Impact* + -Discontinue use of the removed settings. Specifying these settings in -`elasticsearch.yml` will result in an error on startup. -==== - -.Settings used to defer cluster recovery pending a certain number of master nodes have been removed. -[%collapsible] -==== -*Details* + -The following cluster settings have been removed: - -* `gateway.expected_nodes` -* `gateway.expected_master_nodes` -* `gateway.recover_after_nodes` -* `gateway.recover_after_master_nodes` - -It is safe to recover the cluster as soon as a majority of master-eligible -nodes have joined so there is no benefit in waiting for any additional -master-eligible nodes to start. - -*Impact* + -Discontinue use of the removed settings. If needed, use -`gateway.expected_data_nodes` or `gateway.recover_after_data_nodes` to defer -cluster recovery pending a certain number of data nodes. -==== - -.Legacy role settings have been removed. -[%collapsible] -==== -*Details* + -The legacy role settings: - -* `node.data` -* `node.ingest` -* `node.master` -* `node.ml` -* `node.remote_cluster_client` -* `node.transform` -* `node.voting_only` - -have been removed. Instead, use the `node.roles` setting. If you were previously -using the legacy role settings on a 7.13 or later cluster, you will have a -deprecation log message on each of your nodes indicating the exact replacement -value for `node.roles`. - -*Impact* + -Discontinue use of the removed settings. Specifying these settings in -`elasticsearch.yml` will result in an error on startup. -==== - -[[system-call-filter-setting]] -.The system call filter setting has been removed. -[%collapsible] -==== -*Details* + -Elasticsearch uses system call filters to remove its ability to fork another -process. This is useful to mitigate remote code exploits. These system call -filters are enabled by default, and were previously controlled via the setting -`bootstrap.system_call_filter`. Starting in Elasticsearch 8.0, system call -filters will be required. As such, the setting `bootstrap.system_call_filter` -was deprecated in Elasticsearch 7.13.0, and is removed as of Elasticsearch -8.0.0. - -*Impact* + -Discontinue use of the removed setting. Specifying this setting in Elasticsearch -configuration will result in an error on startup. -==== - -[[tier-filter-setting]] -.Tier filtering settings have been removed. -[%collapsible] -==== -*Details* + -The cluster and index level settings ending in `._tier` used for filtering the allocation of a shard -to a particular set of nodes have been removed. Instead, the -{ref}/data-tier-shard-filtering.html#tier-preference-allocation-filter[tier -preference setting], `index.routing.allocation.include._tier_preference` should -be used. The removed settings are: - -Cluster level settings: - -- `cluster.routing.allocation.include._tier` -- `cluster.routing.allocation.exclude._tier` -- `cluster.routing.allocation.require._tier` - -Index settings: - -- `index.routing.allocation.include._tier` -- `index.routing.allocation.exclude._tier` -- `index.routing.allocation.require._tier` - -*Impact* + -Discontinue use of the removed settings. Specifying any of these cluster settings in Elasticsearch -configuration will result in an error on startup. Any indices using these settings will have the -settings archived (and they will have no effect) when the index metadata is loaded. -==== - -[[shared-data-path-setting]] -.Shared data path and per index data path settings are deprecated. -[%collapsible] -==== -*Details* + -Elasticsearch uses the shared data path as the base path of per index data -paths. This feature was previously used with shared replicas. Starting in -7.13.0, these settings are deprecated. Starting in 8.0 only existing -indices created in 7.x will be capable of using the shared data path and -per index data path settings. - -*Impact* + -Discontinue use of the deprecated settings. -==== - -[[single-data-node-watermark-setting]] -.The single data node watermark setting is deprecated and now only accepts `true`. -[%collapsible] -==== -*Details* + -In 7.14, setting `cluster.routing.allocation.disk.watermark.enable_for_single_data_node` -to false was deprecated. Starting in 8.0, the only legal value will be -true. In a future release, the setting will be removed completely, with same -behavior as if the setting was `true`. - -If the old behavior is desired for a single data node cluster, disk based -allocation can be disabled by setting -`cluster.routing.allocation.disk.threshold_enabled: false` - -*Impact* + -Discontinue use of the deprecated setting. -==== - -[[auto-import-dangling-indices-removed]] -.The `gateway.auto_import_dangling_indices` setting has been removed. -[%collapsible] -==== -*Details* + -The `gateway.auto_import_dangling_indices` cluster setting has been removed. -Previously, you could use this setting to automatically import -{ref}/modules-gateway.html#dangling-indices[dangling indices]. However, -automatically importing dangling indices is unsafe. Use the -{ref}/indices.html#dangling-indices-api[dangling indices APIs] to manage and -import dangling indices instead. - -*Impact* + -Discontinue use of the removed setting. Specifying the setting in -`elasticsearch.yml` will result in an error on startup. -==== - -.The `listener` thread pool has been removed. -[%collapsible] -==== -*Details* + -Previously, the transport client used the thread pool to ensure listeners aren't -called back on network threads. The transport client has been removed -in 8.0, and the thread pool is no longer needed. - -*Impact* + -Remove `listener` thread pool settings from `elasticsearch.yml` for any nodes. -Specifying `listener` thread pool settings in `elasticsearch.yml` will result in -an error on startup. -==== - -.The `fixed_auto_queue_size` thread pool type has been removed. -[%collapsible] -==== -*Details* + -The `fixed_auto_queue_size` thread pool type, previously marked as an -experimental feature, was deprecated in 7.x and has been removed in 8.0. -The `search` and `search_throttled` thread pools have the `fixed` type now. - -*Impact* + -No action needed. -==== - -.Several `transport` settings have been replaced. -[%collapsible] -==== -*Details* + -The following settings have been deprecated in 7.x and removed in 8.0. Each setting has a replacement -setting that was introduced in 6.7. - -- `transport.tcp.port` replaced by `transport.port` -- `transport.tcp.compress` replaced by `transport.compress` -- `transport.tcp.connect_timeout` replaced by `transport.connect_timeout` -- `transport.tcp_no_delay` replaced by `transport.tcp.no_delay` -- `transport.profiles.profile_name.tcp_no_delay` replaced by `transport.profiles.profile_name.tcp.no_delay` -- `transport.profiles.profile_name.tcp_keep_alive` replaced by `transport.profiles.profile_name.tcp.keep_alive` -- `transport.profiles.profile_name.reuse_address` replaced by `transport.profiles.profile_name.tcp.reuse_address` -- `transport.profiles.profile_name.send_buffer_size` replaced by `transport.profiles.profile_name.tcp.send_buffer_size` -- `transport.profiles.profile_name.receive_buffer_size` replaced by `transport.profiles.profile_name.tcp.receive_buffer_size` - -*Impact* + -Use the replacement settings. Discontinue use of the removed settings. -Specifying the removed settings in `elasticsearch.yml` will result in an error -on startup. -==== - -.Selective transport compression has been enabled by default. -[%collapsible] -==== -*Details* + -Prior to 8.0, transport compression was disabled by default. Starting in 8.0, -`transport.compress` defaults to `indexing_data`. This configuration means that -the propagation of raw indexing data will be compressed between nodes. - -*Impact* + -Inter-node transit will get reduced along the indexing path. In some scenarios, -CPU usage could increase. -==== - -.Transport compression defaults to lz4. -[%collapsible] -==== -*Details* + -Prior to 8.0, the `transport.compression_scheme` setting defaulted to `deflate`. Starting in -8.0, `transport.compress_scheme` defaults to `lz4`. - -Prior to 8.0, the `cluster.remote..transport.compression_scheme` -setting defaulted to `deflate` when `cluster.remote..transport.compress` -was explicitly configured. Starting in 8.0, -`cluster.remote..transport.compression_scheme` will fallback to -`transport.compression_scheme` by default. - -*Impact* + -This configuration means that transport compression will produce somewhat lower -compression ratios in exchange for lower CPU load. -==== - -.The `repositories.fs.compress` node-level setting has been removed. -[%collapsible] -==== -*Details* + -For shared file system repositories (`"type": "fs"`), the node level setting `repositories.fs.compress` could -previously be used to enable compression for all shared file system repositories where `compress` was not specified. -The `repositories.fs.compress` setting has been removed. - -*Impact* + -Discontinue use of the `repositories.fs.compress` node-level setting. Use the -repository-specific `compress` setting to enable compression instead. Refer to -{ref}/snapshots-filesystem-repository.html#filesystem-repository-settings[Shared -file system repository settings]. -==== - -// This change is not notable because it should not have any impact on upgrades -// However we document it here out of an abundance of caution -[[fips-default-hash-changed]] -.When FIPS mode is enabled the default password hash is now PBKDF2_STRETCH -[%collapsible] -==== -*Details* + -If `xpack.security.fips_mode.enabled` is true (see <>), -the value of `xpack.security.authc.password_hashing.algorithm` now defaults to -`pbkdf2_stretch`. - -In earlier versions this setting would always default to `bcrypt` and a runtime -check would prevent a node from starting unless the value was explicitly set to -a "pbkdf2" variant. - -There is no change for clusters that do not enable FIPS 140 mode. - -*Impact* + -This change should not have any impact on upgraded nodes. -Any node with an explicitly configured value for the password hashing algorithm -will continue to use that configured value. -Any node that did not have an explicitly configured password hashing algorithm in -{es} 6.x or {es} 7.x would have failed to start. -==== - -.The `xpack.monitoring.history.duration` will not delete indices created by metricbeat or elastic agent -[%collapsible] -==== -*Details* + - -Prior to 8.0, Elasticsearch would internally handle removal of all monitoring indices according to the -`xpack.monitoring.history.duration` setting. - -When using metricbeat or elastic agent >= 8.0 to collect monitoring data, indices are managed via an ILM policy. If the setting is present, the policy will be created using the `xpack.monitoring.history.duration` as an initial retention period. - -If you need to customize retention settings for monitoring data collected with metricbeat, please update the `.monitoring-8-ilm-policy` ILM policy directly. - -The `xpack.monitoring.history.duration` setting will only apply to monitoring indices written using (legacy) internal -collection, not indices created by metricbeat or agent. - -*Impact* + -After upgrading, insure that the `.monitoring-8-ilm-policy` ILM policy aligns with your desired retention settings. - -If you only use -metricbeat or agent to collect monitoring data, you can also remove any custom `xpack.monitoring.history.duration` -settings. - -==== diff --git a/docs/reference/migration/migrate_8_0/command-line-tool-changes.asciidoc b/docs/reference/migration/migrate_8_0/command-line-tool-changes.asciidoc deleted file mode 100644 index 7af28a1ae95cc..0000000000000 --- a/docs/reference/migration/migrate_8_0/command-line-tool-changes.asciidoc +++ /dev/null @@ -1,20 +0,0 @@ -[discrete] -[[breaking_80_command_line_tool_changes]] -==== Command line tool changes - -TIP: {ess-skip-section} - -[[migrate-tool-removed]] -.The `elasticsearch-migrate` tool has been removed. -[%collapsible] -==== -*Details* + -The `elasticsearch-migrate` tool provided a way to convert file -realm users and roles into the native realm. It has been deprecated -since {es} 7.2.0. Users and roles should now be created in the native -realm directly. - -*Impact* + -Discontinue use of the `elasticsearch-migrate` tool. Attempts to use the -`elasticsearch-migrate` tool will result in an error. -==== diff --git a/docs/reference/migration/migrate_8_0/index-setting-changes.asciidoc b/docs/reference/migration/migrate_8_0/index-setting-changes.asciidoc deleted file mode 100644 index 60e5588a187a7..0000000000000 --- a/docs/reference/migration/migrate_8_0/index-setting-changes.asciidoc +++ /dev/null @@ -1,122 +0,0 @@ -[discrete] -[[breaking_80_index_setting_changes]] -==== Index setting changes - -[[deprecation-system-indices]] -.Direct access to system indices is deprecated. -[%collapsible] -==== -*Details* + -Directly accessing system indices is deprecated, and may be prevented in a -future version. If you must access a system index, create a security role with -an index permission that targets the specific index and set the -`allow_restricted_indices` permission to `true`. Refer to -{ref}/defining-roles.html#roles-indices-priv[indices privileges] for -information on adding this permission to an index privilege. - -*Impact* + -Accessing system indices directly results in warnings in the header of API -responses. If available, use {kib} or the associated feature's {es} APIs to -manage the data that you want to access. -==== - -[[deprecate-max-merge-at-once-explicit-setting]] -.`index.merge.policy.max_merge_at_once_explicit` is deprecated and has no effect. -[%collapsible] -==== -*Details* + -The `index.merge.policy.max_merge_at_once_explicit` index setting is deprecated -and has no effect. - -Previously, you could specify `index.merge.policy.max_merge_at_once_explicit` to -set the maximum number of segments to merge at the same time during a force -merge or when expunging deleted documents. In 8.0, this number is unlimited, -regardless of the setting. - -*Impact* + -Specifying `index.merge.policy.max_merge_at_once_explicit` will have no effect -but will generate deprecation warnings. - -To avoid these deprecation warnings, discontinue use of the setting. Don't -specify the setting when creating new indices, and remove the setting from -index and component templates. - -To remove the setting from an existing data stream or index, specify the -setting's value as `null` using the update index settings API. - -[source,console] ----- -PUT my-index-000001/_settings -{ - "index.merge.policy.max_merge_at_once_explicit": null -} ----- -// TEST[setup:my_index] - -==== - -[[index-max-adjacency-matrix-filters-removed]] -.The `index.max_adjacency_matrix_filters` index setting has been removed. -[%collapsible] -==== -*Details* + -The `index.max_adjacency_matrix_filters` index setting has been removed. -Previously, you could use this setting to configure the maximum number of -filters for the -{ref}/search-aggregations-bucket-adjacency-matrix-aggregation.html[adjacency -matrix aggregation]. The `indices.query.bool.max_clause_count` index setting now -determines the maximum number of filters for the aggregation. - -*Impact* + -Discontinue use of the `index.max_adjacency_matrix_filters` index setting. - -Requests that include the index setting will return an error. If you upgrade a -cluster with a 7.x index that already contains the setting, {es} will -{ref}/archived-settings.html#archived-index-settings[archive the setting]. - -Remove the index setting from index and component templates. Attempts to use a -template that contains the setting will fail and return an error. This includes -automated operations, such the {ilm-init} rollover action. -==== - -.The `index.force_memory_term_dictionary` setting has been removed. -[%collapsible] -==== -*Details* + -The `index.force_memory_term_dictionary` setting was introduced in 7.0 as a -temporary measure to allow users to opt-out of the optimization that leaves the -term dictionary on disk when appropriate. This optimization is now mandatory -and the setting is removed. - -*Impact* + -Discontinue use of the `index.force_memory_term_dictionary` index setting. -Requests that include this setting will return an error. -==== - -.The `index.soft_deletes.enabled` setting has been removed. -[%collapsible] -==== -*Details* + -Creating indices with soft deletes disabled was deprecated in 7.6 and -is no longer supported in 8.0. The `index.soft_deletes.enabled` setting -can no longer be set to `false`. - -*Impact* + -Discontinue use of the `index.soft_deletes.enabled` index setting. Requests that -set `index.soft_deletes.enabled` to `false` will return an error. -==== - -.The `index.translog.retention.age` and `index.translog.retention.size` settings have been removed. -[%collapsible] -==== -*Details* + -Translog retention settings `index.translog.retention.age` and -`index.translog.retention.size` were effectively ignored in 7.4, deprecated in -7.7, and removed in 8.0 in favor of -{ref}/index-modules-history-retention.html[soft deletes]. - -*Impact* + -Discontinue use of the `index.translog.retention.age` and -`index.translog.retention.size` index settings. Requests that -include these settings will return an error. -==== diff --git a/docs/reference/migration/migrate_8_0/java-api-changes.asciidoc b/docs/reference/migration/migrate_8_0/java-api-changes.asciidoc deleted file mode 100644 index 22e1caf1bf5e4..0000000000000 --- a/docs/reference/migration/migrate_8_0/java-api-changes.asciidoc +++ /dev/null @@ -1,50 +0,0 @@ -[discrete] -[[breaking_80_java_api_changes]] -==== Java API changes - -[[ilm-hlrc-rename]] -.The `indexlifecycle` package has been renamed `ilm` in the Java High Level REST Client. -[%collapsible] -==== -*Details* + -In the high level REST client, the `indexlifecycle` package has been -renamed to `ilm` to match the package rename inside the {es} code. - -*Impact* + -Update your workflow and applications to use the `ilm` package in place of -`indexlifecycle`. -==== - -.Changes to `Fuzziness`. -[%collapsible] -==== -*Details* + -To create `Fuzziness` instances, use the `fromString` and `fromEdits` method -instead of the `build` method that used to accept both Strings and numeric -values. Several fuzziness setters on query builders (e.g. -MatchQueryBuilder#fuzziness) now accept only a `Fuzziness` instance instead of -an Object. - -Fuzziness used to be lenient when it comes to parsing arbitrary numeric values -while silently truncating them to one of the three allowed edit distances 0, 1 -or 2. This leniency is now removed and the class will throw errors when trying -to construct an instance with another value (e.g. floats like 1.3 used to get -accepted but truncated to 1). - -*Impact* + -Use the available constants (e.g. `Fuzziness.ONE`, `Fuzziness.AUTO`) or build -your own instance using the above mentioned factory methods. Use only allowed -`Fuzziness` values. -==== - -.Changes to `Repository`. -[%collapsible] -==== -*Details* + -Repository has no dependency on IndexShard anymore. The contract of restoreShard -and snapshotShard has been reduced to Store and MappingService in order to improve -testability. - -*Impact* + -No action needed. -==== diff --git a/docs/reference/migration/migrate_8_0/jvm-option-changes.asciidoc b/docs/reference/migration/migrate_8_0/jvm-option-changes.asciidoc deleted file mode 100644 index bdcffe4667ca4..0000000000000 --- a/docs/reference/migration/migrate_8_0/jvm-option-changes.asciidoc +++ /dev/null @@ -1,54 +0,0 @@ -[discrete] -[[breaking_80_jvm_option_changes]] -==== JVM option changes - -TIP: {ess-skip-section} - -[[breaking_80_allocation_change_flood_stage_block_always_removed]] -.`es.disk.auto_release_flood_stage_block` has been removed. -[%collapsible] -==== -*Details* + -If a node exceeds the flood-stage disk watermark then we add a block to all of -its indices to prevent further writes as a last-ditch attempt to prevent the -node completely exhausting its disk space. By default, from 7.4 onwards the -block is automatically removed when a node drops below the high watermark -again, but this behaviour could be disabled by setting the system property -`es.disk.auto_release_flood_stage_block` to `false`. This behaviour is no -longer optional, and this system property must now not be set. - -*Impact* + -Discontinue use of the `es.disk.auto_release_flood_stage_block` system property. -Setting this system property will result in an error on startup. -==== - -.`es.rest.url_plus_as_space` has been removed. -[%collapsible] -==== -*Details* + -Starting in version 7.4, a `+` in a URL will be encoded as `%2B` by all REST API functionality. Prior versions handled a `+` as a single space. -In these previous versions, if your application required handling `+` as a single space, you could return to the old behaviour by setting the system property -`es.rest.url_plus_as_space` to `true`. Note that this behaviour is deprecated and setting this system property to `true` will cease -to be supported in version 8. - -*Impact* + -Update your application or workflow to assume `+` in a URL is encoded as `%2B`. -==== - -.`es.unsafely_permit_handshake_from_incompatible_builds` has been removed. -[%collapsible] -==== -*Details* + -{es} has a check that verifies that communicating pairs of nodes of the same -version are running exactly the same build and therefore using the same wire -format as each other. In previous versions this check can be bypassed by -setting the system property -`es.unsafely_permit_handshake_from_incompatible_builds` to `true`. The use of -this system property is now forbidden. - -*Impact* + -Discontinue use of the `es.unsafely_permit_handshake_from_incompatible_builds` -system property, and ensure that all nodes of the same version are running -exactly the same build. Setting this system property will result in an error -on startup. -==== diff --git a/docs/reference/migration/migrate_8_0/logging-changes.asciidoc b/docs/reference/migration/migrate_8_0/logging-changes.asciidoc deleted file mode 100644 index 63c025746a64c..0000000000000 --- a/docs/reference/migration/migrate_8_0/logging-changes.asciidoc +++ /dev/null @@ -1,53 +0,0 @@ -[discrete] -[[breaking_80_logging_changes]] -==== Logging changes - -.{es} JSON logs now comply with ECS. -[%collapsible] -==== -*Details* + -{es}'s {ref}/logging.html[JSON logs] now comply with the -{ecs-ref}/index.html[Elastic Common Schema (ECS)]. Previously, {es}'s JSON logs -used a custom schema. - -*Impact* + -If your application parses {es}'s JSON logs, update it to support the new ECS -format. -==== - -.{es} no longer emits deprecation logs or slow logs in plaintext. -[%collapsible] -==== -*Details* + -{es} no longer emits a plaintext version of the following logs: - -* Deprecation logs -* Indexing slow logs -* Search slow logs - -These logs are now only available in JSON. - -Server logs are still available in both a JSON and plaintext format. - -*Impact* + -If your application parses {es}'s plaintext logs, update it to use the new ECS -JSON logs. -==== - -[[audit-logs-are-rolled-over-and-archived-by-size]] -.Audit logs are rolled-over and archived by size. -[%collapsible] -==== -*Details* + -In addition to the existing daily rollover, the security audit logs are -now rolled-over by disk size limit as well. Moreover, the rolled-over logs -are also gzip compressed. - -*Impact* + -The names of rolled over audit log files (but not the name of the current log) -have changed. -If you've set up automated tools to consume these files, you must configure them -to use the new names and to possibly account for `gzip` archives instead of -plain text. The Docker build of {es} is not affected because it logs on `stdout`, -where rollover is not performed. -==== diff --git a/docs/reference/migration/migrate_8_0/mapping-changes.asciidoc b/docs/reference/migration/migrate_8_0/mapping-changes.asciidoc deleted file mode 100644 index 7b3922cf0a5dd..0000000000000 --- a/docs/reference/migration/migrate_8_0/mapping-changes.asciidoc +++ /dev/null @@ -1,133 +0,0 @@ -[discrete] -[[breaking_80_mapping_changes]] -==== Mapping changes - -.Indices created in {es} 6.x and earlier versions are not supported. -[%collapsible] -==== -*Details* + -Elasticsearch 8.0 can read indices created in version 7.0 or above. An -Elasticsearch 8.0 node will not start in the presence of indices created in a -version of Elasticsearch before 7.0. - -*Impact* + -Reindex indices created in {es} 6.x or before with {es} 7.x if they need to be carried forward to {es} 8.x. -==== - -.Closed indices created in {es} 6.x and earlier versions are not supported. -[%collapsible] -==== -*Details* + -In earlier versions a node would start up even if it had data from indices -created in a version before the previous major version, as long as those -indices were closed. {es} now ensures that it is compatible with every index, -open or closed, at startup time. - -*Impact* + -Reindex closed indices created in {es} 6.x or before with {es} 7.x if they need -to be carried forward to {es} 8.x. -==== - -.The maximum number of completion contexts per field is now 10. -[%collapsible] -==== -*Details* + -The number of completion contexts within a single completion field -has been limited to 10. - -*Impact* + -Use a maximum of 10 completion contexts in a completion field. Specifying more -than 10 completion contexts will return an error. -==== - -.Multi-fields within multi-fields is no longer supported. -[%collapsible] -==== -*Details* + -Previously, it was possible to define a multi-field within a multi-field. -Defining chained multi-fields was deprecated in 7.3 and is now no longer -supported. - -*Impact* + -To migrate mappings, all instances of `fields` that occur within -a `fields` block should be removed, either by flattening the chained `fields` -blocks into a single level, or by switching to `copy_to` if appropriate. -==== - -[[fieldnames-enabling]] -.The `_field_names` metadata field's `enabled` parameter has been removed. -[%collapsible] -==== -*Details* + -The setting has been deprecated with 7.5 and is no longer supported on new indices. -Mappings for older indices will continue to work but emit a deprecation warning. - -*Impact* + -The `enabled` setting for `_field_names` should be removed from templates and mappings. -Disabling _field_names is not necessary because it no longer carries a large index overhead. -==== - -[[mapping-boosts]] -.The `boost` parameter on field mappings has been removed. -[%collapsible] -==== -*Details* + -Index-time boosts have been deprecated since the 5x line, but it was still possible -to declare field-specific boosts in the mappings. This is now removed completely. -Indexes built in 7x that contain mapping boosts will emit warnings, and the boosts -will have no effect in 8.0. New indexes will not permit boosts to be set in their -mappings at all. - -*Impact* + -The `boost` setting should be removed from templates and mappings. Use boosts -directly on queries instead. -==== - -.Java-time date formats replace joda-time formats. -[%collapsible] -==== -*Details* + -In 7.0, {es} switched from joda time to java time for date-related parsing, -formatting, and calculations. Indices created in 7.0 and later versions are -already required to use mappings with java-time date formats. However, -earlier indices using joda-time formats must be reindexed to use -mappings with java-time formats. - -*Impact* + -For a detailed migration guide, see the {ref}/migrate-to-java-time.html[Java -time migration guide]. -==== - -[[geo-shape-strategy]] -.Several `geo_shape` mapping parameters have been removed. -[%collapsible] -==== -*Details* + -The following `geo_shape` mapping parameters were deprecated in 6.6: - -* `tree` -* `tree_levels` -* `strategy` -* `distance_error_pct` - -These parameters have been removed in 8.0.0. - -*Impact* + -In 8.0, you can no longer create mappings that include these parameters. -However, 7.x indices that use these mapping parameters will continue to work. -==== - -.The `sparse_vector` field data type has been removed. -[%collapsible] -==== -*Details* + -The `sparse_vector` field type was deprecated in 7.6 and is now removed in -8.0. We have not seen much interest in this experimental field type, and don't -see a clear use case as it's currently designed. If you have feedback or -suggestions around sparse vector functionality, please let us know through -GitHub or the 'discuss' forums. - -*Impact* + -Discontinue use of the `sparse_vector` field data type. Requests containing -a mapping for this field data type will return an error. -==== diff --git a/docs/reference/migration/migrate_8_0/migrate_to_java_time.asciidoc b/docs/reference/migration/migrate_8_0/migrate_to_java_time.asciidoc deleted file mode 100644 index c86eddc04c013..0000000000000 --- a/docs/reference/migration/migrate_8_0/migrate_to_java_time.asciidoc +++ /dev/null @@ -1,314 +0,0 @@ -[[migrate-to-java-time]] -=== Java time migration guide - -With 7.0, {es} switched from joda time to java time for date-related parsing, -formatting, and calculations. This guide is designed to help you determine -if your cluster is impacted and, if so, prepare for the upgrade. - - -[discrete] -[[java-time-convert-date-formats]] -==== Convert date formats - -To upgrade to {es} 8, you'll need to convert any joda-time date formats -to their java-time equivalents. - -[discrete] -[[java-time-migration-impacted-features]] -=== Impacted features -The switch to java time only impacts custom <> and -<> formats. - -These formats are commonly used in: - -* <> -* <> -* <> - -If you don't use custom date formats, you can skip the rest of this guide. -Most custom date formats are compatible. However, several require -an update. - -To see if your date format is impacted, use the <> -or the {kibana-ref-all}/{prev-major-last}/upgrade-assistant.html[Kibana Upgrade Assistant]. - -[discrete] -[[java-time-migration-incompatible-date-formats]] -=== Incompatible date formats -Custom date formats containing the following joda-time literals should be -migrated. - -`Y` (Year of era):: -+ --- -Replace with `y`. - -*Example:* -`YYYY-MM-dd` should become `yyyy-MM-dd`. - -In java time, `Y` is used for -https://docs.oracle.com/javase/8/docs/api/java/time/temporal/WeekFields.html[week-based year]. -Using `Y` in place of `y` could result in off-by-one errors in year calculation. - -For pattern `YYYY-ww` and date `2019-01-01T00:00:00.000Z` will give `2019-01` -For pattern `YYYY-ww` and date `2018-12-31T00:00:00.000Z` will give `2019-01` (counter-intuitive) because there is >4 days of that week in 2019 --- - -`y` (Year):: -+ --- -Replace with `u`. - -*Example:* -`yyyy-MM-dd` should become `uuuu-MM-dd`. - -In java time, `y` is used for year of era. `u` can contain non-positive -values while `y` cannot. `y` can also be associated with an era field. --- - - -`C` (Century of era):: -+ --- -Century of era is not supported in java time. -There is no replacement. Instead, we recommend you preprocess your input. --- - -`x` (Week year):: -+ --- -Replace with `Y`. - -In java time, `x` means https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html[zone-offset]. - -[WARNING] -==== -Failure to properly convert `x` (Week year) to `Y` could result in data loss. -==== --- - -`Z` (Zone offset/id):: -+ --- -Replace with multiple `X`'s. - -`Z` has a similar meaning in java time. However, java time expects different -numbers of literals to parse different forms. - -Consider migrating to `X`, which gives you more control over how time is parsed. -For example, the joda-time format `YYYY-MM-dd'T'hh:mm:ssZZ` accepts the following dates: - -``` -2010-01-01T01:02:03Z -2010-01-01T01:02:03+01 -2010-01-01T01:02:03+01:02 -2010-01-01T01:02:03+01:02:03 -``` - -In java time, you cannot parse all these dates using a single format -Instead, you must specify 3 separate formats: - -``` -2010-01-01T01:02:03Z -2010-01-01T01:02:03+01 -both parsed with yyyy-MM-dd'T'hh:mm:ssX - -2010-01-01T01:02:03+01:02 -yyyy-MM-dd'T'hh:mm:ssXXX - -2010-01-01T01:02:03+01:02:03 -yyyy-MM-dd'T'hh:mm:ssXXXXX -``` - - -The formats must then be delimited using `||`: -[source,txt] --------------------------------------------------- -yyyy-MM-dd'T'hh:mm:ssX||yyyy-MM-dd'T'hh:mm:ssXXX||yyyy-MM-dd'T'hh:mm:ssXXXXX --------------------------------------------------- - -The same applies if you expect your pattern to occur without a colon (`:`): -For example, the `YYYY-MM-dd'T'hh:mm:ssZ` format accepts the following date forms: -``` -2010-01-01T01:02:03Z -2010-01-01T01:02:03+01 -2010-01-01T01:02:03+0102 -2010-01-01T01:02:03+010203 -``` -To accept all these forms in java time, you must use the `||` delimiter: -[source,txt] --------------------------------------------------- -yyyy-MM-dd'T'hh:mm:ssX||yyyy-MM-dd'T'hh:mm:ssXX||yyyy-MM-dd'T'hh:mm:ssXXXX --------------------------------------------------- --- - -`d` (Day):: -+ --- -In java time, `d` is still interpreted as "day" but is less flexible. - -For example, the joda-time date format `YYYY-MM-dd` accepts `2010-01-01` or -`2010-01-1`. - -In java time, you must use the `||` delimiter to provide specify each format: - -[source,txt] --------------------------------------------------- -yyyy-MM-dd||yyyy-MM-d --------------------------------------------------- - -In java time, `d` also does not accept more than 2 digits. To accept days with more -than two digits, you must include a text literal in your java-time date format. -For example, to parse `2010-01-00001`, you must use the following java-time date format: - -[source,txt] --------------------------------------------------- -yyyy-MM-'000'dd --------------------------------------------------- --- - -`e` (Name of day):: -+ --- -In java time, `e` is still interpreted as "name of day" but does not parse -short- or full-text forms. - -For example, the joda-time date format `EEE YYYY-MM` accepts both -`Wed 2020-01` and `Wednesday 2020-01`. - -To accept both of these dates in java time, you must specify each format using -the `||` delimiter: - -[source,txt] --------------------------------------------------- -cccc yyyy-MM||ccc yyyy-MM --------------------------------------------------- - -The joda-time literal `E` is interpreted as "day of week." -The java-time literal `c` is interpreted as "localized day of week." -`E` does not accept full-text day formats, such as `Wednesday`. --- - -`EEEE` and similar text forms:: -+ --- -Support for full-text forms depends on the locale data provided with your Java -Development Kit (JDK) and other implementation details. We recommend you -test formats containing these patterns carefully before upgrading. --- - -`z` (Time zone text):: -+ --- -In java time, `z` outputs 'Z' for Zulu when given a UTC timezone. --- - -[discrete] -[[java-time-migration-test]] -=== Test with your data - -We strongly recommend you test any date format changes using real data before -deploying in production. - -[discrete] -[[java-time-migrate-update-mappings]] -=== Update index mappings -To update joda-time date formats in index mappings, you must create a new index -with an updated mapping and reindex your data to it. - -The following `my-index-000001` index contains a mapping for the `datetime` field, a -`date` field with a custom joda-time date format. -//// -[source,console] --------------------------------------------------- -PUT my-index-000001 -{ - "mappings": { - "properties": { - "datetime": { - "type": "date", - "format": "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis" - } - } - } -} --------------------------------------------------- -//// - -[source,console] --------------------------------------------------- -GET my-index-000001/_mapping --------------------------------------------------- -// TEST[continued] - -[source,console-result] --------------------------------------------------- -{ - "my-index-000001" : { - "mappings" : { - "properties" : { - "datetime": { - "type": "date", - "format": "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis" - } - } - } - } -} --------------------------------------------------- - - -To change the date format for the `datetime` field, create a separate index -containing an updated mapping and date format. - -For example, the following `my-index-000002` index changes the `datetime` field's -date format to `uuuu/MM/dd HH:mm:ss||uuuu/MM/dd||epoch_millis`. - -[source,console] --------------------------------------------------- -PUT my-index-000002 -{ - "mappings": { - "properties": { - "datetime": { - "type": "date", - "format": "uuuu/MM/dd HH:mm:ss||uuuu/MM/dd||epoch_millis" - } - } - } -} --------------------------------------------------- -// TEST[continued] - -Next, reindex data from the old index to the new index. - -The following <> API request reindexes data from -`my-index-000001` to `my-index-000002`. - -[source,console] --------------------------------------------------- -POST _reindex -{ - "source": { - "index": "my-index-000001" - }, - "dest": { - "index": "my-index-000002" - } -} --------------------------------------------------- -// TEST[continued] - -If you use index aliases, update them to point to the new index. - -[source,console] --------------------------------------------------- -POST /_aliases -{ - "actions" : [ - { "remove" : { "index" : "my-index-000001", "alias" : "my-index" } }, - { "add" : { "index" : "my-index-000002", "alias" : "my-index" } } - ] -} --------------------------------------------------- -// TEST[continued] diff --git a/docs/reference/migration/migrate_8_0/packaging-changes.asciidoc b/docs/reference/migration/migrate_8_0/packaging-changes.asciidoc deleted file mode 100644 index 7e0c2c72ee6d7..0000000000000 --- a/docs/reference/migration/migrate_8_0/packaging-changes.asciidoc +++ /dev/null @@ -1,60 +0,0 @@ -[discrete] -[[breaking_80_packaging_changes]] -==== Packaging changes - -TIP: {ess-skip-section} - -.The layout of the data folder has changed. -[%collapsible] -==== -*Details* + -Each node's data is now stored directly in the data directory set by the -`path.data` setting, rather than in `${path.data}/nodes/0`, because the removal -of the `node.max_local_storage_nodes` setting means that nodes may no longer -share a data path. - -*Impact* + -At startup, {es} will automatically migrate the data path to the new layout. -This automatic migration will not proceed if the data path contains data for -more than one node. You should move to a configuration in which each node has -its own data path before upgrading. - -If you try to upgrade a configuration in which there is data for more than one -node in a data path then the automatic migration will fail and {es} -will refuse to start. To resolve this you will need to perform the migration -manually. The data for the extra nodes are stored in folders named -`${path.data}/nodes/1`, `${path.data}/nodes/2` and so on, and you should move -each of these folders to an appropriate location and then configure the -corresponding node to use this location for its data path. If your nodes each -have more than one data path in their `path.data` settings then you should move -all the corresponding subfolders in parallel. Each node uses the same subfolder -(e.g. `nodes/2`) across all its data paths. -==== - -.The default Maxmind geoip databases have been removed. -[%collapsible] -==== -*Details* + -The default Maxmind geoip databases that shipped by default with Elasticsearch -have been removed. These databases are out dated and stale and using these -databases will likely result in incorrect geoip lookups. - -By default since 7.13, these pre-packaged geoip databases -were used in case no database were specified in the config directory or before -the geoip downloader downloaded the geoip databases. When the geoip database -downloader completed downloading the new databases then these pre-packaged -databases were no longer used. - -*Impact* + -If the geoip downloader is disabled and no geoip databases are provided -in the config directory of each ingest node then the geoip processor will -no longer perform geoip lookups and tag these documents with the fact that -the requested database is no longer available. - -After a cluster has been started and before the geoip downloader has completed -downloading the most up to data databases, the geoip processor will not perform -any geoip lookups and tag documents that the requested database is not available. -After the geoip downloader has completed downloading the most up to data databases -then the geoip processor will function as normal. The window of time that the -geoip processor can't do geoip lookups after cluster startup should be very small. -==== diff --git a/docs/reference/migration/migrate_8_0/painless-changes.asciidoc b/docs/reference/migration/migrate_8_0/painless-changes.asciidoc deleted file mode 100644 index 601866cb8995d..0000000000000 --- a/docs/reference/migration/migrate_8_0/painless-changes.asciidoc +++ /dev/null @@ -1,42 +0,0 @@ -[discrete] -[[breaking_80_painless_changes]] -==== Painless changes - -.The `JodaCompatibleZonedDateTime` class has been removed. -[%collapsible] -==== -*Details* + -As a transition from Joda datetime to Java datetime, scripting used -an intermediate class called `JodaCompatibleZonedDateTime`. This class -has been removed and is replaced by `ZonedDateTime`. Any use of casting -to a `JodaCompatibleZonedDateTime` or use of method calls only available -in `JodaCompatibleZonedDateTime` in a script will result in a compilation -error, and may not allow the upgraded node to start. - -*Impact* + -Before upgrading, replace `getDayOfWeek` with `getDayOfWeekEnum().value` in any -scripts. Any use of `getDayOfWeek` expecting a return value of `int` will result -in a compilation error or runtime error and may not allow the upgraded node to -start. - -The following `JodaCompatibleZonedDateTime` methods must be replaced using -`ZonedDateTime` methods prior to upgrade: - -* `getMillis()` -> `toInstant().toEpochMilli()` -* `getCenturyOfEra()` -> `get(ChronoField.YEAR_OF_ERA) / 100` -* `getEra()` -> `get(ChronoField.ERA)` -* `getHourOfDay()` -> `getHour()` -* `getMillisOfDay()` -> `get(ChronoField.MILLI_OF_DAY)` -* `getMillisOfSecond()` -> `get(ChronoField.MILLI_OF_SECOND)` -* `getMinuteOfDay()` -> `get(ChronoField.MINUTE_OF_DAY)` -* `getMinuteOfHour()` -> `getMinute()` -* `getMonthOfYear()` -> `getMonthValue()` -* `getSecondOfDay()` -> `get(ChronoField.SECOND_OF_DAY)` -* `getSecondOfMinute()` -> `getSecond()` -* `getWeekOfWeekyear()` -> `get(IsoFields.WEEK_OF_WEEK_BASED_YEAR)` -* `getWeekyear()` -> `get(IsoFields.WEEK_BASED_YEAR)` -* `getYearOfCentury()` -> `get(ChronoField.YEAR_OF_ERA) % 100` -* `getYearOfEra()` -> `get(ChronoField.YEAR_OF_ERA)` -* `toString(String)` -> a DateTimeFormatter -* `toString(String, Locale)` -> a DateTimeFormatter -==== diff --git a/docs/reference/migration/migrate_8_0/plugin-changes.asciidoc b/docs/reference/migration/migrate_8_0/plugin-changes.asciidoc deleted file mode 100644 index 42baf8f7f2a69..0000000000000 --- a/docs/reference/migration/migrate_8_0/plugin-changes.asciidoc +++ /dev/null @@ -1,64 +0,0 @@ -[discrete] -[[breaking_80_plugin_changes]] -==== Plugin changes - -TIP: {ess-skip-section} - -.The S3, GCS and Azure repository plugins are now included in Elasticsearch -[%collapsible] -==== -*Details* + -In previous versions of {es}, in order to register a snapshot repository -backed by Amazon S3, Google Cloud Storage (GCS) or Microsoft Azure Blob -Storage, you first had to install the corresponding Elasticsearch plugin, -for example `repository-s3`. These plugins are now included in {es} by -default. - -*Impact* + -You no longer need to install the following plugins, and not should attempt -to do so. - -* `repository-azure` -* `repository-gcs` -* `repository-s3` - -{es} and the `elasticsearch-plugin` CLI tool have been changed to tolerate -attempted installation and removal of these plugins in order to avoid -breaking any existing automation. In the future, attempting to install -these plugins will be an error. - -Specifically, the `elasticsearch-plugin` CLI tool will not fail if you -attempt to install any of the above plugins, and will instead print a -warning and skip the plugins. If any of these plugins are already -installed, for example because you installed them when running an older -version of {es}, then you can still remove them with -`elasticsearch-plugin`. Attempting to remove them if they are not installed -will succeed but print a warnings. - -If you run {es} using Docker and you are managing plugins using a -{plugins}/manage-plugins-using-configuration-file.html[configuration file], then when -{es} first starts after you upgrade it, it will remove the above plugins if -they already installed. If any of these plugins are specified in your -configuration file, {es} will ignore them and emit a warning log message. -==== - -.Third party plugins can no longer intercept REST requests (`RestHandlerWrapper`) -[%collapsible] -==== -*Details* + -In previous versions of {es}, third-party plugins could implement the -`getRestHandlerWrapper` method to intercept all REST requests to the node. A -common use of this feature was to implement custom security plugins to replace -the built-in {security-features}. This extension point is no longer available -to third-party plugins. - - -*Impact* + -Some third-party plugins that were designed to work with earlier versions of -{es} might not be compatible with {es} version 8.0 or later. - -If you depend on any plugins that are not produced and supported by Elastic, -check with the plugin author and ensure that the plugin is available for your -target version of {es} before upgrading. - -==== diff --git a/docs/reference/migration/migrate_8_0/rest-api-changes.asciidoc b/docs/reference/migration/migrate_8_0/rest-api-changes.asciidoc deleted file mode 100644 index 99c09b9b05385..0000000000000 --- a/docs/reference/migration/migrate_8_0/rest-api-changes.asciidoc +++ /dev/null @@ -1,1138 +0,0 @@ -[discrete] -[[breaking_80_rest_api_changes]] -==== REST API changes - -.REST API endpoints containing `_xpack` have been removed. -[%collapsible] -==== -*Details* + -In 7.0, we deprecated REST endpoints that contain `_xpack` in their path. These -endpoints are now removed in 8.0. Each endpoint that was deprecated and removed -is replaced with a new endpoint that does not contain `_xpack`. As an example, -`/{index}/_xpack/graph/_explore` is replaced by `/{index}/_graph/explore`. - -*Impact* + -Use the replacement REST API endpoints. Requests submitted to the `_xpack` -API endpoints will return an error. - -*Compatibility* + -When {ref}/rest-api-compatibility.html[rest-api-compatibility] is -{ref}/rest-api-compatibility.html[requested], any requests that include -the`_xpack` prefix are rerouted to the corresponding URL without the `_xpack` -prefix. -==== - -[[remove-mapping-type-api-endpoints]] -.REST API endpoints containing mapping types have been removed. -[%collapsible] -==== -*Details* + -Mapping types have been removed. API endpoints that contain a mapping type have -also been removed. Use a typeless endpoint instead. - -[options="header",cols="<1,<3,<1"] -|==== -| API | Typed API endpoint | Typeless API endpoint - -| {ref}/docs-bulk.html[Bulk] -| `//_bulk` -| `/_bulk` - -| {ref}/search-count.html[Count] -| `//_count` -| `/_count` - -| {ref}/docs-delete.html[Delete] -| `//<_id>` -| `/_doc/<_id>` - -| {ref}/docs-delete-by-query.html[Delete by query] -| `//_delete_by_query` -| `/_delete_by_query` - -| {ref}/search-explain.html[Explain] -| `//<_id>/_explain` -| `/_explain/<_id>` - -| {ref}/docs-get.html[Get] -| `//<_id>` -| `/_doc/<_id>` - -| -| `//<_id>/_source` -| `/_source/<_id>` - -| {ref}/indices-get-field-mapping.html[Get field mapping] -| `_mapping//field/` -| `_mapping/field/` - -| -| `/_mapping//field/` -| `/_mapping/field/` - -| {ref}/indices-get-mapping.html[Get mapping] -| `_mapping/` -| `_mapping` or `/_mapping` - -| -| `//_mapping` -| `/_mapping` - -| -| `/_mapping/` -| `/_mapping` - -| {ref}/graph-explore-api.html[Graph explore] -| `//_graph/explore` -| `/_graph/explore` - -| {ref}/docs-index_.html[Index] -| `//<_id>/_create` -| `/_create/<_id>` - -| -| `/` -| `/_doc` - -| -| `//<_id>` -| `/_doc/<_id>` - -| {ref}/docs-multi-get.html[Multi get] -| `//_mget` -| `/_mget` - -| {ref}/search-multi-search.html[Multi search] -| `//_msearch` -| `/_msearch` - -| {ref}/multi-search-template.html[Multi search template] -| `//_msearch/template` -| `/_msearch/template` - -| {ref}/docs-multi-termvectors.html[Multi term vectors] -| `//_mtermvectors` -| `/_mtermvectors` - -| {ref}/rollup-search.html[Rollup search] -| `//_rollup_search` -| `/_rollup_search` - -| {ref}/search-search.html[Search] -| `//_search` -| `/_search` - -| {ref}/search-template-api.html[Search template] -| `//_search/template` -| `/_search/template` - -| {ref}/docs-termvectors.html[Term vectors] -| `//<_id>/_termvectors` -| `/_termvectors<_id>` - -| -| `//_termvectors` -| `/_termvectors` - -| {ref}/docs-update.html[Update] -| `//<_id>/_update` -| `/_update/<_id>` - -| {ref}/docs-update-by-query.html[Update by query] -| `//_update_by_query` -| `/_update_by_query` - -| {ref}/indices-put-mapping.html[Update mapping] -| `//_mapping` -| `/_mapping` - -| -| `/_mapping/` -| `/_mapping` - -| -| `_mapping/` -| `/_mapping` - -| {ref}/search-validate.html[Validate] -| `//_validate/query` -| `/_validate/query` - -|==== - -*Impact* + -Update your application to use typeless REST API endpoints. Requests to -endpoints that contain a mapping type will return an error. - -*Compatibility* + -When {ref}/rest-api-compatibility.html[rest-api-compatibility] is -{ref}/rest-api-compatibility.html[requested], if a request includes a custom -mapping type it is ignored. The request is rerouted to the corresponding -typeless URL. Custom mapping types in request bodies and type related HTTP -parameters are ignored, and responses, where warranted, include `_type` : -`_doc`. - -==== - -.{ccs-cap} ({ccs-init}) is now only backward-compatible with the previous minor version. -[%collapsible] -==== -*Details* + -In 8.0+, Elastic supports searches from a local cluster to a remote cluster -running: - -* The previous minor version. -* The same version. -* A newer minor version in the same major version. - -Elastic also supports searches from a local cluster running the last minor -version of a major version to a remote cluster running any minor version in the -following major version. For example, a local 7.17 cluster can search any -remote 8.x cluster. - -include::{es-ref-dir}/search/search-your-data/ccs-version-compat-matrix.asciidoc[] - -IMPORTANT: For the {ref}/eql-search-api.html[EQL search API], the local and -remote clusters must use the same {es} version if they have versions prior to 7.17.7 (included) or prior to 8.5.1 (included). - -For example, a local 8.0 cluster can search a remote 7.17 or any remote 8.x -cluster. However, a search from a local 8.0 cluster to a remote 7.16 or 6.8 -cluster is not supported. - -Previously, we also supported searches on remote clusters running: - -* Any minor version of the local cluster's major version. -* The last minor release of the previous major version. - -However, such searches can result in undefined behavior. - -*Impact* + -If you only run cross-cluster searches on remote clusters using the same or a -newer version, no changes are needed. - -If you previously searched remote clusters running an earlier version of {es}, -see {ref}/modules-cross-cluster-search.html#ensure-ccs-support[Ensure {ccs} -support] for recommended solutions. - -A {ccs} using an unsupported configuration may still work. However, such -searches aren't tested by Elastic, and their behavior isn't guaranteed. -==== - -[[remove-term-order-key]] -.The `terms` aggregation no longer supports the `_term` order key. -[%collapsible] -==== -*Details* + -The `terms` aggregation no longer supports the `_term` key in `order` values. To -sort buckets by their term, use `_key` instead. - -*Impact* + -Discontinue use of the `_term` order key. Requests that include a `_term` order -key will return an error. - -*Compatibility* + -When {ref}/rest-api-compatibility.html[rest-api-compatibility] is -{ref}/rest-api-compatibility.html[requested], the `_term` order is ignored and -`key` is used instead. -==== - -[[remove-time-order-key]] -.The `date_histogram` aggregation no longer supports the `_time` order key. -[%collapsible] -==== -*Details* + -The `date_histogram` aggregation no longer supports the `_time` key in `order` -values. To sort buckets by their key, use `_key` instead. - -*Impact* + -Discontinue use of the `_time` order key. Requests that include a `_time` order -key will return an error. - -*Compatibility* + -When {ref}/rest-api-compatibility.html[rest-api-compatibility] is -{ref}/rest-api-compatibility.html[requested], the `_time` order is ignored and -`_key` is used instead. -==== - -[[remove-moving-avg-agg]] -.The `moving_avg` aggregation has been removed. -[%collapsible] -==== -*Details* + -The `moving_avg` aggregation was deprecated in 6.4 and has been removed. To -calculate moving averages, use the -{ref}/search-aggregations-pipeline-movfn-aggregation.html[`moving_fn` -aggregation] instead. - -*Impact* + -Discontinue use of the `moving_avg` aggregation. Requests that include the -`moving_avg` aggregation will return an error. - - -==== - -[[percentile-duplication]] -.The `percentiles` aggregation's `percents` parameter no longer supports duplicate values. -[%collapsible] -==== -*Details* + -If you specify the `percents` parameter with the -{ref}/search-aggregations-metrics-percentile-aggregation.html[`percentiles` aggregation], -its values must be unique. Otherwise, an exception occurs. - -*Impact* + -Use unique values in the `percents` parameter of the `percentiles` aggregation. -Requests containing duplicate values in the `percents` parameter will return -an error. - -==== - -[[date-histogram-interval]] -.The `date_histogram` aggregation's `interval` parameter is no longer valid. -[%collapsible] -==== -*Details* + -It is now an error to specify the `interval` parameter to the -{ref}/search-aggregations-bucket-datehistogram-aggregation.html[`date_histogram` -aggregation] or the -{ref}/search-aggregations-bucket-composite-aggregation.html#_date_histogram[`composite -date_histogram` source. Instead, please use either `calendar_interval` or -`fixed_interval` as appropriate. - -*Impact* + -Uses of the `interval` parameter in either the `date_histogram` aggregation or -the `date_histogram` composite source will now generate an error. Instead -please use the more specific `fixed_interval` or `calendar_interval` -parameters. - -*Compatibility* + -When {ref}/rest-api-compatibility.html[rest-api-compatibility] is -{ref}/rest-api-compatibility.html[requested], the `interval` parameter is -adapted to either a fixed or calendar interval. -==== - -[[ngram-edgengram-filter-names-removed]] -.The `nGram` and `edgeNGram` token filter names have been removed. -[%collapsible] -==== -*Details* + -The `nGram` and `edgeNGram` token filter names that have been deprecated since -version 6.4 have been removed. Both token filters can only be used by their -alternative names `ngram` and `edge_ngram` since version 7.0. - -*Impact* + -Use the equivalent `ngram` and `edge_ngram` token filters. Requests containing -the `nGram` and `edgeNGram` token filter names will return an error. -==== - -[[nGram-edgeNGram-tokenizer-dreprecation]] -.The `nGram` and `edgeNGram` tokenizer names have been removed. -[%collapsible] -==== -*Details* + -The `nGram` and `edgeNGram` tokenizer names haven been deprecated with 7.6 and are no longer -supported on new indices. Mappings for indices created after 7.6 will continue to work but -emit a deprecation warning. The tokenizer name should be changed to the fully equivalent -`ngram` or `edge_ngram` names for new indices and in index templates. - -*Impact* + -Use the `ngram` and `edge_ngram` tokenizers. Requests to create new indices -using the `nGram` and `edgeNGram` tokenizer names will return an error. -==== - -.The `in_flight_requests` stat has been renamed `inflight_requests` in logs and diagnostic APIs. -[%collapsible] -==== -*Details* + -The name of the in flight requests circuit breaker in log output and diagnostic APIs (such as the node stats API) changes from `in_flight_requests` to `inflight_requests` to align it with the name of the corresponding settings. - -*Impact* + -Update your workflow and applications to use the `inflight_requests` stat in -place of `in_flight_requests`. -==== - -.The voting configuration exclusions API endpoint has changed. -[%collapsible] -==== -*Details* + -The `POST /_cluster/voting_config_exclusions/{node_filter}` API has been -removed in favour of `POST /_cluster/voting_config_exclusions?node_names=...` -and `POST /_cluster/voting_config_exclusions?node_ids=...` which allow you to -specify the names or IDs of the nodes to exclude. - -*Impact* + -Use `POST /_cluster/voting_config_exclusions?node_ids=...` and specify the nodes -to exclude instead of using a node filter. Requests submitted to the -`/_cluster/voting_config_exclusions/{node_filter}` endpoint will return an -error. -==== - -.Remote system indices are not followed automatically if they match an auto-follow pattern. -[%collapsible] -==== -*Details* + -Remote system indices matching an {ref}/ccr-auto-follow.html[auto-follow -pattern] won't be configured as a follower index automatically. - -*Impact* + -Explicitly {ref}/ccr-put-follow.html[create a follower index] to follow a remote -system index if that's the wanted behaviour. -==== - -.The EQL `wildcard` function has been removed. -[%collapsible] -==== -*Details* + -The `wildcard` function was deprecated in {es} 7.13.0 and has been removed. - -*Impact* + -Use the `like` or `regex` {ref}/eql-syntax.html#eql-syntax-pattern-comparison-keywords[keywords] instead. -==== - -[[ilm-freeze-noop]] -.The ILM `freeze` action is now a no-op. -[%collapsible] -==== -*Details* + -The ILM freeze action is now a no-op and performs no action on the index, as the freeze API endpoint -has been removed in 8.0. - -*Impact* + -Update your ILM policies to remove the `freeze` action from the `cold` phase. -==== - -[[ilm-policy-validation]] -.Additional validation for ILM policies. -[%collapsible] -==== -*Details* + -Creating or updating an ILM policy now requires that any referenced snapshot repositories and SLM -policies exist. - -*Impact* + -Update your code or configuration management to ensure that repositories and SLM policies are created -before any policies that reference them. -==== - -.The deprecated `_upgrade` API has been removed. -[%collapsible] -==== -*Details* + -Previously, the `_upgrade` API upgraded indices from the previous major -version to the current version. The `_reindex` API should be used -instead for that purpose. - -*Impact* + -Requests made to the old `_upgrade` API will return an error. -==== - -.The deprecated freeze index API has been removed. -[%collapsible] -==== -*Details* + -The freeze index API (`POST //_freeze`) has been removed. -https://www.elastic.co/blog/significantly-decrease-your-elasticsearch-heap-memory-usage[Improvements -in heap memory usage] have eliminated the reason to freeze indices. -You can still unfreeze existing frozen indices using the -{ref}/unfreeze-index-api.html[unfreeze index API]. For some use cases, the -frozen tier may be a suitable replacement for frozen indices. See -{ref}/data-tiers.html[data tiers] for more information. - -*Impact* + -Requests made to the old freeze index API will return an error. -==== - -.The force merge API's `max_num_segments` and `only_expunge_deletes` parameters cannot both be specified in the same request. -[%collapsible] -==== -*Details* + -Previously, the force merge API allowed the parameters `only_expunge_deletes` -and `max_num_segments` to be set to a non default value at the same time. But -the `max_num_segments` was silently ignored when `only_expunge_deletes` is set -to `true`, leaving the false impression that it has been applied. - -*Impact* + -When using the {ref}/indices-forcemerge.html[force merge API], do not specify -values for both the `max_num_segments` and `only_expunge_deletes` parameters. -Requests that include values for both parameters will return an error. -==== - -.The create or update index template API's `template` parameter has been removed. -[%collapsible] -==== -*Details* + -In 6.0, we deprecated the `template` parameter in create or update index -template requests in favor of using `index_patterns`. Support for the `template` -parameter is now removed in 8.0. - -*Impact* + -Use the {ref}/indices-templates-v1.html[create or update index template API]'s -`index_patterns` parameter. Requests that include the `template` parameter will -return an error. - -*Compatibility* + -When {ref}/rest-api-compatibility.html[rest-api-compatibility] is -{ref}/rest-api-compatibility.html[requested], the `template` parameter is mapped -to `index_patterns`. -==== - -.Synced flush has been removed. -[%collapsible] -==== -*Details* + -Synced flush was deprecated in 7.6 and is removed in 8.0. Use a regular flush -instead as it has the same effect as a synced flush in 7.6 and later. - -*Impact* + -Use the {ref}/indices-flush.html[flush API]. Requests to the -`//flush/synced` or `/flush/synced` endpoints will return an error. - -*Compatibility* + -When {ref}/rest-api-compatibility.html[rest-api-compatibility] is -{ref}/rest-api-compatibility.html[requested], the request to synced flush is -routed to the equivalent non-synced flush URL. -==== - -.The default for the `?wait_for_active_shards` parameter on the close index API has changed. -[%collapsible] -==== -*Details* + -When closing an index in earlier versions, by default {es} would not wait for -the shards of the closed index to be properly assigned before returning. From -version 8.0 onwards the default behaviour is to wait for shards to be assigned -according to the -{ref}/docs-index_.html#index-wait-for-active-shards[`index.write.wait_for_active_shards` -index setting]. - -*Impact* + -Accept the new behaviour, or specify `?wait_for_active_shards=0` to preserve -the old behaviour if needed. -==== - -.The index stats API's `types` query parameter has been removed. -[%collapsible] -==== -*Details* + -The index stats API's `types` query parameter has been removed. Previously, you -could combine `types` with the `indexing` query parameter to return indexing -stats for specific mapping types. Mapping types have been removed in 8.0. - -*Impact* + -Discontinue use of the `types` query parameter. Requests that include the -parameter will return an error. - -*Compatibility* + -When {ref}/rest-api-compatibility.html[rest-api-compatibility] is -{ref}/rest-api-compatibility.html[requested], the `types` query parameter is -ignored and stats are returned for the entire index. -==== - -.The `user_agent` ingest processor's `ecs` parameter has no effect. -[%collapsible] -==== -*Details* + -In 7.2, we deprecated the `ecs` parameter for the `user_agent` ingest processor. -In 8.x, the `user_agent` ingest processor will only return {ecs-ref}[Elastic -Common Schema (ECS)] fields, regardless of the `ecs` value. - -*Impact* + -To avoid deprecation warnings, remove the parameter from your ingest pipelines. -If a pipeline specifies an `ecs` value, the value is ignored. -==== - -.The `include_type_name` query parameter has been removed. -[%collapsible] -==== -*Details* + -The `include_type_name` query parameter has been removed from the index -creation, index template, and mapping APIs. Previously, you could set -`include_type_name` to `true` to indicate that requests and responses should -include a mapping type name. Mapping types have been removed in 8.x. - -*Impact* + -Discontinue use of the `include_type_name` query parameter. Requests that -include the parameter will return an error. - -*Compatibility* + -When {ref}/rest-api-compatibility.html[rest-api-compatibility] is -{ref}/rest-api-compatibility.html[requested], the `include_type_name` query -parameter is ignored and any custom mapping types in the request are removed. -==== - -.Reindex from remote now re-encodes URL-encoded index names. -[%collapsible] -==== -*Details* + -Reindex from remote would previously allow URL-encoded index names and not -re-encode them when generating the search request for the remote host. This -leniency has been removed such that all index names are correctly encoded when -reindex generates remote search requests. - -*Impact* + -Specify unencoded index names for reindex from remote requests. -==== - -.In the reindex, delete by query, and update by query APIs, the `size` parameter has been renamed. -[%collapsible] -==== -*Details* + -Previously, a `_reindex` request had two different size specifications in the body: - -- Outer level, determining the maximum number of documents to process -- Inside the `source` element, determining the scroll/batch size. - -The outer level `size` parameter has now been renamed to `max_docs` to -avoid confusion and clarify its semantics. - -Similarly, the `size` parameter has been renamed to `max_docs` for -`_delete_by_query` and `_update_by_query` to keep the 3 interfaces consistent. - -*Impact* + -Use the replacement parameters. Requests containing the `size` parameter will -return an error. - -*Compatibility* + -When {ref}/rest-api-compatibility.html[rest-api-compatibility] is -{ref}/rest-api-compatibility.html[requested], the `size` parameter is mapped to -the `max_docs` parameter. -==== - -.The update by query API now rejects unsupported `script` fields. -[%collapsible] -==== -*Details* + -An update by query API request that includes an unsupported field in the -`script` object now returns an error. Previously, the API would silently ignore -these unsupported fields. - -*Impact* + -To avoid errors, remove unsupported fields from the `script` object. -==== - -.The cat node API's `local` query parameter has been removed. -[%collapsible] -==== -*Details* + -The `?local` parameter to the `GET _cat/nodes` API was deprecated in 7.x and is -rejected in 8.0. This parameter caused the API to use the local cluster state -to determine the nodes returned by the API rather than the cluster state from -the master, but this API requests information from each selected node -regardless of the `?local` parameter which means this API does not run in a -fully node-local fashion. - -*Impact* + -Discontinue use of the `?local` query parameter. {ref}/cat-nodes.html[cat node -API] requests that include this parameter will return an error. -==== - -.The cat shard API's `local` query parameter has been removed. -[%collapsible] -==== -*Details* + -The `?local` parameter to the `GET _cat/shards` API was deprecated in 7.x and is -rejected in 8.0. This parameter caused the API to use the local cluster state -to determine the nodes returned by the API rather than the cluster state from -the master, but this API requests information from each selected node -regardless of the `?local` parameter which means this API does not run in a -fully node-local fashion. - -*Impact* + -Discontinue use of the `?local` query parameter. {ref}/cat-shards.html[cat shards -API] requests that include this parameter will return an error. -==== - -.The cat indices API's `local` query parameter has been removed. -[%collapsible] -==== -*Details* + -The `?local` parameter to the `GET _cat/indices` API was deprecated in 7.x and is -rejected in 8.0. This parameter caused the API to use the local cluster state -to determine the nodes returned by the API rather than the cluster state from -the master, but this API requests information from each selected node -regardless of the `?local` parameter which means this API does not run in a -fully node-local fashion. - -*Impact* + -Discontinue use of the `?local` query parameter. {ref}/cat-indices.html[cat indices -API] requests that include this parameter will return an error. -==== - -.The get field mapping API's `local` query parameter has been removed. -[%collapsible] -==== -*Details* + -The `local` parameter for get field mapping API was deprecated in 7.8 and is -removed in 8.0. This parameter is a no-op and field mappings are always retrieved -locally. - -*Impact* + -Discontinue use of the `local` query parameter. -{ref}/indices-get-field-mapping.html[get field mapping API] requests that -include this parameter will return an error. -==== - -.Post data to jobs API is deprecated. -[%collapsible] -==== -*Details* + -The {ml} {ref}/ml-post-data.html[post data to jobs API] is deprecated starting in 7.11.0 -and will be removed in a future major version. - -*Impact* + -Use {ref}/ml-ad-apis.html#ml-api-datafeed-endpoint[{dfeeds}] instead. -==== - -.The `job_id` property of the Update {dfeeds} API has been removed. -[%collapsible] -==== -*Details* + -The ability to update a `job_id` in a {dfeed} was deprecated in 7.3.0. and is -removed in 8.0. - -*Impact* + -It is not possible to move {dfeeds} between {anomaly-jobs}. -==== - -.Create repository and delete repository API's return `409` status code when a repository is in use instead of `500`. -[%collapsible] -==== -*Details* + -The {ref}/put-snapshot-repo-api.html[Create or update snapshot repository API] and -{ref}/delete-snapshot-repo-api.html[Delete snapshot repository API] return `409` -status code when the request is attempting to modify an existing repository that's in use instead of status code `500`. - -*Impact* + -Update client code that handles creation and deletion of repositories to reflect this change. -==== - -.The `allow_no_datafeeds` property has been removed from {ml} APIs. -[%collapsible] -==== -*Details* + -The `allow_no_datafeeds` property was deprecated in the -{ref}/cat-datafeeds.html[cat {dfeeds}], -{ref}/ml-get-datafeed.html[get {dfeeds}], -{ref}/ml-get-datafeed-stats.html[get {dfeed} statistics], and -{ref}/ml-stop-datafeed.html[stop {dfeeds}] APIs in 7.10.0. - -*Impact* + -Use `allow_no_match` instead. -==== - -.The `allow_no_jobs` property has been removed from {ml} APIs. -[%collapsible] -==== -*Details* + -The `allow_no_jobs` property was deprecated in the -{ref}/cat-anomaly-detectors.html[cat anomaly detectors], -{ref}/ml-close-job.html[close {anomaly-jobs}], -{ref}/ml-get-job.html[get {anomaly-jobs}], -{ref}/ml-get-job-stats.html[get {anomaly-job} statistics], and -{ref}/ml-get-overall-buckets.html[get overall buckets] APIs in 7.10.0. - -*Impact* + -Use `allow_no_match` instead. -==== - -.The StartRollupJob endpoint now returns a success status if a job has already started. -[%collapsible] -==== -*Details* + -Previously, attempting to start an already-started rollup job would -result in a `500 InternalServerError: Cannot start task for Rollup Job -[job] because state was [STARTED]` exception. - -Now, attempting to start a job that is already started will just -return a successful `200 OK: started` response. - -*Impact* + -Update your workflow and applications to assume that a 200 status in response to -attempting to start a rollup job means the job is in an actively started state. -The request itself may have started the job, or it was previously running and so -the request had no effect. -==== - -.Stored scripts no longer support empty scripts or search templates. -[%collapsible] -==== -*Details* + -The {ref}/create-stored-script-api.html[create or update stored script API]'s -`source` parameter cannot be empty. - -*Impact* + -Before upgrading, use the {ref}/delete-stored-script-api.html[delete stored -script API] to delete any empty stored scripts or search templates. -In 8.0, {es} will drop any empty stored scripts or empty search templates from -the cluster state. Requests to create a stored script or search template with -an empty `source` will return an error. -==== - -.The create or update stored script API's `code` parameter has been removed. -[%collapsible] -==== -*Details* + -The {ref}/create-stored-script-api.html[create or update stored script API]'s -`code` parameter has been removed. Use the `source` parameter instead. - -*Impact* + -Discontinue use of the `code` parameter. Requests that include the parameter -will return an error. -==== - -[[_type-search-matches-no-docs]] -.Searches on the `_type` field are no longer supported. -[%collapsible] -==== -*Details* + -In 8.x, the `_type` metadata field has been removed. {es} now handles a search -on the `_type` field as a search on a non-existent field. A search on a -non-existent field matches no documents, regardless of the query string. - -In 7.x, a search for `_doc` in the `_type` field would match the same documents -as a `match_all` query. - -*Impact* + -Remove queries on the `_type` field from your search requests and search -templates. Searches that include these queries may return no results. -==== - -[[msearch-empty-line-support]] -.The multi search API now parses an empty first line as action metadata in text files. -[%collapsible] -==== -*Details* + -The multi search API now parses an empty first line as empty action metadata -when you provide a text file as the request body, such as when using curl's -`--data-binary` flag. - -The API no longer supports text files that contain: - -* An empty first line followed by a line containing only `{}`. -* An empty first line followed by another empty line. - -*Impact* + -Don't provide an unsupported text file to the multi search API. Requests that -include an unsupported file will return an error. -==== - -[[remove-unmapped-type-string]] -.The `unmapped_type: string` sort option has been removed. -[%collapsible] -==== -*Details* + -Search requests no longer support the `unmapped_type: string` sort option. -Instead, use `unmapped_type: keyword` to handle an unmapped field as if it had -the `keyword` field type but ignore its values for sorting. - -*Impact* + -Discontinue use of `unmapped_type: string`. Search requests that include the -`unmapped_type: string` sort option will return shard failures. -==== - -[[id-field-data]] -.Aggregating and sorting on `_id` is disallowed by default. -[%collapsible] -==== -*Details* + -Previously, it was possible to aggregate and sort on the built-in `_id` field -by loading an expensive data structure called fielddata. This was deprecated -in 7.6 and is now disallowed by default in 8.0. - -*Impact* + -Aggregating and sorting on `_id` should be avoided. As an alternative, the -`_id` field's contents can be duplicated into another field with docvalues -enabled (note that this does not apply to auto-generated IDs). -==== - -.The `common` query has been removed. -[%collapsible] -==== -*Details* + -The `common` query, deprecated in 7.x, has been removed in 8.0. -The same functionality can be achieved by the `match` query if the total number of hits is not tracked. - -*Impact* + -Discontinue use of the `common` query. Search requests containing a `common` -query will return an error. -==== - -.The `cutoff_frequency` parameter has been removed from the `match` and `multi_match` query. -[%collapsible] -==== -*Details* + -The `cutoff_frequency` parameter, deprecated in 7.x, has been removed in 8.0 from `match` and `multi_match` queries. -The same functionality can be achieved without any configuration provided that the total number of hits is not tracked. - -*Impact* + -Discontinue use of the `cutoff_frequency` parameter. Search requests containing -this parameter in a `match` or `multi_match` query will return an error. -==== - -.The `nested_filter` and `nested_path` properties have been removed from the search API's `sort` request body parameter. -[%collapsible] -==== -*Details* + -The `nested_filter` and `nested_path` options, deprecated in 6.x, have been removed in favor of the `nested` context. - -*Impact* + -Discontinue use of the `sort` request body parameter's `nested_filter` and -`nested_path` properties. Requests containing these properties will return an -error. -==== - -.Search and get requests are now routed to shards using adaptive replica selection by default. -[%collapsible] -==== -*Details* + -{es} will no longer prefer using shards in the same location (with the same awareness attribute values) to process -`_search` and `_get` requests. Adaptive replica selection (activated by default in this version) will route requests -more efficiently using the service time of prior inter-node communications. - -*Impact* + -No action needed. -==== - -.Vector functions using `(query, doc['field'])` are no longer supported. -[%collapsible] -==== -*Details* + -The vector functions of the form `function(query, doc['field'])` were -deprecated in 7.6, and are now removed in 8.x. The form -`function(query, 'field')` should be used instead. For example, -`cosineSimilarity(query, doc['field'])` is replaced by -`cosineSimilarity(query, 'field')`. - -*Impact* + -Use the `function(query, 'field')` form. Discontinue use of the `function(query, -doc['field'])` form. Requests containing the `function(query, -doc['field'])` form will return an error. -==== - -.The search API's `indices_boost` request body parameter no longer accepts object values. -[%collapsible] -==== -*Details* + -The `indices_boost` option in the search request used to accept the boosts -both as an object and as an array. The object format has been deprecated since -5.2 and is now removed in 8.0. - -*Impact* + -Use only array values in the `indices_boost` parameter. Requests containing an -object value in the `indices_boost` parameter will return an error. -==== - -.The search API's `use_field_mapping` request body parameter has been removed. -[%collapsible] -==== -*Details* + -In 7.0, we began formatting `docvalue_fields` by default using each field's -mapping definition. To ease the transition from 6.x, we added the format -option `use_field_mapping`. This parameter was deprecated in 7.0, and is now -removed in 8.0. - -*Impact* + -Discontinue use of the `use_field_mapping` request body parameter. Requests -containing this parameter will return an error. - -*Compatibility* + -When {ref}/rest-api-compatibility.html[rest-api-compatibility] is -{ref}/rest-api-compatibility.html[requested], the `use_field_mapping` parameter -is ignored. -==== - -.The search API's `from` request body and url parameter cannot be negative. -[%collapsible] -==== -*Details* + -Search request used to accept `-1` as a `from` in the search body and the url, -treating it as the default value of 0. Other negative values got rejected with -an error already. We now also reject `-1` as an invalid value. - -*Impact* + -Change any use of `-1` as `from` parameter in request body or url parameters by either -setting it to `0` or omitting it entirely. Requests containing negative values will -return an error. -==== - -.Range queries on date fields treat numeric values alwas as milliseconds-since-epoch. -[%collapsible] -==== -*Details* + -Range queries on date fields used to misinterpret small numbers (e.g. four digits like 1000) -as a year when no additional format was set, but would interpret other numeric values as -milliseconds since epoch. We now treat all numeric values in absence of a specific `format` -parameter as milliseconds since epoch. If you want to query for years instead, with a missing -`format` you now need to quote the input value (e.g. "1984"). - -*Impact* + -If you query date fields without a specified `format`, check if the values in your queries are -actually meant to be milliseconds-since-epoch and use a numeric value in this case. If not, use -a string value which gets parsed by either the date format set on the field in the mappings or -by `strict_date_optional_time` by default. -==== - -.The `geo_bounding_box` query's `type` parameter has been removed. -[%collapsible] -==== -*Details* + -The `geo_bounding_box` query's `type` parameter was deprecated in 7.14.0 and has -been removed in 8.0.0. This parameter is a no-op and has no effect on the query. - -*Impact* + -Discontinue use of the `type` parameter. `geo_bounding_box` queries that include -this parameter will return an error. -==== - -.The `type` query has been removed. -[%collapsible] -==== -*Details* + -The `type` query has been removed. Mapping types have been removed in 8.0. - -*Impact* + -Discontinue use of the `type` query. Requests that include the `type` query -will return an error. - -==== - -.The `kibana_user` role has been renamed `kibana_admin`. -[%collapsible] -==== -*Details* + -Users who were previously assigned the `kibana_user` role should instead be assigned -the `kibana_admin` role. This role grants the same set of privileges as `kibana_user`, but has been -renamed to better reflect its intended use. - -*Impact* + -Assign users with the `kibana_user` role to the `kibana_admin` role. -Discontinue use of the `kibana_user` role. -==== - -[[snapshot-resolve-system-indices]] -.For snapshot and {slm-init} APIs, the `indices` parameter no longer resolves to system indices or system data streams. -[%collapsible] -==== -*Details* + -For snapshot and {slm-init} APIs, the `indices` parameter no longer resolves to -system indices or system data streams. -{ref}/snapshot-restore.html#feature-state[Feature states] are now the only way -to back up and restore system indices or system data streams from a snapshot. - -You can no longer use the `indices` parameter for the -{ref}/slm-api-put-policy.html[create {slm-init} policy API] or the -{ref}/create-snapshot-api.html[create snapshot API] to include a system index in -a snapshot. To back up a system index, use the `include_global_state` and -`feature_states` parameters to include the corresponding feature state instead. -By default, the `include_global_state` and `feature_states` parameters include -all system indices. - -Similarly, you can no longer use the {ref}/restore-snapshot-api.html[restore snapshot -API]'s `indices` parameter to restore a system index from a snapshot. To restore -a system index, use the `include_global_state` and `feature_states` parameters -to restore the corresponding feature state instead. By default, the -`include_global_state` and `feature_states` parameters don't restore any system -indices. - -*Impact* + -If you previously used the `indices` parameter to back up or restore system -indices, update your {slm-init} policies and application to use the -`include_global_state` and `feature_states` parameters instead. - -An {slm-init} policy that explicitly specifies a system index in the `indices` -parameter will fail to create snapshots. Similarly, a create snapshot API or -restore snapshot API request that explicitly specifies a system index in the -`indices` parameter will fail and return an error. If the `indices` value -includes a wildcard (`*`) pattern, the pattern will no longer match system -indices. -==== - -.Snapshots compress metadata files by default. -[%collapsible] -==== -*Details* + -Previously, the default value for `compress` was `false`. The default has been changed to `true`. - -This change will affect both newly created repositories and existing repositories where `compress=false` has not been -explicitly specified. - -*Impact* + -Update your workflow and applications to assume a default value of `true` for -the `compress` parameter. -==== - -.S3 snapshot repositories now use a DNS-style access pattern by default. -[%collapsible] -==== -*Details* + -Starting in version 7.4, `s3` snapshot repositories no longer use the -now-deprecated path-style access pattern by default. In versions 7.0, 7.1, 7.2 -and 7.3 `s3` snapshot repositories always used the path-style access pattern. -This is a breaking change for deployments that only support path-style access -but which are recognized as supporting DNS-style access by the AWS SDK. This -breaking change was made necessary by -https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/[AWS's -announcement] that the path-style access pattern is deprecated and will be -unsupported on buckets created after September 30th 2020. - -*Impact* + -If your deployment only supports path-style access and is affected by this -change then you must configure the S3 client setting `path_style_access` to -`true`. -==== - -.Restore requests no longer accept settings. -[%collapsible] -==== -*Details* + -In earlier versions, you could pass both `settings` and `index_settings` in the -body of a restore snapshot request, but the `settings` value was ignored. The -restore snapshot API now rejects requests that include a `settings` value. - -*Impact* + -Discontinue use of the `settings` parameter in restore -snapshot request. Requests that include these parameters will return an error. -==== - -.The repository stats API has been removed. -[%collapsible] -==== -*Details* + -The repository stats API has been removed. We deprecated this experimental API -in 7.10.0. - -*Impact* + -Use the {ref}/repositories-metering-apis.html[repositories metering APIs] -instead. -==== - -.Watcher history now writes to a hidden data stream. -[%collapsible] -==== -*Details* + -In 8.x, {es} writes Watcher history to a hidden -`.watcher-history-` data stream. Previously, {es} wrote -Watcher history to hidden -`.watcher-history--` indices. - -*Impact* + -Update your requests to target the Watcher history data stream. For example, use -the `.watcher-history-*` wildcard expression. Requests that specifically target -non-existent Watcher history indices may return an error. -==== - -.HTTP Status code has changed for the Cluster Health API in case of a server timeout. -[%collapsible] -==== -*Details* + -The {ref}/cluster-health.html[cluster health API] includes options for waiting -for certain health conditions to be satisfied. If the requested conditions are -not satisfied within a timeout then {es} will send back a normal response -including the field `"timed_out": true`. In earlier versions it would also use -the HTTP response code `408 Request timeout` if the request timed out, and `200 -OK` otherwise. The `408 Request timeout` response code is not appropriate for -this situation, so from version 8.0.0 {es} will use the response code `200 OK` -for both cases. - -*Impact* + -To detect a server timeout, check the `timed_out` field of the JSON response. -==== diff --git a/docs/reference/migration/migrate_8_0/sql-jdbc-changes.asciidoc b/docs/reference/migration/migrate_8_0/sql-jdbc-changes.asciidoc deleted file mode 100644 index 71efa21e032f5..0000000000000 --- a/docs/reference/migration/migrate_8_0/sql-jdbc-changes.asciidoc +++ /dev/null @@ -1,22 +0,0 @@ -[discrete] -[[breaking_80_jdbc_changes]] -==== SQL JDBC changes - -.JDBC driver returns geometry objects as well-known-text string instead of `org.elasticsearch.geo` objects. -[%collapsible] -==== -*Details* + -To reduce the dependency of the JDBC driver onto Elasticsearch classes, the JDBC driver returns geometry data -as strings using the WKT (well-known text) format instead of classes from the `org.elasticsearch.geometry`. -Users can choose the geometry library desired to convert the string representation into a full-blown objects -either such as the `elasticsearch-geo` library (which returned the object `org.elasticsearch.geo` as before), -jts or spatial4j. - -*Impact* + -Before upgrading, replace any `org.elasticsearch.geo` classes on the `ResultSet#getObject` or `ResultSet#setObject` -Elasticsearch JDBC driver with their WKT representation by simply calling `toString` or -`org.elasticsearch.geometry.utils.WellKnownText#toWKT/fromWKT` methods. - -This change does NOT impact users that do not use geometry classes. - -==== diff --git a/docs/reference/migration/migrate_8_0/system-req-changes.asciidoc b/docs/reference/migration/migrate_8_0/system-req-changes.asciidoc deleted file mode 100644 index df698eaeb2dfd..0000000000000 --- a/docs/reference/migration/migrate_8_0/system-req-changes.asciidoc +++ /dev/null @@ -1,59 +0,0 @@ -[discrete] -[[breaking_80_system_req_changes]] -==== System requirement changes - -TIP: {ess-skip-section} - -.Several EOL operating systems are no longer supported. -[%collapsible] -==== -*Details* + -The following operating systems have reached their end of life and are no longer -supported by {es}: - -* Amazon Linux -* CentOS 6 -* Debian 8 -* openSUSE Leap 42 -* Oracle Enterprise Linux 6 -* Ubuntu 16.04 - -We've also removed support for `SysV init`. No supported operating systems use -the `SysV init` process. - -*Details* + -Ensure your nodes use a -https://www.elastic.co/support/matrix#matrix_os[supported operating system]. -Running {es} on an unsupported operating system can result in unexpected errors -or failures. -==== - -.Java 17 is required. -[%collapsible] -==== -*Details* + -Java 17 or higher is now required to run {es} and any of its command -line tools. - -*Impact* + -Use Java 17 or higher. Attempts to run {es} 8.0 using earlier Java versions will -fail. - -There is not yet a FIPS-certified security module for Java 17 -that you can use when running {es} 8.0 in FIPS 140-2 mode. -If you run in FIPS 140-2 mode, you will either need to request an exception -from your security organization to upgrade to {es} 8.0, -or remain on {es} 7.x until Java 17 is certified. -==== - -.`JAVA_HOME` is no longer supported. -[%collapsible] -==== -*Details* + -`JAVA_HOME` is no longer supported to set the path for the JDK. Instead, use -the bundled JDK (preferable), or set `ES_JAVA_HOME`. - -*Impact* + -Use the bundled JDK (preferable), or set `ES_JAVA_HOME`. `JAVA_HOME` will be -ignored. -==== diff --git a/docs/reference/migration/migrate_8_0/transform.asciidoc b/docs/reference/migration/migrate_8_0/transform.asciidoc deleted file mode 100644 index aa47e28d83750..0000000000000 --- a/docs/reference/migration/migrate_8_0/transform.asciidoc +++ /dev/null @@ -1,16 +0,0 @@ -[discrete] -[[breaking_80_transform_changes]] -==== Transform changes - -.{transforms-cap} created in 7.4 or earlier versions must be upgraded. -[%collapsible] -==== -*Details* + -Early beta versions of {transforms} had configuration information in a format -that is no longer supported. - - -*Impact* + -Use the {ref}/upgrade-transforms.html[upgrade {transforms} API] to fix your -{transforms}. This upgrade does not affect the source or destination indices. -==== diff --git a/docs/reference/migration/migrate_8_1.asciidoc b/docs/reference/migration/migrate_8_1.asciidoc deleted file mode 100644 index 692559205f735..0000000000000 --- a/docs/reference/migration/migrate_8_1.asciidoc +++ /dev/null @@ -1,109 +0,0 @@ -[[migrating-8.1]] -== Migrating to 8.1 -++++ -8.1 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.1. - -See also <> and <>. - - -[discrete] -[[breaking-changes-8.1]] -=== Breaking changes - -The following changes in {es} 8.1 might affect your applications -and prevent them from operating normally. -Before upgrading to 8.1, review these changes and take the described steps -to mitigate the impact. - -[discrete] -[[breaking_81_rest_api_changes]] -==== REST API changes - -[[search_apis_fields_parameter_normalizes_geometry_objects_cross_international_dateline]] -.The search API's `fields` parameter now normalizes geometry objects that cross the international dateline -[%collapsible] -==== -*Details* + -The search API's `fields` parameter now normalizes `geo_shape` objects that -cross the international dateline (+/-180° longitude). For example, if a polygon -crosses the dateline, the `fields` parameter returns it as two polygons. You can -still retrieve original, unnormalized geometry objects from `_source`. - -*Impact* + -If your application requires unnormalized geometry objects, retrieve them from -`_source` rather than using the `fields` parameter. -==== - - -[discrete] -[[deprecated-8.1]] -=== Deprecations - -The following functionality has been deprecated in {es} 8.1 -and will be removed in a future version. -While this won't have an immediate impact on your applications, -we strongly encourage you take the described steps to update your code -after upgrading to 8.1. - -To find out if you are using any deprecated functionality, -enable <>. - -[discrete] -[[deprecations_81_cluster_and_node_setting]] -==== Cluster and node setting deprecations - -[[legacy_values_for_discovery_type_setting_are_deprecated]] -.Legacy values for the `discovery.type` setting are deprecated -[%collapsible] -==== -*Details* + -Legacy values for the `discovery.type` setting are deprecated and will be -forbidden in a future version. - -*Impact* + -Do not set `discovery.type` to any value except `single-node` or `multi-node`. -All other values are equivalent to the default discovery type which is -`multi-node`. Where possible, omit this setting so that {es} uses the default -discovery type. -==== - -[discrete] -[[deprecations_81_rest_api]] -==== REST API deprecations - -[[lenient_parsing_of_bulk_actions_deprecated]] -.Lenient parsing of bulk actions is deprecated -[%collapsible] -==== -*Details* + -Older versions of {es} parse the action lines of bulk requests very permissively -and would silently ignore invalid or malformed actions. This lenience is -deprecated and a future version will reject bulk requests containing invalid -actions. - -*Impact* + -Ensure that bulk actions are well-formed JSON objects containing a single entry -with the correct key. -==== - -[[deprecate_index_include_frozen_request_parameter_in_sql_api]] -.Deprecate `index_include_frozen` request parameter in `_sql` API -[%collapsible] -==== -*Details* + -Following the deprecation of frozen indices, the `index_include_frozen` -parameter and `FROZEN` syntax is now also deprecated. - -*Impact* + -You should unfreeze frozen indices using the -{ref}/unfreeze-index-api.html[unfreeze index API] and stop using the -`index_include_frozen` parameter or the `FROZEN` keyword in SQL -queries. For some use cases, the frozen tier may be a suitable -replacement for frozen indices. See {ref}/data-tiers.html[data tiers] -for more information. -==== - diff --git a/docs/reference/migration/migrate_8_10.asciidoc b/docs/reference/migration/migrate_8_10.asciidoc deleted file mode 100644 index a1d132812ad03..0000000000000 --- a/docs/reference/migration/migrate_8_10.asciidoc +++ /dev/null @@ -1,89 +0,0 @@ -[[migrating-8.10]] -== Migrating to 8.10 -++++ -8.10 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.10. - -See also <> and <>. - -[discrete] -[[breaking-changes-8.10]] -=== Breaking changes - -The following changes in {es} 8.10 might affect your applications -and prevent them from operating normally. -Before upgrading to 8.10, review these changes and take the described steps -to mitigate the impact. - - -There are no notable breaking changes in {es} 8.10. -But there are some less critical breaking changes. - -[discrete] -[[breaking_810_cluster_and_node_setting_changes]] -==== Cluster and node setting changes - -[[remove_unused_executor_builder_for_vector_tile_plugin]] -.Remove the unused executor builder for vector tile plugin -[%collapsible] -==== -*Details* + -The threadpool called `vectortile` is a left over from the original development of the vector tile search end point and it is used nowhere. It can still be a breaking change if it is configured on the elasticsearch yml file, for example by changing the threadpool size `thread_pool.vectortile.size=8`' - -*Impact* + -In the case the threadpool appears on the yaml file, Elasticsearch will not start until those lines are removed. -==== - -[discrete] -[[breaking_810_java_api_changes]] -==== Java API changes - -[[change_pre_configured_cached_analyzer_components_to_use_indexversion_instead_of_version-highlight]] -.Change pre-configured and cached analyzer components to use IndexVersion instead of Version -[%collapsible] -==== -*Details* + -This PR changes the types used to obtain pre-configured components from Version to IndexVersion, -with corresponding changes to method names. - -Prior to 8.10, there is a one-to-one mapping between node version and index version, with corresponding constants -in the IndexVersion class. -Starting in 8.10, IndexVersion is versioned independently of node version, and will be a simple incrementing number. -For more information on how to use IndexVersion and other version types, please see the contributing guide. - -*Impact* + -Analysis components now take IndexVersion instead of Version -==== - - -[discrete] -[[deprecated-8.10]] -=== Deprecations - -The following functionality has been deprecated in {es} 8.10 -and will be removed in a future version. -While this won't have an immediate impact on your applications, -we strongly encourage you to take the described steps to update your code -after upgrading to 8.10. - -To find out if you are using any deprecated functionality, -enable <>. - -[discrete] -[[deprecations_810_authorization]] -==== Authorization deprecations - -[[mark_apm_user_for_removal_in_future_major_release]] -.Mark `apm_user` for removal in a future major release -[%collapsible] -==== -*Details* + -The `apm_user` role has been deprecated and will be removed in a future major release. Users should migrate to `editor` and `viewer` roles - -*Impact* + -Users will have to migrate to `editor` and `viewer` roles -==== - diff --git a/docs/reference/migration/migrate_8_11.asciidoc b/docs/reference/migration/migrate_8_11.asciidoc deleted file mode 100644 index 098456e1aca42..0000000000000 --- a/docs/reference/migration/migrate_8_11.asciidoc +++ /dev/null @@ -1,69 +0,0 @@ -[[migrating-8.11]] -== Migrating to 8.11 -++++ -8.11 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.11. - -See also <> and <>. - - -[discrete] -[[breaking-changes-8.11]] -=== Breaking changes - -The following changes in {es} 8.11 might affect your applications -and prevent them from operating normally. -Before upgrading to 8.11, review these changes and take the described steps -to mitigate the impact. - - -There are no notable breaking changes in {es} 8.11. -But there are some less critical breaking changes. - -[discrete] -[[breaking_811_rest_api_changes]] -==== REST API changes - -[[remove_transport_versions_from_cluster_state_api]] -.Remove `transport_versions` from cluster state API -[%collapsible] -==== -*Details* + -The `transport_versions` subobject of the response to `GET _cluster/state` has been replaced by the `nodes_versions` subobject. - -*Impact* + -If needed, retrieve the per-node transport versions from the `nodes_versions` subobject. -==== - - -[discrete] -[[deprecated-8.11]] -=== Deprecations - -The following functionality has been deprecated in {es} 8.11 -and will be removed in a future version. -While this won't have an immediate impact on your applications, -we strongly encourage you to take the described steps to update your code -after upgrading to 8.11. - -To find out if you are using any deprecated functionality, -enable <>. - -[discrete] -[[deprecations_811_rollup]] -==== Rollup deprecations - -[[rollup_functionality_deprecated]] -.Rollup functionality is now deprecated -[%collapsible] -==== -*Details* + -<> has been deprecated and will be removed in a future release. Previously, rollups were available in technical preview. - -*Impact* + -Use <> to reduce storage costs for time series data by storing it at reduced granularity. -==== - diff --git a/docs/reference/migration/migrate_8_12.asciidoc b/docs/reference/migration/migrate_8_12.asciidoc deleted file mode 100644 index c7f4aa8728693..0000000000000 --- a/docs/reference/migration/migrate_8_12.asciidoc +++ /dev/null @@ -1,74 +0,0 @@ -[[migrating-8.12]] -== Migrating to 8.12 -++++ -8.12 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.12. - -See also <> and <>. - -[discrete] -[[breaking-changes-8.12]] -=== Breaking changes - -There are no breaking changes in 8.12 - -[discrete] -[[notable-changes-8.12]] -=== Notable changes - -There are notable changes in 8.12 that you need to be aware of, items that we may consider as notable changes are - -* Changes to features that are in Technical Preview. -* Changes to log formats. -* Changes to non public APIs. -* Behaviour changes that repair critical bugs. - - -[discrete] -[[breaking_812_authorization_changes]] -==== Authorization changes - -[[fixed_jwt_principal_from_claims]] -.Fixed JWT principal from claims -[%collapsible] -==== -*Details* + -This changes the format of a JWT's principal before the JWT is actually validated by any JWT realm. The JWT's principal is a convenient way to refer to a JWT that has not yet been verified by a JWT realm. The JWT's principal is printed in the audit and regular logs (notably for auditing authn failures) as well as the smart realm chain reordering optimization. The JWT principal is NOT required to be identical to the JWT-authenticated user's principal, but in general, they should be similar. Previously, the JWT's principal was built by individual realms in the same way the realms built the authenticated user's principal. This had the advantage that, in simpler JWT realms configurations (e.g. a single JWT realm in the chain), the JWT principal and the authenticated user's principal are very similar. However the drawback is that, in general, the JWT principal and the user principal can be very different (i.e. in the case where one JWT realm builds the JWT principal and a different one builds the user principal). Another downside is that the (unauthenticated) JWT principal depended on realm ordering, which makes identifying the JWT from its principal dependent on the ES authn realm configuration. This PR implements a consistent fixed logic to build the JWT principal, which only depends on the JWT's claims and no ES configuration. - -*Impact* + -Users will observe changed format and values for the `user.name` attribute of `authentication_failed` audit log events, in the JWT (failed) authn case. -==== - -[discrete] -[[breaking_812_java_api_changes]] -==== Java API changes - -[[plugin_createcomponents_method_has_been_refactored_to_take_single_pluginservices_object]] -.Plugin.createComponents method has been refactored to take a single PluginServices object -[%collapsible] -==== -*Details* + -Plugin.createComponents currently takes several different service arguments. The signature of this method changes every time a new service is added. The method has now been modified to take a single interface object that new services are added to. This will reduce API incompatibility issues when a new service is introduced in the future. - -*Impact* + -Plugins that override createComponents will need to be refactored to override the new method on ES 8.12+ -==== - -[discrete] -[[breaking_812_rest_api_changes]] -==== REST API changes - -[[es_ql_pow_function_always_returns_double]] -.[ES|QL] pow function always returns double -[%collapsible] -==== -*Details* + -This corrects an earlier mistake in the ES|QL language design. Initially we had thought to have pow return the same type as its inputs, but in practice even for integer inputs this quickly grows out of the representable range, and we returned null much of the time. This also created a lot of edge cases around casting to/from doubles (which the underlying java function uses). The version in this PR follows the java spec, by always casting its inputs to doubles, and returning a double. Doing it this way also allows for a rather significant reduction in lines of code. - -*Impact* + -low. Most queries should continue to function with the change. -==== - diff --git a/docs/reference/migration/migrate_8_13.asciidoc b/docs/reference/migration/migrate_8_13.asciidoc deleted file mode 100644 index dca10671e57bc..0000000000000 --- a/docs/reference/migration/migrate_8_13.asciidoc +++ /dev/null @@ -1,137 +0,0 @@ -[[migrating-8.13]] -== Migrating to 8.13 -++++ -8.13 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.13. - -See also <> and <>. - -coming::[8.13.0] - - -[discrete] -[[breaking-changes-8.13]] -=== Breaking changes - -There are no breaking changes in 8.13. - -[discrete] -[[migrate-notable-changes-8.13]] -=== Notable changes -The following are notable, non-breaking updates to be aware of: - -* Changes to features that are in Technical Preview. -* Changes to log formats. -* Changes to non-public APIs. -* Behaviour changes that repair critical bugs. - -[discrete] -[[breaking_813_index_setting_changes]] -==== Index setting changes - -[[change_index_look_ahead_time_index_settings_default_value_from_2_hours_to_30_minutes]] -.Change `index.look_ahead_time` index setting's default value from 2 hours to 30 minutes. -[%collapsible] -==== -*Details* + -Lower the `index.look_ahead_time` index setting's max value from 2 hours to 30 minutes. - -*Impact* + -Documents with @timestamp of 30 minutes or more in the future will be rejected. Before documents with @timestamp of 2 hours or more in the future were rejected. If the previous behaviour should be kept, then update the `index.look_ahead_time` setting to two hours before performing the upgrade. -==== - -[[lower_look_ahead_time_index_settings_max_value]] -.Lower the `look_ahead_time` index setting's max value -[%collapsible] -==== -*Details* + -Lower the `look_ahead_time` index setting's max value from 7 days to 2 hours. - -*Impact* + -Any value between 2 hours and 7 days will be as a look ahead time of 2 hours is defined -==== - -[discrete] -[[breaking_813_rest_api_changes]] -==== REST API changes - -[[esql_grammar_from_metadata_no_longer_requires]] -.ESQL: Grammar - FROM METADATA no longer requires [] -[%collapsible] -==== -*Details* + -Remove [ ] for METADATA option inside FROM command statements - -*Impact* + -Previously to return metadata fields, one had to use square brackets: (eg. 'FROM index [METADATA _index]'). This is no longer needed: the [ ] are dropped and do not have to be specified, thus simplifying the command above to:'FROM index METADATA _index'. -==== - -[[es_ql_remove_project_keyword_from_grammar]] -.ES|QL: remove PROJECT keyword from the grammar -[%collapsible] -==== -*Details* + -Removes the PROJECT keyword (an alias for KEEP) from ES|QL grammar - -*Impact* + -Before this change, users could use PROJECT as an alias for KEEP in ESQL queries, (eg. 'FROM idx | PROJECT name, surname') the parser replaced PROJECT with KEEP, emitted a warning: 'PROJECT command is no longer supported, please use KEEP instead' and the query was executed normally. With this change, PROJECT command is no longer recognized by the query parser; queries using PROJECT command now return a parsing exception. -==== - -[[esql_remove_nan_finite_infinite]] -.[ESQL] Remove is_nan, is_finite, and `is_infinite` -[%collapsible] -==== -*Details* + -Removes the functions `is_nan`, `is_finite`, and `is_infinite`. - -*Impact* + -Attempting to use the above functions will now be a planner time error. These functions are no longer supported. -==== - - -[discrete] -[[deprecated-8.13]] -=== Deprecations - -The following functionality has been deprecated in {es} 8.13 -and will be removed in a future version. -While this won't have an immediate impact on your applications, -we strongly encourage you to take the described steps to update your code -after upgrading to 8.13. - -To find out if you are using any deprecated functionality, -enable <>. - -[discrete] -[[deprecations_813_cluster_and_node_setting]] -==== Cluster and node setting deprecations - -[[deprecate_client_type]] -.Deprecate `client.type` -[%collapsible] -==== -*Details* + -The node setting `client.type` has been ignored since the node client was removed in 8.0. The setting is now deprecated and will be removed in a future release. - -*Impact* + -Remove the `client.type` setting from `elasticsearch.yml` -==== - -[discrete] -[[deprecations_813_rest_api]] -==== REST API deprecations - -[[desirednode_deprecate_node_version_field_make_it_optional_for_current_version]] -.`DesiredNode:` deprecate `node_version` field and make it optional for the current version -[%collapsible] -==== -*Details* + -The desired_node API includes a `node_version` field to perform validation on the new node version required. This kind of check is too broad, and it's better done by external logic, so it has been removed, making the `node_version` field not necessary. The field will be removed in a later version. - -*Impact* + -Users should update their usages of `desired_node` to not include the `node_version` field anymore. -==== - diff --git a/docs/reference/migration/migrate_8_14.asciidoc b/docs/reference/migration/migrate_8_14.asciidoc deleted file mode 100644 index 2e6cd439ebed0..0000000000000 --- a/docs/reference/migration/migrate_8_14.asciidoc +++ /dev/null @@ -1,90 +0,0 @@ -[[migrating-8.14]] -== Migrating to 8.14 -++++ -8.14 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.14. - -See also <> and <>. - -coming::[8.14.0] - - -[discrete] -[[breaking-changes-8.14]] -=== Breaking changes - -The following changes in {es} 8.14 might affect your applications -and prevent them from operating normally. -Before upgrading to 8.14, review these changes and take the described steps -to mitigate the impact. - - -There are no notable breaking changes in {es} 8.14. -But there are some less critical breaking changes. - -[discrete] -[[breaking_814_rest_api_changes]] -==== REST API changes - -[[prevent_dls_fls_if_replication_assigned]] -.Prevent DLS/FLS if `replication` is assigned -[%collapsible] -==== -*Details* + -For cross-cluster API keys, {es} no longer allows specifying document-level security (DLS) or field-level security (FLS) in the `search` field, if `replication` is also specified. {es} likewise blocks the use of any existing cross-cluster API keys that meet this condition. - -*Impact* + -Remove any document-level security (DLS) or field-level security (FLS) definitions from the `search` field for cross-cluster API keys that also have a `replication` field, or create two separate cross-cluster API keys, one for search and one for replication. -==== - - -[discrete] -[[breaking_814_dls_changes]] -==== Stricter Document Level Security (DLS) - -[[stricter_dls_814]] -.Document Level Security (DLS) applies stricter checks for the validate query API and for terms aggregations when min_doc_count is set to 0. - -[%collapsible] -==== -*Details* + -When Document Level Security (DLS) is applied to terms aggregations and min_doc_count is set to 0, stricter security rules apply. - -When Document Level Security (DLS) is applied to the validate query API with the rewrite parameter, stricter security rules apply. - -*Impact* + -If needed, test workflows with DLS enabled to ensure that the stricter security rules do not impact your application. -==== - - -[discrete] -[[deprecated-8.14]] -=== Deprecations - -The following functionality has been deprecated in {es} 8.14 -and will be removed in a future version. -While this won't have an immediate impact on your applications, -we strongly encourage you to take the described steps to update your code -after upgrading to 8.14. - -To find out if you are using any deprecated functionality, -enable <>. - -[discrete] -[[deprecations_814_mapping]] -==== Mapping deprecations - -[[deprecate_allowing_fields_in_scenarios_where_it_ignored]] -.Deprecate allowing `fields` in scenarios where it is ignored -[%collapsible] -==== -*Details* + -The following mapped types have always ignored `fields` when using multi-fields. This deprecation makes this clearer and we will completely disallow `fields` for these mapped types in the future. - -*Impact* + -In the future, `join`, `aggregate_metric_double`, and `constant_keyword`, will all disallow supplying `fields` as a parameter in the mapping. -==== - diff --git a/docs/reference/migration/migrate_8_15.asciidoc b/docs/reference/migration/migrate_8_15.asciidoc deleted file mode 100644 index 1961230da1bbf..0000000000000 --- a/docs/reference/migration/migrate_8_15.asciidoc +++ /dev/null @@ -1,140 +0,0 @@ -[[migrating-8.15]] -== Migrating to 8.15 -++++ -8.15 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.15. - -See also <> and <>. - -coming::[8.15.0] - - -[discrete] -[[breaking-changes-8.15]] -=== Breaking changes - -The following changes in {es} 8.15 might affect your applications -and prevent them from operating normally. -Before upgrading to 8.15, review these changes and take the described steps -to mitigate the impact. - -[discrete] -[[breaking_815_cluster_and_node_setting_changes]] -==== Cluster and node setting changes - -[[change_skip_unavailable_remote_cluster_setting_default_value_to_true]] -.Change `skip_unavailable` remote cluster setting default value to true -[%collapsible] -==== -*Details* + -The default value of the `skip_unavailable` setting is now set to true. All existing and future remote clusters that do not define this setting will use the new default. This setting only affects cross-cluster searches using the _search or _async_search API. - -*Impact* + -Unavailable remote clusters in a cross-cluster search will no longer cause the search to fail unless skip_unavailable is configured to be `false` in elasticsearch.yml or via the `_cluster/settings` API. Unavailable clusters with `skip_unavailable`=`true` (either explicitly or by using the new default) are marked as SKIPPED in the search response metadata section and do not fail the entire search. If users want to ensure that a search returns a failure when a particular remote cluster is not available, `skip_unavailable` must be now be set explicitly. -==== - -[discrete] -[[breaking_815_rollup_changes]] -==== Rollup changes - -[[disallow_new_rollup_jobs_in_clusters_with_no_rollup_usage]] -.Disallow new rollup jobs in clusters with no rollup usage -[%collapsible] -==== -*Details* + -The put rollup API will fail with an error when a rollup job is created in a cluster with no rollup usage - -*Impact* + -Clusters with no rollup usage (either no rollup job or index) can not create new rollup jobs -==== - -[discrete] -[[breaking_815_rest_api_changes]] -==== REST API changes - -[[interpret_timeout_1_as_infinite_ack_timeout]] -.Interpret `?timeout=-1` as infinite ack timeout -[%collapsible] -==== -*Details* + -Today {es} accepts the parameter `?timeout=-1` in many APIs, but interprets -this to mean the same as `?timeout=0`. From 8.15 onwards `?timeout=-1` will -mean to wait indefinitely, aligning the behaviour of this parameter with -other similar parameters such as `?master_timeout`. - -*Impact* + -Use `?timeout=0` to force relevant operations to time out immediately -instead of `?timeout=-1` -==== - -[[replace_model_id_with_inference_id]] -.Replace `model_id` with `inference_id` in GET inference API -[%collapsible] -==== -*Details* + -From 8.15 onwards the <> response will return an -`inference_id` field instead of a `model_id`. - -*Impact* + -If your application uses the `model_id` in a GET inference API response, -switch it to use `inference_id` instead. -==== - - -[discrete] -[[deprecated-8.15]] -=== Deprecations - -The following functionality has been deprecated in {es} 8.15 -and will be removed in a future version. -While this won't have an immediate impact on your applications, -we strongly encourage you to take the described steps to update your code -after upgrading to 8.15. - -To find out if you are using any deprecated functionality, -enable <>. - -[discrete] -[[deprecations_815_cluster_and_node_setting]] -==== Cluster and node setting deprecations - -[[deprecate_absolute_size_values_for_indices_breaker_total_limit_setting]] -.Deprecate absolute size values for `indices.breaker.total.limit` setting -[%collapsible] -==== -*Details* + -Previously, the value of `indices.breaker.total.limit` could be specified as an absolute size in bytes. This setting controls the overal amount of memory the server is allowed to use before taking remedial actions. Setting this to a specific number of bytes led to strange behaviour when the node maximum heap size changed because the circut breaker limit would remain unchanged. This would either leave the value too low, causing part of the heap to remain unused; or it would leave the value too high, causing the circuit breaker to be ineffective at preventing OOM errors. The only reasonable behaviour for this setting is that it scales with the size of the heap, and so absolute byte limits are now deprecated. - -*Impact* + -Users must change their configuration to specify a percentage instead of an absolute number of bytes for `indices.breaker.total.limit`, or else accept the default, which is already specified as a percentage. -==== - -[discrete] -[[deprecations_815_rest_api]] -==== REST API deprecations - -[[deprecate_text_expansion_weighted_tokens_queries]] -.Deprecate `text_expansion` and `weighted_tokens` queries -[%collapsible] -==== -*Details* + -The `text_expansion` and `weighted_tokens` queries have been replaced by `sparse_vector`. - -*Impact* + -Please update your existing `text_expansion` and `weighted_tokens` queries to use `sparse_vector.` -==== - -[[deprecate_using_slm_privileges_to_access_ilm]] -.Deprecate using slm privileges to access ilm -[%collapsible] -==== -*Details* + -The `read_slm` privilege can get the ILM status, and the `manage_slm` privilege can start and stop ILM. Access to these APIs should be granted using the `read_ilm` and `manage_ilm` privileges instead. Access to ILM APIs will be removed from SLM privileges in a future major release, and is now deprecated. - -*Impact* + -Users that need access to the ILM status API should now use the `read_ilm` privilege. Users that need to start and stop ILM, should use the `manage_ilm` privilege. -==== - diff --git a/docs/reference/migration/migrate_8_16.asciidoc b/docs/reference/migration/migrate_8_16.asciidoc deleted file mode 100644 index 950b8f7ec3964..0000000000000 --- a/docs/reference/migration/migrate_8_16.asciidoc +++ /dev/null @@ -1,37 +0,0 @@ -[[migrating-8.16]] -== Migrating to 8.16 -++++ -8.16 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.16. - -See also <> and <>. - -coming::[8.16.0] - - -[discrete] -[[breaking-changes-8.16]] -=== Breaking changes - -The following changes in {es} 8.16 might affect your applications -and prevent them from operating normally. -Before upgrading to 8.16, review these changes and take the described steps -to mitigate the impact. - -[discrete] -[[breaking_816_locale_change]] -==== JDK locale database change - -{es} 8.16 changes the version of the JDK that is included from version 22 to version 23. This changes -the locale database that is used by Elasticsearch from the _COMPAT_ database to the _CLDR_ database. -This can result in significant changes to custom textual date field formats, -and calculations for custom week-date date fields. - -For more information see <>. - -If you run {es} 8.16 on JDK version 22 or below, it will use the _COMPAT_ locale database -to match the behavior of 8.15. However, please note that starting with {es} 9.0, -{es} will use the _CLDR_ database regardless of JDK version it is run on. diff --git a/docs/reference/migration/migrate_8_17.asciidoc b/docs/reference/migration/migrate_8_17.asciidoc deleted file mode 100644 index 15bc6431c60ba..0000000000000 --- a/docs/reference/migration/migrate_8_17.asciidoc +++ /dev/null @@ -1,20 +0,0 @@ -[[migrating-8.17]] -== Migrating to 8.17 -++++ -8.17 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.17. - -See also <> and <>. - -coming::[8.17.0] - - -[discrete] -[[breaking-changes-8.17]] -=== Breaking changes - -There are no breaking changes in {es} 8.17. - diff --git a/docs/reference/migration/migrate_8_2.asciidoc b/docs/reference/migration/migrate_8_2.asciidoc deleted file mode 100644 index 3630456aed6fd..0000000000000 --- a/docs/reference/migration/migrate_8_2.asciidoc +++ /dev/null @@ -1,16 +0,0 @@ -[[migrating-8.2]] -== Migrating to 8.2 -++++ -8.2 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.2. - -See also <> and <>. - -[discrete] -[[breaking-changes-8.2]] -=== Breaking changes - -There are no breaking changes in {es} 8.2. diff --git a/docs/reference/migration/migrate_8_3.asciidoc b/docs/reference/migration/migrate_8_3.asciidoc deleted file mode 100644 index 1dfc2d1b8cd23..0000000000000 --- a/docs/reference/migration/migrate_8_3.asciidoc +++ /dev/null @@ -1,61 +0,0 @@ -[[migrating-8.3]] -== Migrating to 8.3 -++++ -8.3 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.3. - -See also <> and <>. - - -[discrete] -[[breaking-changes-8.3]] -=== Breaking changes - -There are no breaking changes in {es} 8.3. - - - -[discrete] -[[deprecated-8.3]] -=== Deprecations - -The following functionality has been deprecated in {es} 8.3 -and will be removed in a future version. -While this won't have an immediate impact on your applications, -we strongly encourage you take the described steps to update your code -after upgrading to 8.3. - -To find out if you are using any deprecated functionality, -enable <>. - - -[discrete] -[[deprecations_83_cluster_and_node_setting]] -==== Cluster and node setting deprecations - -[[configuring_bind_dn_in_an_ldap_or_active_directory_ad_realm_without_corresponding_bind_password_deprecated]] -.Configuring a bind DN in an LDAP or Active Directory (AD) realm without a corresponding bind password is deprecated -[%collapsible] -==== -*Details* + -For LDAP or AD authentication realms, setting a bind DN (via the -`xpack.security.authc.realms.ldap.*.bind_dn` realm setting) without a -bind password is a misconfiguration that may prevent successful -authentication to the node. In the next major release, nodes will fail -to start if a bind DN is specified without a password. - -*Impact* + -If you have a bind DN configured for an LDAP or AD authentication -realm, set a bind password for {ref}/ldap-realm.html#ldap-realm-configuration[LDAP] -or {ref}/active-directory-realm.html#ad-realm-configuration[Active Directory]. -Configuring a bind DN without a password generates a warning in the -deprecation logs. - -*Note:* This deprecation only applies if your current LDAP or AD -configuration specifies a bind DN without a password. This scenario is -unlikely, but might impact a small subset of users. -==== - diff --git a/docs/reference/migration/migrate_8_4.asciidoc b/docs/reference/migration/migrate_8_4.asciidoc deleted file mode 100644 index d9aab317d70f7..0000000000000 --- a/docs/reference/migration/migrate_8_4.asciidoc +++ /dev/null @@ -1,46 +0,0 @@ -[[migrating-8.4]] -== Migrating to 8.4 -++++ -8.4 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.4. - -See also <> and <>. - -[discrete] -[[breaking-changes-8.4]] -=== Breaking changes - -There are no breaking changes in {es} 8.4. - -[discrete] -[[deprecated-8.4]] -=== Deprecations - -The following functionality has been deprecated in {es} 8.4 -and will be removed in a future version. -While this won't have an immediate impact on your applications, -we strongly encourage you to take the described steps to update your code -after upgrading to 8.4. - -To find out if you are using any deprecated functionality, -enable <>. - - -[discrete] -[[deprecations_84_rest_api]] -==== REST API deprecations - -[[deprecate_knn_search_endpoint]] -.Deprecate the `_knn_search` endpoint -[%collapsible] -==== -*Details* + --| The kNN search API is deprecated in favor of the new 'knn' option inside the search API. The 'knn' option is now the recommended way of running ANN search. - -*Impact* + -Users should switch from `_knn_search` to the search `knn` option. -==== - diff --git a/docs/reference/migration/migrate_8_5.asciidoc b/docs/reference/migration/migrate_8_5.asciidoc deleted file mode 100644 index 1f040946670e1..0000000000000 --- a/docs/reference/migration/migrate_8_5.asciidoc +++ /dev/null @@ -1,101 +0,0 @@ -[[migrating-8.5]] -== Migrating to 8.5 -++++ -8.5 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.5. - -See also <> and <>. - -[discrete] -[[breaking-changes-8.5]] -=== Breaking changes - -The following changes in {es} 8.5 might affect your applications and prevent -them from operating normally. Before upgrading to 8.5, review these changes and -take the described steps to mitigate the impact. - -[discrete] -[[breaking_85_rest_api_changes]] -==== REST API changes - -[[breaking_85_bulk_action_stricter]] -.The bulk API now rejects requests containing unrecognized actions -[%collapsible] -==== -*Details* + -Requests to the bulk API comprise a sequence of items, each of which starts with -a JSON object describing the item. This object includes the type of action to -perform with the item which should be one of `create`, `update`, `index`, or -`delete`. Earlier versions of {es} had a bug that caused them to ignore items -with an unrecognized type, skipping the next line in the request, but this -lenient behaviour meant that there is no way for the client to associate the -items in the response with the items in the request, and in some cases it would -cause the remainder of the request to be parsed incorrectly. - -From version 8.5 onwards, requests to the bulk API must comprise only items -with recognized types. {es} will reject requests containing any items with an -unrecognized type with a `400 Bad Request` error response. - -We consider this change to be a bugfix but list it here as a breaking change -since it may affect the behaviour of applications which rely on being able to -send unrecognized actions to {es}. - -*Impact* + -Ensure your application only sends items with type `create`, `update`, `index` -or `delete` to the bulk API. -==== - -[discrete] -[[deprecated-8.5]] -=== Deprecations - -The following functionality has been deprecated in {es} 8.5 -and will be removed in a future version. -While this won't have an immediate impact on your applications, -we strongly encourage you to take the described steps to update your code -after upgrading to 8.5. - -To find out if you are using any deprecated functionality, -enable <>. - - -[discrete] -[[deprecations_85_plugins]] -==== Plugin API deprecations - -[[network_plugins_deprecated]] -Plugins that extend the NetworkPlugin interface are deprecated. -[%collapsible] -==== -*Details* + -Plugins may override funcionality that controls how nodes connect -with other nodes over TCP/IP. These plugins extend the NetworkPlugin -interface. In the next major release, these plugins will fail -to install. - -*Impact* + -Discontinue using any plugins which extend NetworkPlugin. You can -see if any plugins use deprecated functionality by checking -the Elasticsearch deprecation log. -==== - -[[discoveryplugin_joinvalidator_and_election_strategies_deprecated]] -.Extending DiscoveryPlugin to override join validators or election strategies is deprecated -[%collapsible] -==== -*Details* + -Plugins that extend DiscoveryPlugin may override getJoinValidator and -getElectionStrategies. These methods are implementation details of the -clustering mechanism within Elasticsearch. They should not be overriden. -In the next major release, plugins overriding getJoinValidator or -getElectionStrategies will fail to install. - -*Impact* + -Discontinue using any plugins that override getJoinValidator or -getElectionStrategies in DiscoveryPlugin. You can see if any plugins -use deprecated functionality by checking the Elasticsearch deprecation log. -==== - diff --git a/docs/reference/migration/migrate_8_6.asciidoc b/docs/reference/migration/migrate_8_6.asciidoc deleted file mode 100644 index 80c0ece8c1e37..0000000000000 --- a/docs/reference/migration/migrate_8_6.asciidoc +++ /dev/null @@ -1,92 +0,0 @@ -[[migrating-8.6]] -== Migrating to 8.6 -++++ -8.6 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.6. - -See also <> and <>. - -[discrete] -[[breaking-changes-8.6]] -=== Breaking changes - -There are no breaking changes in {es} 8.6. - -[discrete] -[[deprecated-8.6]] -=== Deprecations - -The following functionality has been deprecated in {es} 8.6 -and will be removed in a future version. -While this won't have an immediate impact on your applications, -we strongly encourage you to take the described steps to update your code -after upgrading to 8.6. - -To find out if you are using any deprecated functionality, -enable <>. - - -[discrete] -[[deprecations_86_crud]] -==== CRUD deprecations - -[[deprecate_remove_binary_default_of_false_for_ingest_attachment_processor]] -.Deprecate 'remove_binary' default of false for ingest attachment processor -[%collapsible] -==== -*Details* + -The default "remove_binary" option for the attachment processor will be changed from false to true in a later Elasticsearch release. This means that the binary file sent to Elasticsearch will not be retained. - -*Impact* + -Users should update the "remove_binary" option to be explicitly true or false, instead of relying on the default value, so that no default value changes will affect Elasticsearch. -==== - -[discrete] -[[deprecations_86_cluster_and_node_setting]] -==== Cluster and node setting deprecations - -[[ensure_balance_threshold_at_least_1]] -.Ensure balance threshold is at least 1 -[%collapsible] -==== -*Details* + -Values for `cluster.routing.allocation.balance.threshold` smaller than `1` are now ignored. Support for values less than `1` for this setting is deprecated and will be forbidden in a future version. - -*Impact* + -Set `cluster.routing.allocation.balance.threshold` to be at least `1`. -==== - -[discrete] -[[deprecations_86_mapping]] -==== Mapping deprecations - -[[deprecate_silently_ignoring_type_fields_copy_to_boost_in_metadata_field_definition]] -.Deprecate silently ignoring type, fields, copy_to and boost in metadata field definition -[%collapsible] -==== -*Details* + -Unsupported parameters like type, fields, copy_to and boost are silently ignored when provided as part of the configuration of a metadata field in the index mappings. They will cause a deprecation warning when used in the mappings for indices that are created from 8.6 onwards. - -*Impact* + -To resolve the deprecation warning, remove the mention of type, fields, copy_to or boost from any metadata field definition as part of index mappings. They take no effect so removing them won't have any impact besides resolving the deprecation warning. -==== - -[discrete] -[[deprecations_86_rest_api]] -==== REST API deprecations - -[[state_field_deprecated_in_cluster_reroute_response]] -.state field is deprecated in /_cluster/reroute response -[%collapsible] -==== -*Details* + -`state` field is deprecated in `/_cluster/reroute` response. Cluster state does not provide meaningful information -about the result of reroute/commands execution. There are no guarantees that this exact state would be applied. - -*Impact* + -Reroute API users should not rely on `state` field and instead use `explain` to request result of commands execution. -==== - diff --git a/docs/reference/migration/migrate_8_7.asciidoc b/docs/reference/migration/migrate_8_7.asciidoc deleted file mode 100644 index 2061743e1be4a..0000000000000 --- a/docs/reference/migration/migrate_8_7.asciidoc +++ /dev/null @@ -1,43 +0,0 @@ -[[migrating-8.7]] -== Migrating to 8.7 -++++ -8.7 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.7. - -See also <> and <>. - -[discrete] -[[breaking-changes-8.7]] -=== Breaking changes - -The following changes in {es} 8.7 might affect your applications -and prevent them from operating normally. -Before upgrading to 8.7, review these changes and take the described steps -to mitigate the impact. - -There are no notable breaking changes in {es} 8.7. -But there are some less critical breaking changes. - -[discrete] -[[breaking_87_ingest_changes]] -==== Ingest changes - -[[making_jsonprocessor_stricter_so_it_does_not_silently_drop_data]] -.Making `JsonProcessor` stricter so that it does not silently drop data -[%collapsible] -==== -*Details* + -The ingest node's `json` processor was previously lenient. It would accept invalid JSON data if it started with valid JSON data. -Anything after the valid part would be silently discarded. From 8.7 onwards, the default behavior is to reject invalid JSON data with -an exception so that data is not silently lost. The old behavior can be reproduced by passing `false` as the value of the new -`strict_json_parsing` processor parameter. -We consider this change to be a bugfix but list it here as a breaking change since it may affect the behavior of applications which -were sending invalid JSON data to the `json` processor. - -*Impact* + -Ensure your application only sends valid JSON data to the `json` processor, or modify the `json` processors in your pipelines to set -the `strict_json_parsing` parameter to `false`. -==== diff --git a/docs/reference/migration/migrate_8_8.asciidoc b/docs/reference/migration/migrate_8_8.asciidoc deleted file mode 100644 index 22c5ae2a33750..0000000000000 --- a/docs/reference/migration/migrate_8_8.asciidoc +++ /dev/null @@ -1,47 +0,0 @@ -[[migrating-8.8]] -== Migrating to 8.8 -++++ -8.8 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.8. - -See also <> and <>. - - -[discrete] -[[breaking-changes-8.8]] -=== Breaking changes - -There are no breaking changes in {es} 8.8. - -[discrete] -[[deprecated-8.8]] -=== Deprecations - -The following functionality has been deprecated in {es} 8.8 -and will be removed in a future version. -While this won't have an immediate impact on your applications, -we strongly encourage you to take the described steps to update your code -after upgrading to 8.8. - -To find out if you are using any deprecated functionality, -enable <>. - - -[discrete] -[[deprecations_88_cluster_and_node_setting]] -==== Cluster and node setting deprecations - -[[deprecate_cluster_routing_allocation_type]] -.Deprecate `cluster.routing.allocation.type` -[%collapsible] -==== -*Details* + -The `cluster.routing.allocation.type` setting is deprecated and will be removed in a future version. - -*Impact* + -Discontinue use of the `cluster.routing.allocation.type` setting. -==== - diff --git a/docs/reference/migration/migrate_8_9.asciidoc b/docs/reference/migration/migrate_8_9.asciidoc deleted file mode 100644 index e2f54dc58bfe5..0000000000000 --- a/docs/reference/migration/migrate_8_9.asciidoc +++ /dev/null @@ -1,35 +0,0 @@ -[[migrating-8.9]] -== Migrating to 8.9 -++++ -8.9 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to {es} 8.9. - -See also <> and <>. - -[discrete] -[[breaking-changes-8.9]] -=== Breaking changes - -The following changes in {es} 8.9 might affect your applications -and prevent them from operating normally. -Before upgrading to 8.9, review these changes and take the described steps -to mitigate the impact. - -[discrete] -[[breaking_89_rest_api_changes]] -==== REST API changes - -[[switch_tdigeststate_to_use_hybriddigest_by_default]] -.Switch TDigestState to use `HybridDigest` by default -[%collapsible] -==== -*Details* + -The default implementation for TDigest in percentile calculations switches to a new internal implementation offering superior performance (2x-10x speedup), at a very small accuracy penalty for very large sample populations. - -*Impact* + -This change leads to generating slightly different results in percentile calculations. If the highest possible accuracy is desired, or it's crucial to produce exactly the same results as in previous versions, one can either set `execution_hint` to `high_accuracy` in the `tdigest` spec of a given percentile calculation, or set `search.aggs.tdigest_execution_hint` to `high_accuracy` in cluster settings to apply to all percentile queries. -==== - diff --git a/docs/reference/migration/migrate_9_0.asciidoc b/docs/reference/migration/migrate_9_0.asciidoc new file mode 100644 index 0000000000000..6569647fd993e --- /dev/null +++ b/docs/reference/migration/migrate_9_0.asciidoc @@ -0,0 +1,319 @@ +// THIS IS A GENERATED FILE. DO NOT EDIT DIRECTLY. +// The content generated here are is not correct and most has been manually commented out until it can be fixed. +// See ES-9931 for more details. +[[migrating-9.0]] +== Migrating to 9.0 +++++ +9.0 +++++ + +This section discusses the changes that you need to be aware of when migrating +your application to {es} 9.0. + +See also <> and <>. + +coming::[9.0.0] + + +[discrete] +[[breaking-changes-9.0]] +=== Breaking changes + +The following changes in {es} 9.0 might affect your applications +and prevent them from operating normally. +Before upgrading to 9.0, review these changes and take the described steps +to mitigate the impact. +// +// [discrete] +// [[breaking_90_analysis_changes]] +// ==== Analysis changes +// +// [[set_lenient_to_true_by_default_when_using_updateable_synonyms]] +// .Set lenient to true by default when using updateable synonyms +// [%collapsible] +// ==== +// *Details* + +// When a `synonym` or `synonym_graph` token filter is configured with `updateable: true`, the default `lenient` +// value will now be `true`. +// +// *Impact* + +// `synonym` or `synonym_graph` token filters configured with `updateable: true` will ignore invalid synonyms by +// default. This prevents shard initialization errors on invalid synonyms. +// ==== +// +// [discrete] +// [[breaking_90_mapping_changes]] +// ==== Mapping changes +// +// [[jdk_locale_database_change]] +// .JDK locale database change +// [%collapsible] +// ==== +// *Details* + +// {es} 8.16 changes the version of the JDK that is included from version 22 to version 23. This changes the locale database that is used by Elasticsearch from the COMPAT database to the CLDR database. This change can cause significant differences to the textual date formats accepted by Elasticsearch, and to calculated week-dates. +// +// If you run {es} 8.16 on JDK version 22 or below, it will use the COMPAT locale database to match the behavior of 8.15. However, starting with {es} 9.0, {es} will use the CLDR database regardless of JDK version it is run on. +// +// *Impact* + +// This affects you if you use custom date formats using textual or week-date field specifiers. If you use date fields or calculated week-dates that change between the COMPAT and CLDR databases, then this change will cause Elasticsearch to reject previously valid date fields as invalid data. You might need to modify your ingest or output integration code to account for the differences between these two JDK versions. +// +// Starting in version 8.15.2, Elasticsearch will log deprecation warnings if you are using date format specifiers that might change on upgrading to JDK 23. These warnings are visible in Kibana. +// +// For detailed guidance, refer to <> and the https://ela.st/jdk-23-locales[Elastic blog]. +// ==== +// +// [discrete] +// [[breaking_90_analysis_changes]] +// ==== Analysis changes +// +// [[snowball_stemmers_have_been_upgraded]] +// .Snowball stemmers have been upgraded +// [%collapsible] +// ==== +// *Details* + +// Lucene 10 ships with an upgrade of its Snowball stemmers. For details see https://github.com/apache/lucene/issues/13209. Users using Snowball stemmers that are experiencing changes in search behaviour on existing data are advised to reindex. +// +// *Impact* + +// The upgrade should generally provide improved stemming results. Small changes in token analysis can lead to mismatches with previously index data, so existing indices using Snowball stemmers as part of their analysis chain should be reindexed. +// ==== +// +// [[german2_snowball_stemmer_an_alias_for_german_stemmer]] +// .The "german2" snowball stemmer is now an alias for the "german" stemmer +// [%collapsible] +// ==== +// *Details* + +// Lucene 10 has merged the improved "german2" snowball language stemmer with the "german" stemmer. For Elasticsearch, "german2" is now a deprecated alias for "german". This may results in slightly different tokens being generated for terms with umlaut substitution (like "ue" for "ü" etc...) +// +// *Impact* + +// Replace usages of "german2" with "german" in analysis configuration. Old indices that use the "german" stemmer should be reindexed if possible. +// ==== +// +// [[persian_analyzer_has_stemmer_by_default]] +// .The 'persian' analyzer has stemmer by default +// [%collapsible] +// ==== +// *Details* + +// Lucene 10 has added a final stemming step to its PersianAnalyzer that Elasticsearch exposes as 'persian' analyzer. Existing indices will keep the old non-stemming behaviour while new indices will see the updated behaviour with added stemming. Users that wish to maintain the non-stemming behaviour need to define their own analyzer as outlined in https://www.elastic.co/guide/en/elasticsearch/reference/8.15/analysis-lang-analyzer.html#persian-analyzer. Users that wish to use the new stemming behaviour for existing indices will have to reindex their data. +// +// *Impact* + +// Indexing with the 'persian' analyzer will produce slightly different tokens. Users should check if this impacts their search results. If they wish to maintain the legacy non-stemming behaviour they can define their own analyzer equivalent as explained in https://www.elastic.co/guide/en/elasticsearch/reference/8.15/analysis-lang-analyzer.html#persian-analyzer. +// ==== +// +// [[korean_dictionary_for_nori_has_been_updated]] +// .The Korean dictionary for Nori has been updated +// [%collapsible] +// ==== +// *Details* + +// Lucene 10 ships with an updated Korean dictionary (mecab-ko-dic-2.1.1). For details see https://github.com/apache/lucene/issues/11452. Users experiencing changes in search behaviour on existing data are advised to reindex. +// +// *Impact* + +// The change is small and should generally provide better analysis results. Existing indices for full-text use cases should be reindexed though. +// ==== +// +// [discrete] +// [[breaking_90_cluster_and_node_setting_changes]] +// ==== Cluster and node setting changes +// +// [[remove_unsupported_legacy_value_for_discovery_type]] +// .Remove unsupported legacy value for `discovery.type` +// [%collapsible] +// ==== +// *Details* + +// Earlier versions of {es} had a `discovery.type` setting which permitted values that referred to legacy discovery types. From v9.0.0 onwards, the only supported values for this setting are `multi-node` (the default) and `single-node`. +// +// *Impact* + +// Remove any value for `discovery.type` from your `elasticsearch.yml` configuration file. +// ==== +// +// [discrete] +// [[breaking_90_es_ql_changes]] +// ==== ES|QL changes +// +// [[esql_entirely_remove_meta_functions]] +// .ESQL: Entirely remove META FUNCTIONS +// [%collapsible] +// ==== +// *Details* + +// Removes an undocumented syntax from ESQL: META FUNCTION. This was never +// reliable or really useful. Consult the documentation instead. +// +// *Impact* + +// Removes an undocumented syntax from ESQL: META FUNCTION +// ==== +// +// [discrete] +// [[breaking_90_rest_api_changes]] +// ==== REST API changes +// +// [[remove_cluster_state_from_cluster_reroute_response]] +// .Remove cluster state from `/_cluster/reroute` response +// [%collapsible] +// ==== +// *Details* + +// The `POST /_cluster/reroute` API no longer returns the cluster state in its response. The `?metric` query parameter to this API now has no effect and its use will be forbidden in a future version. +// +// *Impact* + +// Cease usage of the `?metric` query parameter when calling the `POST /_cluster/reroute` API. +// ==== +// +// [[remove_deprecated_local_attribute_from_alias_apis]] +// .Remove deprecated local attribute from alias APIs +// [%collapsible] +// ==== +// *Details* + +// The following APIs no longer accept the `?local` query parameter: `GET /_alias`, `GET /_aliases`, `GET /_alias/{name}`, `HEAD /_alias/{name}`, `GET /{index}/_alias`, `HEAD /{index}/_alias`, `GET /{index}/_alias/{name}`, `HEAD /{index}/_alias/{name}`, `GET /_cat/aliases`, and `GET /_cat/aliases/{alias}`. This parameter has been deprecated and ignored since version 8.12. +// +// *Impact* + +// Cease usage of the `?local` query parameter when calling the listed APIs. +// ==== +// +// [[reworking_rrf_retriever_to_be_evaluated_during_rewrite_phase]] +// .Reworking RRF retriever to be evaluated during rewrite phase +// [%collapsible] +// ==== +// *Details* + +// In this release (8.16), we have introduced major changes to the retrievers framework +// and how they can be evaluated, focusing mainly on compound retrievers +// like `rrf` and `text_similarity_reranker`, which allowed us to support full +// composability (i.e. any retriever can be nested under any compound retriever), +// as well as supporting additional search features like collapsing, explaining, +// aggregations, and highlighting. +// +// To ensure consistency, and given that this rework is not available until 8.16, +// `rrf` and `text_similarity_reranker` retriever queries would now +// throw an exception in a mixed cluster scenario, where there are nodes +// both in current or later (i.e. >= 8.16) and previous ( <= 8.15) versions. +// +// As part of the rework, we have also removed the `_rank` property from +// the responses of an `rrf` retriever. +// +// *Impact* + +// - Users will not be able to use the `rrf` and `text_similarity_reranker` retrievers in a mixed cluster scenario +// with previous releases (i.e. prior to 8.16), and the request will throw an `IllegalArgumentException`. +// - `_rank` has now been removed from the output of the `rrf` retrievers so trying to directly parse the field +// will throw an exception +// ==== +// +// [[update_data_stream_lifecycle_telemetry_to_track_global_retention]] +// .Update data stream lifecycle telemetry to track global retention +// [%collapsible] +// ==== +// *Details* + +// In this release we introduced global retention settings that fulfil the following criteria: +// +// - a data stream managed by the data stream lifecycle, +// - a data stream that is not an internal data stream. +// +// As a result, we defined different types of retention: +// +// - **data retention**: the retention configured on data stream level by the data stream user or owner +// - **default global retention:** the retention configured by an admin on a cluster level and applied to any +// data stream that doesn't have data retention and fulfils the criteria. +// - **max global retention:** the retention configured by an admin to guard against having long retention periods. +// Any data stream that fulfills the criteria will adhere to the data retention unless it exceeds the max retention, +// in which case the max global retention applies. +// - **effective retention:** the retention that applies on the data stream that fulfill the criteria at a given moment +// in time. It takes into consideration all the retention above and resolves it to the retention that will take effect. +// +// Considering the above changes, having a field named `retention` in the usage API was confusing. For this reason, we +// renamed it to `data_retention` and added telemetry about the other configurations too. +// +// *Impact* + +// Users that use the field `data_lifecycle.retention` should use the `data_lifecycle.data_retention` +// ==== + + +[discrete] +[[deprecated-9.0]] +=== Deprecations + +The following functionality has been deprecated in {es} 9.0 +and will be removed in a future version. +While this won't have an immediate impact on your applications, +we strongly encourage you to take the described steps to update your code +after upgrading to 9.0. + +To find out if you are using any deprecated functionality, +enable <>. +// +// [discrete] +// [[deprecations_90_analysis]] +// ==== Analysis deprecations +// +// [[deprecate_dutch_kp_lovins_stemmer_as_they_are_removed_in_lucene_10]] +// .Deprecate dutch_kp and lovins stemmer as they are removed in Lucene 10 +// [%collapsible] +// ==== +// *Details* + +// kp, dutch_kp, dutchKp and lovins stemmers are deprecated and will be removed. +// +// *Impact* + +// These stemmers will be removed and will be no longer supported. +// ==== +// +// [[deprecate_edge_ngram_side_parameter]] +// .deprecate `edge_ngram` side parameter +// [%collapsible] +// ==== +// *Details* + +// edge_ngram will no longer accept the side parameter. +// +// *Impact* + +// Users will need to update any usage of edge_ngram token filter that utilizes `side`. If the `back` value was used, they can achieve the same behavior by using the `reverse` token filter. +// ==== +// +// [discrete] +// [[deprecations_90_crud]] +// ==== CRUD deprecations +// +// [[deprecate_dot_prefixed_indices_composable_template_index_patterns]] +// .Deprecate dot-prefixed indices and composable template index patterns +// [%collapsible] +// ==== +// *Details* + +// Indices beginning with a dot '.' are reserved for system and internal indices, and should not be used by and end-user. Additionally, composable index templates that contain patterns for dot-prefixed indices should also be avoided, as these patterns are meant for internal use only. In a future Elasticsearch version, creation of these dot-prefixed indices will no longer be allowed. +// +// *Impact* + +// Requests performing an action that would create an index beginning with a dot (indexing a document, manual creation, reindex), or creating an index template with index patterns beginning with a dot, will contain a deprecation header warning about dot-prefixed indices in the response. +// ==== +// +// [discrete] +// [[deprecations_90_rest_api]] +// ==== REST API deprecations +// +// [[adding_deprecation_warnings_for_rrf_using_rank_sub_searches]] +// .Adding deprecation warnings for rrf using rank and `sub_searches` +// [%collapsible] +// ==== +// *Details* + +// Search API parameter `sub_searches` will no longer be a supported and will be removed in future releases. Similarly, `rrf` can only be used through the specified `retriever` and no longer though the `rank` parameter +// +// *Impact* + +// Requests specifying rrf through `rank` and/or `sub_searches` elements will be disallowed in a future version. Users should instead utilize the new `retriever` parameter. +// ==== +// +// [[deprecate_legacy_params_from_range_query]] +// .Deprecate legacy params from range query +// [%collapsible] +// ==== +// *Details* + +// Range query will not longer accept `to`, `from`, `include_lower`, and `include_upper` parameters. +// +// *Impact* + +// Instead use `gt`, `gte`, `lt` and `lte` parameters. +// ==== +// +// [[inference_api_deprecate_elser_service]] +// .[Inference API] Deprecate elser service +// [%collapsible] +// ==== +// *Details* + +// The `elser` service of the inference API will be removed in an upcoming release. Please use the elasticsearch service instead. +// +// *Impact* + +// In the current version there is no impact. In a future version, users of the `elser` service will no longer be able to use it, and will be required to use the `elasticsearch` service to access elser through the inference API. +// ==== + +// BELOW WAS MANUALLY ADDED TO FIX THE BUILD +include::migrate_9_0/transient-settings-migration-guide.asciidoc[] +//include::migrate_9_0/rest-api-changes.asciidoc[] //see ES-9932 diff --git a/docs/reference/migration/migrate_9_0/rest-api-changes.asciidoc b/docs/reference/migration/migrate_9_0/rest-api-changes.asciidoc new file mode 100644 index 0000000000000..fc6fc7c011a22 --- /dev/null +++ b/docs/reference/migration/migrate_9_0/rest-api-changes.asciidoc @@ -0,0 +1,5 @@ +[discrete] +[[migrate_rest_api_changes]] +=== REST API changes + +//See https://www.elastic.co/guide/en/elasticsearch/reference/8.0/migrating-8.0.html#breaking_80_rest_api_changes for formatting examples diff --git a/docs/reference/migration/transient-settings-migration-guide.asciidoc b/docs/reference/migration/migrate_9_0/transient-settings-migration-guide.asciidoc similarity index 100% rename from docs/reference/migration/transient-settings-migration-guide.asciidoc rename to docs/reference/migration/migrate_9_0/transient-settings-migration-guide.asciidoc diff --git a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc index 4ee6c429ce730..33692fd182fa7 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc @@ -209,7 +209,7 @@ value. (string) Reserved for future use, currently set to `anomaly_detector`. `job_version`:: -(string) The {ml} configuration version number at which the the job was created. +(string) The {ml} configuration version number at which the job was created. NOTE: From {es} 8.10.0, a new version number is used to track the configuration and state changes in the {ml} plugin. This new diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc index c912b0e62b94d..615e7135365cd 100644 --- a/docs/reference/release-notes.asciidoc +++ b/docs/reference/release-notes.asciidoc @@ -6,135 +6,9 @@ This section summarizes the changes in each release. -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> +* <> -- -include::release-notes/8.17.0.asciidoc[] -include::release-notes/8.16.0.asciidoc[] -include::release-notes/8.15.1.asciidoc[] -include::release-notes/8.15.0.asciidoc[] -include::release-notes/8.14.3.asciidoc[] -include::release-notes/8.14.2.asciidoc[] -include::release-notes/8.14.1.asciidoc[] -include::release-notes/8.14.0.asciidoc[] -include::release-notes/8.13.4.asciidoc[] -include::release-notes/8.13.3.asciidoc[] -include::release-notes/8.13.2.asciidoc[] -include::release-notes/8.13.1.asciidoc[] -include::release-notes/8.13.0.asciidoc[] -include::release-notes/8.12.2.asciidoc[] -include::release-notes/8.12.1.asciidoc[] -include::release-notes/8.12.0.asciidoc[] -include::release-notes/8.11.4.asciidoc[] -include::release-notes/8.11.3.asciidoc[] -include::release-notes/8.11.2.asciidoc[] -include::release-notes/8.11.1.asciidoc[] -include::release-notes/8.11.0.asciidoc[] -include::release-notes/8.10.4.asciidoc[] -include::release-notes/8.10.3.asciidoc[] -include::release-notes/8.10.2.asciidoc[] -include::release-notes/8.10.1.asciidoc[] -include::release-notes/8.10.0.asciidoc[] -include::release-notes/8.9.2.asciidoc[] -include::release-notes/8.9.1.asciidoc[] -include::release-notes/8.9.0.asciidoc[] -include::release-notes/8.8.2.asciidoc[] -include::release-notes/8.8.1.asciidoc[] -include::release-notes/8.8.0.asciidoc[] -include::release-notes/8.7.1.asciidoc[] -include::release-notes/8.7.0.asciidoc[] -include::release-notes/8.6.2.asciidoc[] -include::release-notes/8.6.1.asciidoc[] -include::release-notes/8.6.0.asciidoc[] -include::release-notes/8.5.3.asciidoc[] -include::release-notes/8.5.2.asciidoc[] -include::release-notes/8.5.1.asciidoc[] -include::release-notes/8.5.0.asciidoc[] -include::release-notes/8.4.3.asciidoc[] -include::release-notes/8.4.2.asciidoc[] -include::release-notes/8.4.1.asciidoc[] -include::release-notes/8.4.0.asciidoc[] -include::release-notes/8.3.3.asciidoc[] -include::release-notes/8.3.2.asciidoc[] -include::release-notes/8.3.1.asciidoc[] -include::release-notes/8.3.0.asciidoc[] -include::release-notes/8.2.3.asciidoc[] -include::release-notes/8.2.2.asciidoc[] -include::release-notes/8.2.1.asciidoc[] -include::release-notes/8.2.0.asciidoc[] -include::release-notes/8.1.3.asciidoc[] -include::release-notes/8.1.2.asciidoc[] -include::release-notes/8.1.1.asciidoc[] -include::release-notes/8.1.0.asciidoc[] -include::release-notes/8.0.1.asciidoc[] -include::release-notes/8.0.0.asciidoc[] -include::release-notes/8.0.0-rc2.asciidoc[] -include::release-notes/8.0.0-rc1.asciidoc[] -include::release-notes/8.0.0-beta1.asciidoc[] -include::release-notes/8.0.0-alpha2.asciidoc[] -include::release-notes/8.0.0-alpha1.asciidoc[] +include::release-notes/9.0.0.asciidoc[] diff --git a/docs/reference/release-notes/8.0.0-alpha1.asciidoc b/docs/reference/release-notes/8.0.0-alpha1.asciidoc deleted file mode 100644 index a2c57cd3639f1..0000000000000 --- a/docs/reference/release-notes/8.0.0-alpha1.asciidoc +++ /dev/null @@ -1,473 +0,0 @@ -[[release-notes-8.0.0-alpha1]] -== {es} version 8.0.0-alpha1 - -Also see <>. - -[[breaking-8.0.0-alpha1]] -[float] -=== Breaking changes - -Aggregations:: -* Percentiles aggregation: disallow specifying same percentile values twice {es-pull}52257[#52257] (issue: {es-issue}51871[#51871]) -* Remove Adjacency_matrix setting {es-pull}46327[#46327] (issues: {es-issue}46257[#46257], {es-issue}46324[#46324]) -* Remove `MovingAverage` pipeline aggregation {es-pull}39328[#39328] -* Remove deprecated `_time` and `_term` sort orders {es-pull}39450[#39450] -* Remove deprecated date histo interval {es-pull}75000[#75000] - -Allocation:: -* Breaking change for single data node setting {es-pull}73737[#73737] (issues: {es-issue}55805[#55805], {es-issue}73733[#73733]) -* Remove `include_relocations` setting {es-pull}47717[#47717] (issues: {es-issue}46079[#46079], {es-issue}47443[#47443]) - -Analysis:: -* Cleanup versioned deprecations in analysis {es-pull}41560[#41560] (issue: {es-issue}41164[#41164]) -* Remove preconfigured `delimited_payload_filter` {es-pull}43686[#43686] (issues: {es-issue}41560[#41560], {es-issue}43684[#43684]) - -Authentication:: -* Always add file and native realms unless explicitly disabled {es-pull}69096[#69096] (issue: {es-issue}50892[#50892]) -* Do not set a NameID format in Policy by default {es-pull}44090[#44090] (issue: {es-issue}40353[#40353]) -* Make order setting mandatory for Realm config {es-pull}51195[#51195] (issue: {es-issue}37614[#37614]) - -CCR:: -* Avoid auto following leader system indices in CCR {es-pull}72815[#72815] (issue: {es-issue}67686[#67686]) - -Cluster Coordination:: -* Remove join timeout {es-pull}60873[#60873] (issue: {es-issue}60872[#60872]) -* Remove node filters for voting config exclusions {es-pull}55673[#55673] (issues: {es-issue}47990[#47990], {es-issue}50836[#50836]) -* Remove support for delaying state recovery pending master {es-pull}53845[#53845] (issue: {es-issue}51806[#51806]) - -Distributed:: -* Remove synced flush {es-pull}50882[#50882] (issues: {es-issue}50776[#50776], {es-issue}50835[#50835]) -* Remove the `cluster.remote.connect` setting {es-pull}54175[#54175] (issue: {es-issue}53924[#53924]) - -Engine:: -* Force merge should reject requests with `only_expunge_deletes` and `max_num_segments` set {es-pull}44761[#44761] (issue: {es-issue}43102[#43102]) -* Remove per-type indexing stats {es-pull}47203[#47203] (issue: {es-issue}41059[#41059]) -* Remove translog retention settings {es-pull}51697[#51697] (issue: {es-issue}50775[#50775]) - -Features/CAT APIs:: -* Remove the deprecated `local` parameter for `_cat/indices` {es-pull}64868[#64868] (issue: {es-issue}62198[#62198]) -* Remove the deprecated `local` parameter for `_cat/shards` {es-pull}64867[#64867] (issue: {es-issue}62197[#62197]) - -Features/Features:: -* Remove deprecated ._tier allocation filtering settings {es-pull}73074[#73074] (issue: {es-issue}72835[#72835]) - -Features/ILM+SLM:: -* Add lower bound on `poll_interval` {es-pull}39593[#39593] (issue: {es-issue}39163[#39163]) - -Features/Indices APIs:: -* Change prefer_v2_templates parameter to default to true {es-pull}55489[#55489] (issues: {es-issue}53101[#53101], {es-issue}55411[#55411]) -* Remove deprecated `_upgrade` API {es-pull}64732[#64732] (issue: {es-issue}21337[#21337]) -* Remove local parameter for get field mapping request {es-pull}55100[#55100] (issue: {es-issue}55099[#55099]) -* Remove `include_type_name` parameter from REST layer {es-pull}48632[#48632] (issue: {es-issue}41059[#41059]) -* Remove the `template` field in index templates {es-pull}49460[#49460] (issue: {es-issue}21009[#21009]) - -Features/Watcher:: -* Move watcher history to data stream {es-pull}64252[#64252] - -Geo:: -* Disallow creating `geo_shape` mappings with deprecated parameters {es-pull}70850[#70850] (issue: {es-issue}32039[#32039]) -* Remove bounding box query `type` parameter {es-pull}74536[#74536] - -Infra/Circuit Breakers:: -* Fixed synchronizing inflight breaker with internal variable {es-pull}40878[#40878] - -Infra/Core:: -* Fail when using multiple data paths {es-pull}72184[#72184] (issue: {es-issue}71205[#71205]) -* Limit processors by available processors {es-pull}44894[#44894] (issue: {es-issue}44889[#44889]) -* Remove `nodes/0` folder prefix from data path {es-pull}42489[#42489] -* Remove `bootstrap.system_call_filter` setting {es-pull}72848[#72848] -* Remove `fixed_auto_queue_size` threadpool type {es-pull}52280[#52280] -* Remove `node.max_local_storage_nodes` {es-pull}42428[#42428] (issue: {es-issue}42426[#42426]) -* Remove camel case named formats {es-pull}60044[#60044] -* Remove legacy role settings {es-pull}71163[#71163] (issues: {es-issue}54998[#54998], {es-issue}66409[#66409], {es-issue}71143[#71143]) -* Remove `processors` setting {es-pull}45905[#45905] (issue: {es-issue}45855[#45855]) -* Remove the `local` parameter of `/_cat/nodes` {es-pull}50594[#50594] (issues: {es-issue}50088[#50088], {es-issue}50499[#50499]) -* Remove the listener thread pool {es-pull}53314[#53314] (issue: {es-issue}53049[#53049]) -* Remove the node local storage setting {es-pull}54381[#54381] (issue: {es-issue}54374[#54374]) -* Remove the `pidfile` setting {es-pull}45940[#45940] (issue: {es-issue}45938[#45938]) -* Removes `week_year` date format {es-pull}63384[#63384] (issue: {es-issue}60707[#60707]) - -Infra/Logging:: -* Remove slowlog level {es-pull}57591[#57591] (issue: {es-issue}56171[#56171]) - -Infra/Plugins:: -* Remove deprecated basic license feature enablement settings from 8.0 {es-pull}56211[#56211] (issue: {es-issue}54745[#54745]) - -Infra/REST API:: -* Remove content type required setting {es-pull}61043[#61043] -* Remove deprecated endpoints containing `_xpack` {es-pull}48170[#48170] (issue: {es-issue}35958[#35958]) -* Remove deprecated endpoints of hot threads API {es-pull}55109[#55109] (issue: {es-issue}52640[#52640]) -* Allow parsing Content-Type and Accept headers with version {es-pull}61427[#61427] - -Infra/Resiliency:: -* Fail node containing ancient closed index {es-pull}44264[#44264] (issues: {es-issue}21830[#21830], {es-issue}41731[#41731], {es-issue}44230[#44230]) - -Infra/Scripting:: -* Consolidate script parsing from object {es-pull}59507[#59507] (issue: {es-issue}59391[#59391]) -* Scripting: Move `script_cache` into _nodes/stats {es-pull}59265[#59265] (issues: {es-issue}50152[#50152], {es-issue}59262[#59262]) -* Scripting: Remove general cache settings {es-pull}59262[#59262] (issue: {es-issue}50152[#50152]) - -Infra/Settings:: -* Change default value of `action.destructive_requires_name` to `true` {es-pull}66908[#66908] (issue: {es-issue}61074[#61074]) -* Forbid settings without a namespace {es-pull}45947[#45947] (issues: {es-issue}45905[#45905], {es-issue}45940[#45940]) - -Machine Learning:: -* Remove deprecated `_xpack` endpoints {es-pull}59870[#59870] (issues: {es-issue}35958[#35958], {es-issue}48170[#48170]) -* Remove the ability to update datafeed's `job_id` {es-pull}44752[#44752] (issue: {es-issue}44616[#44616]) - -Mapping:: -* Remove `boost` mapping parameter {es-pull}62639[#62639] (issue: {es-issue}62623[#62623]) -* Remove support for chained multi-fields {es-pull}42333[#42333] (issues: {es-issue}41267[#41267], {es-issue}41926[#41926]) -* Remove support for string in `unmapped_type` {es-pull}45675[#45675] -* Removes typed URLs from mapping APIs {es-pull}41676[#41676] - -Network:: -* Remove client feature tracking {es-pull}44929[#44929] (issues: {es-issue}31020[#31020], {es-issue}42538[#42538], {es-issue}44667[#44667]) -* Remove escape hatch permitting incompatible builds {es-pull}65753[#65753] (issues: {es-issue}65249[#65249], {es-issue}65601[#65601]) - -Packaging:: -* Remove SysV init support {es-pull}51716[#51716] (issue: {es-issue}51480[#51480]) -* Remove support for `JAVA_HOME` {es-pull}69149[#69149] (issue: {es-issue}55820[#55820]) - -Recovery:: -* Remove dangling index auto import functionality {es-pull}59698[#59698] (issue: {es-issue}48366[#48366]) - -Reindex:: -* Reindex from Remote encoding {es-pull}41007[#41007] (issue: {es-issue}40303[#40303]) -* Reindex remove outer level size {es-pull}43373[#43373] (issues: {es-issue}24344[#24344], {es-issue}41894[#41894]) - -Rollup:: -* `RollupStart` endpoint should return OK if job already started {es-pull}41502[#41502] (issues: {es-issue}35928[#35928], {es-issue}39845[#39845]) - -Search:: -* Decouple shard allocation awareness from search and get requests {es-pull}45735[#45735] (issue: {es-issue}43453[#43453]) -* Fix range query on date fields for number inputs {es-pull}63692[#63692] (issue: {es-issue}63680[#63680]) -* Make fuzziness reject illegal values earlier {es-pull}33511[#33511] -* Make remote cluster resolution stricter {es-pull}40419[#40419] (issue: {es-issue}37863[#37863]) -* Parse empty first line in msearch request body as action metadata {es-pull}41011[#41011] (issue: {es-issue}39841[#39841]) -* Remove `CommonTermsQuery` and `cutoff_frequency` param {es-pull}42654[#42654] (issue: {es-issue}37096[#37096]) -* Remove `type` query {es-pull}47207[#47207] (issue: {es-issue}41059[#41059]) -* Remove `use_field_mapping` format option for docvalue fields {es-pull}55622[#55622] -* Remove deprecated `SimpleQueryStringBuilder` parameters {es-pull}57200[#57200] -* Remove deprecated `search.remote` settings {es-pull}42381[#42381] (issues: {es-issue}33413[#33413], {es-issue}38556[#38556]) -* Remove deprecated sort options: `nested_path` and `nested_filter` {es-pull}42809[#42809] (issue: {es-issue}27098[#27098]) -* Remove deprecated vector functions {es-pull}48725[#48725] (issue: {es-issue}48604[#48604]) -* Remove support for `_type` in searches {es-pull}68564[#68564] (issues: {es-issue}41059[#41059], {es-issue}68311[#68311]) -* Remove support for sparse vectors {es-pull}48781[#48781] (issue: {es-issue}48368[#48368]) -* Remove the object format for `indices_boost` {es-pull}55078[#55078] -* Removes type from `TermVectors` APIs {es-pull}42198[#42198] (issue: {es-issue}41059[#41059]) -* Removes typed endpoint from search and related APIs {es-pull}41640[#41640] -* Set max allowed size for stored async response {es-pull}74455[#74455] (issue: {es-issue}67594[#67594]) -* `indices.query.bool.max_clause_count` now limits all query clauses {es-pull}75297[#75297] - -Security:: -* Remove obsolete security settings {es-pull}40496[#40496] -* Remove support of creating CA on the fly when generating certificates {es-pull}65590[#65590] (issue: {es-issue}61884[#61884]) -* Remove the `id` field from the `InvalidateApiKey` API {es-pull}66671[#66671] (issue: {es-issue}66317[#66317]) -* Remove the migrate tool {es-pull}42174[#42174] -* Compress audit logs {es-pull}64472[#64472] (issue: {es-issue}63843[#63843]) -* Remove insecure settings {es-pull}46147[#46147] (issue: {es-issue}45947[#45947]) - -Snapshot/Restore:: -* Blob store compress default to `true` {es-pull}40033[#40033] -* Get snapshots support for multiple repositories {es-pull}42090[#42090] (issue: {es-issue}41210[#41210]) -* Remove repository stats API {es-pull}62309[#62309] (issue: {es-issue}62297[#62297]) -* Remove frozen cache setting leniency {es-pull}71013[#71013] (issue: {es-issue}70341[#70341]) - -TLS:: -* Reject misconfigured/ambiguous SSL server config {es-pull}45892[#45892] -* Remove support for configurable PKCS#11 keystores {es-pull}75404[#75404] -* Remove the client transport profile filter {es-pull}43236[#43236] - - - -[[breaking-java-8.0.0-alpha1]] -[float] -=== Breaking Java changes - -Authentication:: -* Mandate x-pack REST handler installed {es-pull}71061[#71061] (issue: {es-issue}70523[#70523]) - -CCR:: -* Remove the `CcrClient` {es-pull}42816[#42816] - -CRUD:: -* Remove types from `BulkRequest` {es-pull}46983[#46983] (issue: {es-issue}41059[#41059]) -* Remove `Client.prepareIndex(index, type, id)` method {es-pull}48443[#48443] -* Remove deprecated `include-type` methods from HLRC indices client {es-pull}48471[#48471] - - -Client:: -* Remove `SecurityClient` from x-pack {es-pull}42471[#42471] - -Features/ILM+SLM:: -* Remove the `ILMClient` {es-pull}42817[#42817] -* Rename HLRC `indexlifecycle` components to `ilm` {es-pull}44982[#44982] (issues: {es-issue}44725[#44725], {es-issue}44917[#44917]) - -Features/Monitoring:: -* Remove `MonitoringClient` from x-pack {es-pull}42770[#42770] - -Features/Watcher:: -* Remove `WatcherClient` from x-pack {es-pull}42815[#42815] - -Infra/Core:: -* Remove `XPackClient` from x-pack {es-pull}42729[#42729] -* Remove the transport client {es-pull}42538[#42538] -* Remove transport client from x-pack {es-pull}42202[#42202] - -Infra/REST API:: -* Copy HTTP headers to `ThreadContext` strictly {es-pull}45945[#45945] - -Machine Learning:: -* Remove the `MachineLearningClient` {es-pull}43108[#43108] - -Mapping:: -* Remove type filter from `GetMappings` API {es-pull}47364[#47364] (issue: {es-issue}41059[#41059]) -* Remove `type` parameter from `PutMappingRequest.buildFromSimplifiedDef()` {es-pull}50844[#50844] (issue: {es-issue}41059[#41059]) -* Remove unused parameter from `MetadataFieldMapper.TypeParser#getDefault()` {es-pull}51219[#51219] -* Remove `type` parameter from `CIR.mapping(type, object...)` {es-pull}50739[#50739] (issue: {es-issue}41059[#41059]) - -Search:: -* Removes types from `SearchRequest` and `QueryShardContext` {es-pull}42112[#42112] - -Snapshot/Restore:: -* Remove deprecated repository methods {es-pull}42359[#42359] (issue: {es-issue}42213[#42213]) - - - -[[enhancement-8.0.0-alpha1]] -[float] -=== Enhancements - -Analysis:: -* Move `reload_analyzers` endpoint to x-pack {es-pull}43559[#43559] - -Authentication:: -* Reset elastic password CLI tool {es-pull}74892[#74892] (issues: {es-issue}70113[#70113], {es-issue}74890[#74890]) - -EQL:: -* Add option for returning results from the tail of the stream {es-pull}64869[#64869] (issue: {es-issue}58646[#58646]) -* Introduce case insensitive variant `in~` {es-pull}68176[#68176] (issue: {es-issue}68172[#68172]) -* Optimize redundant `toString` {es-pull}71070[#71070] (issue: {es-issue}70681[#70681]) - -Engine:: -* Always use soft-deletes in `InternalEngine` {es-pull}50415[#50415] -* Remove translog retention policy {es-pull}51417[#51417] (issue: {es-issue}50775[#50775]) - -Features/CAT APIs:: -* Remove `size` and add `time` params to `_cat/threadpool` {es-pull}55736[#55736] (issue: {es-issue}54478[#54478]) - -Features/Stats:: -* Add bulk stats track the bulk per shard {es-pull}52208[#52208] (issues: {es-issue}47345[#47345], {es-issue}50536[#50536]) - - -Features/Watcher:: -* Remove Watcher history clean up from monitoring {es-pull}67154[#67154] - -Infra/Core:: -* Remove aliases exist action {es-pull}43430[#43430] -* Remove indices exists action {es-pull}43164[#43164] -* Remove types exists action {es-pull}43344[#43344] - -Infra/Logging:: -* Make Elasticsearch JSON logs ECS compliant {es-pull}47105[#47105] (issue: {es-issue}46119[#46119]) - -Infra/REST API:: -* Allow for field declaration for future compatible versions {es-pull}69774[#69774] (issue: {es-issue}51816[#51816]) -* Introduce stability description to the REST API specification {es-pull}38413[#38413] -* Parsing: Validate that fields are not registered twice {es-pull}70243[#70243] -* Support response content-type with versioned media type {es-pull}65500[#65500] (issue: {es-issue}51816[#51816]) -* [REST API Compatibility] Typed endpoints for index and get APIs {es-pull}69131[#69131] (issue: {es-issue}54160[#54160]) -* [REST API Compatibility] Typed endpoints for put and get mapping and get field mappings {es-pull}71721[#71721] (issues: {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Allow `copy_settings` flag for resize operations {es-pull}75184[#75184] (issues: {es-issue}38514[#38514], {es-issue}51816[#51816]) -* [REST API Compatibility] Allow for type in geo shape query {es-pull}74553[#74553] (issues: {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Always return `adjust_pure_negative` value {es-pull}75182[#75182] (issues: {es-issue}49543[#49543], {es-issue}51816[#51816]) -* [REST API Compatibility] Clean up x-pack/plugin rest compat tests {es-pull}74701[#74701] (issue: {es-issue}51816[#51816]) -* [REST API Compatibility] Do not return `_doc` for empty mappings in template {es-pull}75448[#75448] (issues: {es-issue}51816[#51816], {es-issue}54160[#54160], {es-issue}70966[#70966], {es-issue}74544[#74544]) -* [REST API Compatibility] Dummy REST action for `indices.upgrade` API {es-pull}75136[#75136] (issue: {es-issue}51816[#51816]) -* [REST API Compatibility] REST Terms vector typed response {es-pull}73117[#73117] -* [REST API Compatibility] Rename `BulkItemResponse.Failure` type field {es-pull}74937[#74937] (issue: {es-issue}51816[#51816]) -* [REST API Compatibility] Type metadata for docs used in simulate request {es-pull}74222[#74222] (issues: {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Typed `TermLookups` {es-pull}74544[#74544] (issues: {es-issue}46943[#46943], {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Typed and x-pack graph explore API {es-pull}74185[#74185] (issues: {es-issue}46935[#46935], {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Typed endpoint for bulk API {es-pull}73571[#73571] (issue: {es-issue}51816[#51816]) -* [REST API Compatibility] Typed endpoint for multi-get API {es-pull}73878[#73878] (issue: {es-issue}51816[#51816]) -* [REST API Compatibility] Typed endpoints for `RestUpdateAction` and `RestDeleteAction` {es-pull}73115[#73115] (issues: {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Typed endpoints for `get_source` API {es-pull}73957[#73957] (issues: {es-issue}46587[#46587], {es-issue}46931[#46931], {es-issue}51816[#51816]) -* [REST API Compatibility] Typed endpoints for explain API {es-pull}73901[#73901] (issue: {es-issue}51816[#51816]) -* [REST API Compatibility] Typed endpoints for search `_count` API {es-pull}73958[#73958] (issues: {es-issue}42112[#42112], {es-issue}51816[#51816]) -* [REST API Compatibility] Typed indexing stats {es-pull}74181[#74181] (issues: {es-issue}47203[#47203], {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Types for percolate query API {es-pull}74698[#74698] (issues: {es-issue}46985[#46985], {es-issue}51816[#51816], {es-issue}54160[#54160], {es-issue}74689[#74689]) -* [REST API Compatibility] Validate query typed API {es-pull}74171[#74171] (issues: {es-issue}46927[#46927], {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Voting config exclusion exception message {es-pull}75406[#75406] (issues: {es-issue}51816[#51816], {es-issue}55291[#55291]) -* [REST API Compatibility] `MoreLikeThisQuery` with types {es-pull}75123[#75123] (issues: {es-issue}42198[#42198], {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Update and delete by query using size field {es-pull}69606[#69606] -* [REST API Compatibility] Indicies boost in object format {es-pull}74422[#74422] (issues: {es-issue}51816[#51816], {es-issue}55078[#55078]) -* [REST API Compatibility] Typed endpoints for search and related endpoints {es-pull}72155[#72155] (issues: {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Allow to use size `-1` {es-pull}75342[#75342] (issues: {es-issue}51816[#51816], {es-issue}69548[#69548], {es-issue}70209[#70209]) -* [REST API Compatibility] Ignore `use_field_mapping` option for docvalue {es-pull}74435[#74435] (issue: {es-issue}55622[#55622]) -* [REST API Compatibility] `_time` and `_term` sort orders {es-pull}74919[#74919] (issues: {es-issue}39450[#39450], {es-issue}51816[#51816]) -* [REST API Compatability] `template` parameter and field on PUT index template {es-pull}71238[#71238] (issues: {es-issue}49460[#49460], {es-issue}51816[#51816], {es-issue}68905[#68905]) -* [REST API Compatibility] Make query registration easier {es-pull}75722[#75722] (issue: {es-issue}51816[#51816]) -* [REST API Compatibility] Typed query {es-pull}75453[#75453] (issues: {es-issue}47207[#47207], {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Deprecate the use of synced flush {es-pull}75372[#75372] (issues: {es-issue}50882[#50882], {es-issue}51816[#51816]) -* [REST API Compatibility] Licence `accept_enterprise` and response changes {es-pull}75479[#75479] (issues: {es-issue}50067[#50067], {es-issue}50735[#50735], {es-issue}51816[#51816], {es-issue}58217[#58217]) - -Infra/Scripting:: -* Update `DeprecationMap` to `DynamicMap` {es-pull}56149[#56149] (issue: {es-issue}52103[#52103]) - -Infra/Settings:: -* Fixed inconsistent `Setting.exist()` {es-pull}46603[#46603] (issue: {es-issue}41830[#41830]) -* Remove `index.optimize_auto_generated_id` setting (#27583) {es-pull}27600[#27600] (issue: {es-issue}27583[#27583]) - -License:: -* Add deprecated `accept_enterprise` param to `/_xpack` {es-pull}58220[#58220] (issue: {es-issue}58217[#58217]) -* Support `accept_enterprise` param in get license API {es-pull}50067[#50067] (issue: {es-issue}49474[#49474]) - -Machine Learning:: -* The Windows build platform for the {ml} C++ code now uses Visual Studio 2019 {ml-pull}1352[#1352] -* The macOS build platform for the {ml} C++ code is now Mojave running Xcode 11.3.1, - or Ubuntu 20.04 running clang 8 for cross compilation {ml-pull}1429[#1429] -* The Linux build platform for the {ml} C++ code is now CentOS 7 running gcc 9.3 {ml-pull}1170[#1170] -* Add a new application for evaluating PyTorch models. The app depends on LibTorch - the C++ front end to PyTorch - and performs inference on models stored in the TorchScript format {ml-pull}1902[#1902] - -Mapping:: -* Sparse vector to throw exception consistently {es-pull}62646[#62646] - -Packaging:: -* Make the Docker build more re-usable in Cloud {es-pull}50277[#50277] (issues: {es-issue}46166[#46166], {es-issue}49926[#49926]) -* Update docker-compose.yml to fix bootstrap check error {es-pull}47650[#47650] - -Recovery:: -* Use Lucene index in peer recovery and resync {es-pull}51189[#51189] (issue: {es-issue}50775[#50775]) - -Reindex:: -* Make reindexing managed by a persistent task {es-pull}43382[#43382] (issue: {es-issue}42612[#42612]) -* Reindex restart from checkpoint {es-pull}46055[#46055] (issue: {es-issue}42612[#42612]) -* Reindex search resiliency {es-pull}45497[#45497] (issues: {es-issue}42612[#42612], {es-issue}43187[#43187]) -* Reindex v2 rethrottle sliced fix {es-pull}46967[#46967] (issues: {es-issue}42612[#42612], {es-issue}46763[#46763]) - -Rollup:: -* Adds support for `date_nanos` in Rollup Metric and `DateHistogram` Configs {es-pull}59349[#59349] (issue: {es-issue}44505[#44505]) - -SQL:: -* Add text formatting support for multivalue {es-pull}68606[#68606] -* Add xDBC and CLI support. QA CSV specs {es-pull}68966[#68966] -* Export array values through result sets {es-pull}69512[#69512] -* Improve alias resolution in sub-queries {es-pull}67216[#67216] (issue: {es-issue}56713[#56713]) -* Improve the optimization of null conditionals {es-pull}71192[#71192] -* Push `WHERE` clause inside subqueries {es-pull}71362[#71362] -* Use Java `String` methods for `LTRIM/RTRIM` {es-pull}57594[#57594] -* QL: Make canonical form take into account children {es-pull}71266[#71266] -* QL: Polish optimizer expression rule declaration {es-pull}71396[#71396] -* QL: Propagate nullability constraints across conjunctions {es-pull}71187[#71187] (issue: {es-issue}70683[#70683]) - -Search:: -* Completely disallow setting negative size in search {es-pull}70209[#70209] (issue: {es-issue}69548[#69548]) -* Make `0` as invalid value for `min_children` in `has_child` query {es-pull}41347[#41347] -* Return error when remote indices are locally resolved {es-pull}74556[#74556] (issue: {es-issue}26247[#26247]) - -Security:: -* Add a tool for creating enrollment tokens {es-pull}74890[#74890] -* Add the Enroll Kibana API {es-pull}72207[#72207] -* Change default hashing algorithm for FIPS 140 {es-pull}55544[#55544] -* Create enrollment token {es-pull}73573[#73573] (issues: {es-issue}71438[#71438], {es-issue}72129[#72129]) -* Enroll node API {es-pull}72129[#72129] -* Not encoding the Api Key in Enrollment token {es-pull}74510[#74510] (issue: {es-issue}73573[#73573]) -* Configure security for the initial node CLI {es-pull}74868[#74868] - -Snapshot/Restore:: -* Introduce searchable snapshots index setting for cascade deletion of snapshots {es-pull}74977[#74977] -* Unify blob store compress setting {es-pull}39346[#39346] (issue: {es-issue}39073[#39073]) -* Add recovery state tracking for searchable snapshots {es-pull}60505[#60505] - -TLS:: -* Add `ChaCha20` TLS ciphers on Java 12+ {es-pull}42155[#42155] -* Add support for `KeyStore` filters to `ssl-config` {es-pull}75407[#75407] -* Update TLS ciphers and protocols for JDK 11 {es-pull}41808[#41808] (issues: {es-issue}38646[#38646], {es-issue}41385[#41385]) - - - -[[bug-8.0.0-alpha1]] -[float] -=== Bug fixes - -Aggregations:: -* Fix BWC issues for `x_pack/usage` {es-pull}55181[#55181] (issue: {es-issue}54847[#54847]) -* Fix `DoubleBounds` null serialization {es-pull}59475[#59475] -* Fix `TopHitsAggregationBuilder` adding duplicate `_score` sort clauses {es-pull}42179[#42179] (issue: {es-issue}42154[#42154]) -* Fix `t_test` usage stats {es-pull}54753[#54753] (issue: {es-issue}54744[#54744]) -* Throw exception if legacy interval cannot be parsed in `DateIntervalWrapper` {es-pull}41972[#41972] (issue: {es-issue}41970[#41970]) - -CCR:: -* Fix `AutoFollow` version checks {es-pull}73776[#73776] (issue: {es-issue}72935[#72935]) - -Cluster Coordination:: -* Apply cluster states in system context {es-pull}53785[#53785] (issue: {es-issue}53751[#53751]) - -Distributed:: -* Introduce `?wait_for_active_shards=index-setting` {es-pull}67158[#67158] (issue: {es-issue}66419[#66419]) -* Respect `CloseIndexRequest#waitForActiveShards` in HLRC {es-pull}67374[#67374] (issues: {es-issue}67158[#67158], {es-issue}67246[#67246]) -* Fixes to task result index mapping {es-pull}50359[#50359] (issue: {es-issue}50248[#50248]) - -Features/CAT APIs:: -* Fix cat recovery display of bytes fields {es-pull}40379[#40379] (issue: {es-issue}40335[#40335]) - -Features/Java High Level REST Client:: -* Fix HLRC compatibility with Java 8 {es-pull}74290[#74290] (issues: {es-issue}73910[#73910], {es-issue}74272[#74272], {es-issue}74289[#74289]) -* Avoid `StackOverflowError` due to regex alternate paths {es-pull}61259[#61259] (issue: {es-issue}60889[#60889]) - -Geo:: -* Preprocess polygon rings before processing it for decomposition {es-pull}59501[#59501] (issues: {es-issue}54441[#54441], {es-issue}59386[#59386]) - -Infra/Core:: -* Add searchable snapshot cache folder to `NodeEnvironment` {es-pull}66297[#66297] (issue: {es-issue}65725[#65725]) -* CLI tools: Write errors to stderr instead of stdout {es-pull}45586[#45586] (issue: {es-issue}43260[#43260]) -* Precompute `ParsedMediaType` for XContentType {es-pull}67409[#67409] - -Infra/Logging:: -* Fix NPE when logging null values in JSON {es-pull}53715[#53715] (issue: {es-issue}46702[#46702]) -* Fix stats in slow logs to be a escaped JSON {es-pull}44642[#44642] -* Populate data stream fields when `xOpaqueId` not provided {es-pull}62156[#62156] - -Infra/REST API:: -* Do not allow spaces within `MediaType's` parameters {es-pull}64650[#64650] (issue: {es-issue}51816[#51816]) -* Handle incorrect header values {es-pull}64708[#64708] (issues: {es-issue}51816[#51816], {es-issue}64689[#64689]) -* Ignore media ranges when parsing {es-pull}64721[#64721] (issues: {es-issue}51816[#51816], {es-issue}64689[#64689]) -* `RestController` should not consume request content {es-pull}44902[#44902] (issue: {es-issue}37504[#37504]) - -Infra/Scripting:: -* Change compound assignment structure to support string concatenation {es-pull}61825[#61825] -* Fixes casting in constant folding {es-pull}61508[#61508] -* Several minor Painless fixes {es-pull}61594[#61594] - -Machine Learning:: -* Handle null value of `FieldCapabilitiesResponse` {es-pull}64327[#64327] - -Mapping:: -* Remove assertions that mappings have one top-level key {es-pull}58779[#58779] (issue: {es-issue}58521[#58521]) - -Packaging:: -* Suppress illegal access in plugin install {es-pull}41620[#41620] (issue: {es-issue}41478[#41478]) - -SQL:: -* Introduce dedicated node for `HAVING` declaration {es-pull}71279[#71279] (issue: {es-issue}69758[#69758]) -* Make `RestSqlQueryAction` thread-safe {es-pull}69901[#69901] - -Search:: -* Check for negative `from` values in search request body {es-pull}54953[#54953] (issue: {es-issue}54897[#54897]) -* Fix `VectorsFeatureSetUsage` serialization in BWC mode {es-pull}55399[#55399] (issue: {es-issue}55378[#55378]) -* Handle total hits equal to `track_total_hits` {es-pull}37907[#37907] (issue: {es-issue}37897[#37897]) -* Improve error msg for CCS request on node without remote cluster role {es-pull}60351[#60351] (issue: {es-issue}59683[#59683]) - -Snapshot/Restore:: -* Fix `GET /_snapshot/_all/_all` if there are no repos {es-pull}43558[#43558] (issue: {es-issue}43547[#43547]) - - -[[upgrade-8.0.0-alpha1]] -[float] -=== Upgrades - -Lucene:: -* Upgrade to Lucene 8.9.0 {es-pull}74729[#74729] - diff --git a/docs/reference/release-notes/8.0.0-alpha2.asciidoc b/docs/reference/release-notes/8.0.0-alpha2.asciidoc deleted file mode 100644 index 3d3c0f6335d26..0000000000000 --- a/docs/reference/release-notes/8.0.0-alpha2.asciidoc +++ /dev/null @@ -1,77 +0,0 @@ -[[release-notes-8.0.0-alpha2]] -== {es} version 8.0.0-alpha2 - -Also see <>. - -[[breaking-8.0.0-alpha2]] -[float] -=== Breaking changes - - -ILM+SLM:: -* Make the ILM `freeze` action a no-op {es-pull}77158[#77158] (issue: {es-issue}70192[#70192]) - -Infra/Core:: -* Fail index creation using custom data path {es-pull}76792[#76792] (issue: {es-issue}73168[#73168]) -* System indices treated as restricted indices {es-pull}74212[#74212] (issue: {es-issue}69298[#69298]) - -License:: -* Set `xpack.security.enabled` to true for all licenses {es-pull}72300[#72300] - -Packaging:: -* Remove no-jdk distributions {es-pull}76896[#76896] (issue: {es-issue}65109[#65109]) - -Security:: -* Remove `kibana_dashboard_only_user` reserved role {es-pull}76507[#76507] - - -[[enhancement-8.0.0-alpha2]] -[float] -=== Enhancements - -Authentication:: -* Autogenerate and print elastic password on startup {es-pull}77291[#77291] -* Enroll Kibana API uses Service Accounts {es-pull}76370[#76370] -* Add `reset-kibana-system-user` tool {es-pull}77322[#77322] - -ILM+SLM:: -* Allow for setting the total shards per node in the Allocate ILM action {es-pull}76794[#76794] (issue: {es-issue}76775[#76775]) - -Infra/Core:: -* Retain reference to stdout for exceptional cases {es-pull}77460[#77460] - -Ingest:: -* Add support for `_meta` field to ingest pipelines {es-pull}76381[#76381] - -Machine Learning:: -* Adding new PUT trained model vocabulary endpoint {es-pull}77387[#77387] -* Creating new PUT model definition part API {es-pull}76987[#76987] - -Network:: -* Enable LZ4 transport compression by default {es-pull}76326[#76326] (issue: {es-issue}73497[#73497]) - -Search:: -* [REST API Compatibility] Nested path and filter sort options {es-pull}76022[#76022] (issues: {es-issue}42809[#42809], {es-issue}51816[#51816]) -* [REST API Compatibility] `CommonTermsQuery` and `cutoff_frequency` parameter {es-pull}75896[#75896] (issues: {es-issue}42654[#42654], {es-issue}51816[#51816]) -* [REST API Compatibility] Allow first empty line for `_msearch` {es-pull}75886[#75886] (issues: {es-issue}41011[#41011], {es-issue}51816[#51816]) - - -Security:: -* Adding base `RestHandler` class for Enrollment APIs {es-pull}76564[#76564] (issue: {es-issue}76097[#76097]) -* Generate and store password hash for elastic user {es-pull}76276[#76276] (issue: {es-issue}75310[#75310]) -* Set elastic password and generate enrollment token {es-pull}75816[#75816] (issue: {es-issue}75310[#75310]) -* Add `elasticsearch-enroll-node` tool {es-pull}77292[#77292] -* Default hasher to `PBKDF2_STRETCH` on FIPS mode {es-pull}76274[#76274] - - -[[bug-8.0.0-alpha2]] -[float] -=== Bug Fixes - -ILM+SLM:: -* Ensuring that the `ShrinkAction` does not hang if total shards per node is too low {es-pull}76732[#76732] (issue: {es-issue}44070[#44070]) - - -Security:: -* Allow access to restricted system indices for reserved system roles {es-pull}76845[#76845] - diff --git a/docs/reference/release-notes/8.0.0-beta1.asciidoc b/docs/reference/release-notes/8.0.0-beta1.asciidoc deleted file mode 100644 index 87a2283c9d423..0000000000000 --- a/docs/reference/release-notes/8.0.0-beta1.asciidoc +++ /dev/null @@ -1,238 +0,0 @@ -:es-issue: https://github.com/elastic/elasticsearch/issues/ -:es-pull: https://github.com/elastic/elasticsearch/pull/ - -[[release-notes-8.0.0-beta1]] -== {es} version 8.0.0-beta1 - -Also see <>. - -[[known-issues-8.0.0-beta1]] -[float] -=== Known issues - -* If you're using {ml}, it's not safe to upgrade to `8.0.0-beta1` - if the cluster you're upgrading was first used prior to `7.7.0`. - If you attempt such an upgrade the filtered aliases against - the {ml} results indices will lose their filters, causing {ml} - anomaly detection results for different jobs to be mixed. - Wait for `8.0.0-rc1` before attempting to test upgrading a cluster - where {ml} was used prior to version `7.7.0`. -* System indices are included in wildcard queries when using the `*` pattern. Do - not rely on this behavior, because system indices will be hidden in the 8.0 - release. If you need to access system indices, specify the index name, use a - more specific wildcard, or use the `expand_wildcards` parameter in your query. - -* By default, system indices are included in snapshots of all indices, and are - restored when no indices are specified. This behavior results from including - system indices in the implicit default wildcard (`*`) for snapshot and restore - operations, which was intended for backwards compatibility in 7.x versions. - In 8.0, all system indices should be accessed through the `feature_states` - parameter of the snapshot or restore request. - - -[[breaking-8.0.0-beta1]] -[float] -=== Breaking changes - -ILM+SLM:: -* Always enforce default tier preference {es-pull}79751[#79751] (issue: {es-issue}76147[#76147]) -* Validate that snapshot repository exists for ILM policies at creation/update time {es-pull}78468[#78468] (issues: {es-issue}72957[#72957], {es-issue}77657[#77657]) -* Default `ENFORCE_DEFAULT_TIER_PREFERENCE` to `true` {es-pull}79275[#79275] (issues: {es-issue}76147[#76147], {es-issue}79210[#79210]) - -Indices APIs:: -* Remove endpoint for freezing indices {es-pull}78918[#78918] (issues: {es-issue}70192[#70192], {es-issue}77273[#77273]) - -Infra/Core:: -* Remove Joda dependency {es-pull}79007[#79007] -* Remove Joda support from date formatters {es-pull}78990[#78990] - -Ingest:: -* Remove default maxmind GeoIP databases from distribution {es-pull}78362[#78362] (issue: {es-issue}68920[#68920]) - -License:: -* Enforce license expiration {es-pull}79671[#79671] - -Machine Learning:: -* Remove `allow_no_datafeeds` and `allow_no_jobs` parameters from APIs {es-pull}80048[#80048] (issue: {es-issue}60732[#60732]) - -Packaging:: -* Require Java 17 for running Elasticsearch {es-pull}79873[#79873] - - - -[[deprecation-8.0.0-beta1]] -[float] -=== Deprecations - -Authentication:: -* Deprecate setup-passwords tool {es-pull}76902[#76902] - -CRUD:: -* Remove `indices_segments` 'verbose' parameter {es-pull}78451[#78451] (issue: {es-issue}75955[#75955]) - -Monitoring:: -* Add deprecation info API entries for deprecated monitoring settings {es-pull}78799[#78799] -* Automatically install monitoring templates at plugin initialization {es-pull}78350[#78350] -* Remove Monitoring ingest pipelines {es-pull}77459[#77459] (issue: {es-issue}50770[#50770]) - - - -[[feature-8.0.0-beta1]] -[float] -=== New features - -Security:: -* Auto-configure TLS for new nodes of new clusters {es-pull}77231[#77231] (issues: {es-issue}75144[#75144], {es-issue}75704[#75704]) - -[[enhancement-8.0.0-beta1]] -[float] -=== Enhancements - -Authentication:: -* New CLI tool to reset password for built-in users {es-pull}79709[#79709] -* Upgrade to UnboundID LDAP SDK v6.0.2 {es-pull}79332[#79332] -* Auto-configure the `elastic` user password {es-pull}78306[#78306] - -Cluster Coordination:: -* Prevent downgrades from 8.x to 7.x {es-pull}78586[#78586] (issues: {es-issue}42489[#42489], {es-issue}52414[#52414]) -* Prevent downgrades from 8.x to 7.x {es-pull}78638[#78638] (issues: {es-issue}42489[#42489], {es-issue}52414[#52414]) - -Data streams:: -* Data stream support read and write with custom routing and partition size {es-pull}74394[#74394] (issue: {es-issue}74390[#74390]) - -ILM+SLM:: - -* Inject migrate action regardless of allocate action {es-pull}79090[#79090] (issue: {es-issue}76147[#76147]) - -Infra/Core:: -* Check whether stdout is a real console {es-pull}79882[#79882] -* Share int, long, float, double, and byte pages {es-pull}75053[#75053] -* Revert "Deprecate resolution loss on date field (#78921)" {es-pull}79914[#79914] (issue: {es-issue}78921[#78921]) -* Add two missing entries to the deprecation information API {es-pull}80290[#80290] (issue: {es-issue}80233[#80233]) - -Infra/Scripting:: -* Add nio Buffers to Painless {es-pull}79870[#79870] (issue: {es-issue}79867[#79867]) -* Restore the scripting general cache {es-pull}79453[#79453] (issue: {es-issue}62899[#62899]) - -Ingest:: -* Remove binary field after attachment processor execution {es-pull}79172[#79172] -* Improving cache lookup to reduce recomputing / searches {es-pull}77259[#77259] - - -License:: -* Enforce Transport TLS check on all licenses {es-pull}79602[#79602] (issue: {es-issue}75292[#75292]) - -Machine Learning:: -* Add inference time configuration overrides {es-pull}78441[#78441] (issue: {es-issue}77799[#77799]) -* Optimize source extraction for `categorize_text` aggregation {es-pull}79099[#79099] -* The Linux build platform for the {ml} C++ code is now CentOS 7 running gcc 10.3. {ml-pull}2028[#2028] -* Make ML indices hidden when the node becomes master {es-pull}77416[#77416] (issue: {es-issue}53674[#53674]) - -Mapping:: -* Add support for configuring HNSW parameters {es-pull}79193[#79193] (issue: {es-issue}78473[#78473]) -* Extend `dense_vector` to support indexing vectors {es-pull}78491[#78491] (issue: {es-issue}78473[#78473]) - -Monitoring:: -* Add previously removed Monitoring settings back for 8.0 {es-pull}78784[#78784] -* Change Monitoring plugin cluster alerts to not install by default {es-pull}79657[#79657] - -Packaging:: -* Allow total memory to be overridden {es-pull}78750[#78750] (issue: {es-issue}65905[#65905]) - -Search:: -* Node level can match action {es-pull}78765[#78765] -* TSDB: Add time series information to field caps {es-pull}78790[#78790] (issue: {es-issue}74660[#74660]) -* Add new kNN search endpoint {es-pull}79013[#79013] (issue: {es-issue}78473[#78473]) -* Disallow kNN searches on nested vector fields {es-pull}79403[#79403] (issue: {es-issue}78473[#78473]) -* Ensure kNN search respects authorization {es-pull}79693[#79693] (issue: {es-issue}78473[#78473]) -* Load kNN vectors format with mmapfs {es-pull}78724[#78724] (issue: {es-issue}78473[#78473]) -* Support cosine similarity in kNN search {es-pull}79500[#79500] -* Node level can match action {es-pull}78765[#78765] - - - -Security:: -* Add v7 `restCompat` for invalidating API key with the id field {es-pull}78664[#78664] (issue: {es-issue}66671[#66671]) -* Print enrollment token on startup {es-pull}78293[#78293] -* Startup check for security implicit behavior change {es-pull}76879[#76879] -* Update auto-generated credentials output {es-pull}79755[#79755] (issue: {es-issue}79312[#79312]) -* CLI tool to reconfigure nodes to enroll {es-pull}79690[#79690] (issue: {es-issue}7718[#7718]) -* Security auto-configuration for packaged installations {es-pull}75144[#75144] (issue: {es-issue}78306[#78306]) -* Update to OpenSAML 4 {es-pull}77012[#77012] (issue: {es-issue}71983[#71983]) - -Snapshot/Restore:: -* Allow listing older repositories {es-pull}78244[#78244] -* Optimize SLM Policy Queries {es-pull}79341[#79341] (issue: {es-issue}79321[#79321]) -* Upgrade repository-hdfs plugin to Hadoop 3 {es-pull}76897[#76897] - -Transform:: -* Prevent old beta transforms from starting {es-pull}79712[#79712] - -TSDB:: -* Automatically add timestamp mapper {es-pull}79136[#79136] -* Create a coordinating node level reader for tsdb {es-pull}79197[#79197] -* Fix TSDB shrink test in multi-version cluster {es-pull}79940[#79940] (issue: {es-issue}79936[#79936]) -* Do not allow shadowing metrics or dimensions {es-pull}79757[#79757] - -[[bug-8.0.0-beta1]] -[float] -=== Bug fixes - -Infra/Core:: -* Prevent stack overflow in rounding {es-pull}80450[#80450] - -Infra/Settings:: -* Stricter `UpdateSettingsRequest` parsing on the REST layer {es-pull}79227[#79227] (issue: {es-issue}29268[#29268]) -* Set Auto expand replica on deprecation log data stream {es-pull}79226[#79226] (issue: {es-issue}78991[#78991]) - -Machine Learning:: -* Add timeout parameter for delete trained models API {es-pull}79739[#79739] (issue: {es-issue}77070[#77070]) -* Fix `MlMetadata` backwards compatibility bug with 7.13 through 7.16 {es-pull}80041[#80041] -* Tone down ML unassigned job notifications {es-pull}79578[#79578] (issue: {es-issue}79270[#79270]) -* Use a new annotations index for future annotations {es-pull}79006[#79006] (issue: {es-issue}78439[#78439]) - -Search:: -* Remove unsafe assertion in wildcard field {es-pull}78966[#78966] - -Snapshot/Restore:: -* Don't fill stack traces in `SnapshotShardFailure` {es-pull}80009[#80009] (issue: {es-issue}79718[#79718]) - - - -[[regression-8.0.0-beta1]] -[float] -=== Regressions - -Search:: -* Disable numeric sort optimization conditionally {es-pull}78103[#78103] - - - -[[upgrade-8.0.0-beta1]] -[float] -=== Upgrades - -Search:: -* Update Lucene 9 snapshot {es-pull}79701[#79701] {es-pull}79138[#79138] {es-pull}78548[#78548] {es-pull}78286[#78286] {es-pull}73324[#73324] {es-pull}79461[#79461] - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/reference/release-notes/8.0.0-rc1.asciidoc b/docs/reference/release-notes/8.0.0-rc1.asciidoc deleted file mode 100644 index ced057be013bd..0000000000000 --- a/docs/reference/release-notes/8.0.0-rc1.asciidoc +++ /dev/null @@ -1,112 +0,0 @@ -[[release-notes-8.0.0-rc1]] -== {es} version 8.0.0-rc1 - -Also see <>. - -[[known-issues-8.0.0-rc1]] -[float] -=== Known issues - -* **Do not upgrade production clusters to {es} 8.0.0-rc1.** {es} 8.0.0-rc1 is -a pre-release of {es} 8.0 and is intended for testing purposes only. -+ -Upgrades from pre-release builds are not supported and could result in errors or -data loss. If you upgrade from a released version, such as 7.16, to a -pre-release version for testing, discard the contents of the cluster when you are -done. Do not attempt to upgrade to the final 8.0 release. - -* For {es} 8.0.0-rc1, the {ref}/sql-jdbc.html[{es} SQL JDBC driver] requires -Java 17 or newer. In {es} 8.0.0-rc2, the JDBC driver will only require Java 8 -or newer. {es-pull}82325 - -[[breaking-8.0.0-rc1]] -[float] -=== Breaking changes - -Infra/Core:: -* All system indices are hidden indices {es-pull}79512[#79512] - -Snapshot/Restore:: -* Adjust snapshot index resolution behavior to be more intuitive {es-pull}79670[#79670] (issue: {es-issue}78320[#78320]) - -[[deprecation-8.0.0-rc1]] -[float] -=== Deprecations - -Engine:: -* Deprecate setting `max_merge_at_once_explicit` {es-pull}80574[#80574] - -Machine Learning:: -* Deprecate `estimated_heap_memory_usage_bytes` and replace with `model_size_bytes` {es-pull}80554[#80554] - -Search:: -* Configure `IndexSearcher.maxClauseCount()` based on node characteristics {es-pull}81525[#81525] (issue: {es-issue}46433[#46433]) - -Transform:: -* Improve transform deprecation messages {es-pull}81847[#81847] (issues: {es-issue}81521[#81521], {es-issue}81523[#81523]) - -[[enhancement-8.0.0-rc1]] -[float] -=== Enhancements - -Authorization:: -* Granting `kibana_system` reserved role access to "all" privileges to `.internal.preview.alerts*` index {es-pull}80889[#80889] (issues: {es-issue}76624[#76624], {es-issue}80746[#80746], {es-issue}116374[#116374]) -* Granting `kibana_system` reserved role access to "all" privileges to .preview.alerts* index {es-pull}80746[#80746] -* Granting editor and viewer roles access to alerts-as-data indices {es-pull}81285[#81285] - -ILM+SLM:: -* Make unchanged ILM policy updates into noop {es-pull}82240[#82240] (issue: {es-issue}82065[#82065]) - -Indices APIs:: -* Batch rollover cluster state updates {es-pull}79945[#79945] (issues: {es-issue}77466[#77466], {es-issue}79782[#79782]) -* Reuse `MappingMetadata` instances in Metadata class {es-pull}80348[#80348] (issues: {es-issue}69772[#69772], {es-issue}77466[#77466]) - -Infra/Settings:: -* Implement setting deduplication via string interning {es-pull}80493[#80493] (issues: {es-issue}77466[#77466], {es-issue}78892[#78892]) - -Ingest:: -* Extract more standard metadata from binary files {es-pull}78754[#78754] (issue: {es-issue}22339[#22339]) - -Machine Learning:: -* Add `deployment_stats` to trained model stats {es-pull}80531[#80531] -* The setting `use_auto_machine_memory_percent` now defaults `max_model_memory_limit` {es-pull}80532[#80532] (issue: {es-issue}80415[#80415]) - -Monitoring:: -* Adding default templates for Metricbeat ECS data {es-pull}81744[#81744] - -Network:: -* Improve slow inbound handling to include response type {es-pull}80425[#80425] - -Recovery:: -* Fix `PendingReplicationActions` submitting lots of `NOOP` tasks to `GENERIC` {es-pull}82092[#82092] (issues: {es-issue}77466[#77466], {es-issue}79837[#79837]) - -Reindex:: -* Do not scroll if max docs is less than scroll size (update/delete by query) {es-pull}81654[#81654] (issue: {es-issue}54270[#54270]) - -Security:: -* URL option for `BaseRunAsSuperuserCommand` {es-pull}81025[#81025] (issue: {es-issue}80481[#80481]) - -[[bug-8.0.0-rc1]] -[float] -=== Bug fixes - -Autoscaling:: -* Autoscaling use adjusted total memory {es-pull}80528[#80528] (issue: {es-issue}78750[#78750]) - -Data streams:: -* Prohibit restoring a data stream alias with a conflicting write data stream {es-pull}81217[#81217] (issue: {es-issue}80976[#80976]) - -ILM+SLM:: -* Less verbose serialization of snapshot failure in SLM metadata {es-pull}80942[#80942] (issue: {es-issue}77466[#77466]) - -Indices APIs:: -* Fix `ComposableIndexTemplate` equals when `composed_of` is null {es-pull}80864[#80864] - -Infra/REST API:: -* Handle exceptions thrown from `RestCompatibleVersionHelper` {es-pull}80253[#80253] (issues: {es-issue}78214[#78214], {es-issue}79060[#79060]) - -Ingest:: -* Adjust default geoip logging to be less verbose {es-pull}81404[#81404] (issue: {es-issue}81356[#81356]) - -Machine Learning:: -* Set model state compatibility version to 8.0.0 {ml-pull}2139[#2139] diff --git a/docs/reference/release-notes/8.0.0-rc2.asciidoc b/docs/reference/release-notes/8.0.0-rc2.asciidoc deleted file mode 100644 index 9d3f93e006847..0000000000000 --- a/docs/reference/release-notes/8.0.0-rc2.asciidoc +++ /dev/null @@ -1,131 +0,0 @@ -[[release-notes-8.0.0-rc2]] -== {es} version 8.0.0-rc2 - -Also see <>. - -[[known-issues-8.0.0-rc2]] -[float] -=== Known issues - -* **Do not upgrade production clusters to {es} 8.0.0-rc2.** {es} 8.0.0-rc2 is -a pre-release of {es} 8.0 and is intended for testing purposes only. -+ -Upgrades from pre-release builds are not supported and could result in errors or -data loss. If you upgrade from a released version, such as 7.16, to a -pre-release version for testing, discard the contents of the cluster when you are -done. Do not attempt to upgrade to the final 8.0 release. - -* If you installed {es} from an archive on an aarch64 platform like Linux ARM or macOS M1, the -`elastic` user password and {kib} enrollment token are not generated -automatically when starting your node for the first time. -+ --- -After the node starts, generate the `elastic` password with the -<> tool: - -[source,bash] ----- -bin/elasticsearch-reset-password -u elastic ----- - -Then, create an enrollment token for {kib} with the -<> tool: - -[source,bash] ----- -bin/elasticsearch-create-enrollment-token -s kibana ----- --- -[[deprecation-8.0.0-rc2]] -[float] -=== Deprecations - -Engine:: -* Deprecate setting `max_merge_at_once_explicit` {es-pull}80574[#80574] - -Search:: -* Configure `IndexSearcher.maxClauseCount()` based on node characteristics {es-pull}81525[#81525] (issue: {es-issue}46433[#46433]) - - -[[feature-8.0.0-rc2]] -[float] -=== New features - -Snapshot/Restore:: -* Support IAM roles for Kubernetes service accounts {es-pull}81255[#81255] (issue: {es-issue}52625[#52625]) - -Watcher:: -* Use `startsWith` rather than exact matches for Watcher history template names {es-pull}82396[#82396] - - -[[enhancement-8.0.0-rc2]] -[float] -=== Enhancements - -Cluster Coordination:: -* Make `TaskBatcher` less lock-heavy {es-pull}82227[#82227] (issue: {es-issue}77466[#77466]) - -ILM+SLM:: -* Avoid unnecessary `LifecycleExecutionState` recalculation {es-pull}81558[#81558] (issues: {es-issue}77466[#77466], {es-issue}79692[#79692]) -* Make unchanged ILM policy updates into no-op {es-pull}82240[#82240] (issue: {es-issue}82065[#82065]) - -Infra/Core:: -* Prevent upgrades to 8.0 without first upgrading to the last 7.x release {es-pull}82321[#82321] (issue: {es-issue}81865[#81865]) - -Machine Learning:: -* Add `deployment_stats` to trained model stats {es-pull}80531[#80531] -* The setting `use_auto_machine_memory_percent` now defaults to `max_model_memory_limit` {es-pull}80532[#80532] (issue: {es-issue}80415[#80415]) - -Network:: -* Improve slow inbound handling to include response type {es-pull}80425[#80425] - -Packaging:: -* Convert repository plugins to modules {es-pull}81870[#81870] (issue: {es-issue}81652[#81652]) - -Search:: -* Check nested fields earlier in kNN search {es-pull}80516[#80516] (issue: {es-issue}78473[#78473]) - - -[[bug-8.0.0-rc2]] -[float] -=== Bug fixes - -Autoscaling:: -* Use adjusted total memory instead of total memory {es-pull}80528[#80528] (issue: {es-issue}78750[#78750]) - -Infra/Scripting:: -* Fix duplicated allow lists upon script engine creation {es-pull}82820[#82820] (issue: {es-issue}82778[#82778]) - -Ingest:: -* Adjust default geoip logging to be less verbose {es-pull}81404[#81404] (issue: {es-issue}81356[#81356]) - -Machine Learning:: -* Check that `total_definition_length` is consistent before starting a deployment {es-pull}80553[#80553] -* Fail inference processor more consistently on certain error types {es-pull}81475[#81475] -* Optimize the job stats call to do fewer searches {es-pull}82362[#82362] (issue: {es-issue}82255[#82255]) - -Recovery:: -* Make shard started response handling only return after the cluster state update completes {es-pull}82790[#82790] (issue: {es-issue}81628[#81628]) - -Search:: -* Reject zero-length vectors when using cosine similarity {es-pull}82241[#82241] (issue: {es-issue}81167[#81167]) - -Security:: -* Auto-generated TLS files under fixed config path {es-pull}81547[#81547] (issue: {es-issue}81057[#81057]) -* Bind to non-localhost for transport in some cases {es-pull}82973[#82973] -* Correct file ownership on node reconfiguration {es-pull}82789[#82789] (issue: {es-issue}80990[#80990]) -* Display security auto-configuration with fancy unicode {es-pull}82740[#82740] (issue: {es-issue}82364[#82364]) - -Snapshot/Restore:: -* Remove custom metadata if there is nothing to restore {es-pull}81373[#81373] (issues: {es-issue}81247[#81247], {es-issue}82019[#82019]) - - -[[upgrade-8.0.0-rc2]] -[float] -=== Upgrades - -Infra/Logging:: -* Upgrade ECS logging layout to latest version {es-pull}80500[#80500] - -Search:: -* Upgrade to released lucene 9.0.0 {es-pull}81426[#81426] diff --git a/docs/reference/release-notes/8.0.0.asciidoc b/docs/reference/release-notes/8.0.0.asciidoc deleted file mode 100644 index 835b7b036f954..0000000000000 --- a/docs/reference/release-notes/8.0.0.asciidoc +++ /dev/null @@ -1,717 +0,0 @@ -[[release-notes-8.0.0]] -== {es} version 8.0.0 - -The following list are changes in 8.0.0 as compared to 7.17.0, and combines -release notes from the 8.0.0-alpha1, -alpha2, -beta1, -rc1 and -rc2 releases. - -Also see <>. - -[[known-issues-8.0.0]] -[float] -=== Known issues - -* If you installed {es} from an archive on an aarch64 platform like Linux ARM or macOS M1, the -`elastic` user password and {kib} enrollment token are not generated -automatically when starting your node for the first time. -+ --- -After the node starts, generate the `elastic` password with the -<> tool: - -[source,bash] ----- -bin/elasticsearch-reset-password -u elastic ----- - -Then, create an enrollment token for {kib} with the -<> tool: - -[source,bash] ----- -bin/elasticsearch-create-enrollment-token -s kibana ----- --- -// tag::jackson-filtering-bug[] -* Parsing a request when the last element in an array is filtered out -(for instance using `_source_includes`) fails. This -is due to https://github.com/FasterXML/jackson-core/issues/882[a bug in Jackson parser]. -Fixed in {es} 8.6.1 ({es-pull}92480[#91456]) -// end::jackson-filtering-bug[] - - -[[breaking-8.0.0]] -[float] -=== Breaking changes - -Aggregations:: -* Percentiles aggregation: disallow specifying same percentile values twice {es-pull}52257[#52257] (issue: {es-issue}51871[#51871]) -* Remove adjacency matrix setting {es-pull}46327[#46327] (issues: {es-issue}46257[#46257], {es-issue}46324[#46324]) -* Remove `MovingAverage` pipeline aggregation {es-pull}39328[#39328] -* Remove deprecated `_time` and `_term` sort orders {es-pull}39450[#39450] -* Remove deprecated date histo interval {es-pull}75000[#75000] - -Allocation:: -* Require single data nodes to respect disk watermarks {es-pull}73737[#73737] (issues: {es-issue}55805[#55805], {es-issue}73733[#73733]) -* Remove `include_relocations` setting {es-pull}47717[#47717] (issues: {es-issue}46079[#46079], {es-issue}47443[#47443]) - -Analysis:: -* Cleanup versioned deprecations in analysis {es-pull}41560[#41560] (issue: {es-issue}41164[#41164]) -* Remove preconfigured `delimited_payload_filter` {es-pull}43686[#43686] (issues: {es-issue}41560[#41560], {es-issue}43684[#43684]) - -Authentication:: -* Always add file and native realms unless explicitly disabled {es-pull}69096[#69096] (issue: {es-issue}50892[#50892]) -* Do not set a NameID format in Policy by default {es-pull}44090[#44090] (issue: {es-issue}40353[#40353]) -* Make order setting mandatory for Realm config {es-pull}51195[#51195] (issue: {es-issue}37614[#37614]) - -CCR:: -* Avoid auto following leader system indices in CCR {es-pull}72815[#72815] (issue: {es-issue}67686[#67686]) - -Cluster Coordination:: -* Remove join timeout {es-pull}60873[#60873] (issue: {es-issue}60872[#60872]) -* Remove node filters for voting config exclusions {es-pull}55673[#55673] (issues: {es-issue}47990[#47990], {es-issue}50836[#50836]) -* Remove support for delaying state recovery pending master {es-pull}53845[#53845] (issue: {es-issue}51806[#51806]) - -Distributed:: -* Remove synced flush {es-pull}50882[#50882] (issues: {es-issue}50776[#50776], {es-issue}50835[#50835]) -* Remove the `cluster.remote.connect` setting {es-pull}54175[#54175] (issue: {es-issue}53924[#53924]) - -Engine:: -* Force merge should reject requests with `only_expunge_deletes` and `max_num_segments` set {es-pull}44761[#44761] (issue: {es-issue}43102[#43102]) -* Remove per-type indexing stats {es-pull}47203[#47203] (issue: {es-issue}41059[#41059]) -* Remove translog retention settings {es-pull}51697[#51697] (issue: {es-issue}50775[#50775]) - -Features/CAT APIs:: -* Remove the deprecated `local` parameter for `_cat/indices` {es-pull}64868[#64868] (issue: {es-issue}62198[#62198]) -* Remove the deprecated `local` parameter for `_cat/shards` {es-pull}64867[#64867] (issue: {es-issue}62197[#62197]) - -Features/Features:: -* Remove deprecated `._tier` allocation filtering settings {es-pull}73074[#73074] (issue: {es-issue}72835[#72835]) - -Features/ILM+SLM:: -* Add lower bound on `poll_interval` {es-pull}39593[#39593] (issue: {es-issue}39163[#39163]) -* Make the ILM `freeze` action a no-op {es-pull}77158[#77158] (issue: {es-issue}70192[#70192]) -* Always enforce default tier preference {es-pull}79751[#79751] (issue: {es-issue}76147[#76147]) -* Validate that snapshot repository exists for ILM policies at creation/update time {es-pull}78468[#78468] (issues: {es-issue}72957[#72957], {es-issue}77657[#77657]) -* Default `cluster.routing.allocation.enforce_default_tier_preference` to `true` {es-pull}79275[#79275] (issues: {es-issue}76147[#76147], {es-issue}79210[#79210]) - -Features/Indices APIs:: -* Change `prefer_v2_templates` parameter to default to true {es-pull}55489[#55489] (issues: {es-issue}53101[#53101], {es-issue}55411[#55411]) -* Remove deprecated `_upgrade` API {es-pull}64732[#64732] (issue: {es-issue}21337[#21337]) -* Remove local parameter for get field mapping request {es-pull}55100[#55100] (issue: {es-issue}55099[#55099]) -* Remove `include_type_name` parameter from REST layer {es-pull}48632[#48632] (issue: {es-issue}41059[#41059]) -* Remove the `template` field in index templates {es-pull}49460[#49460] (issue: {es-issue}21009[#21009]) -* Remove endpoint for freezing indices {es-pull}78918[#78918] (issues: {es-issue}70192[#70192], {es-issue}77273[#77273]) - -Features/Watcher:: -* Move watcher history to data stream {es-pull}64252[#64252] - -Geo:: -* Disallow creating `geo_shape` mappings with deprecated parameters {es-pull}70850[#70850] (issue: {es-issue}32039[#32039]) -* Remove bounding box query `type` parameter {es-pull}74536[#74536] - -Infra/Circuit Breakers:: -* Fixed synchronizing inflight breaker with internal variable {es-pull}40878[#40878] - -Infra/Core:: -* Limit processors by available processors {es-pull}44894[#44894] (issue: {es-issue}44889[#44889]) -* Remove `nodes/0` folder prefix from data path {es-pull}42489[#42489] -* Remove `bootstrap.system_call_filter` setting {es-pull}72848[#72848] -* Remove `fixed_auto_queue_size` threadpool type {es-pull}52280[#52280] -* Remove `node.max_local_storage_nodes` {es-pull}42428[#42428] (issue: {es-issue}42426[#42426]) -* Remove camel case named date/time formats {es-pull}60044[#60044] -* Remove legacy role settings {es-pull}71163[#71163] (issues: {es-issue}54998[#54998], {es-issue}66409[#66409], {es-issue}71143[#71143]) -* Remove `processors` setting {es-pull}45905[#45905] (issue: {es-issue}45855[#45855]) -* Remove the `local` parameter of `/_cat/nodes` {es-pull}50594[#50594] (issues: {es-issue}50088[#50088], {es-issue}50499[#50499]) -* Remove the listener thread pool {es-pull}53314[#53314] (issue: {es-issue}53049[#53049]) -* Remove the node local storage setting {es-pull}54381[#54381] (issue: {es-issue}54374[#54374]) -* Remove the `pidfile` setting {es-pull}45940[#45940] (issue: {es-issue}45938[#45938]) -* Removes `week_year` date format {es-pull}63384[#63384] (issue: {es-issue}60707[#60707]) -* System indices treated as restricted indices {es-pull}74212[#74212] (issue: {es-issue}69298[#69298]) -* Remove Joda dependency {es-pull}79007[#79007] -* Remove Joda support from date formatters {es-pull}78990[#78990] -* All system indices are hidden indices {es-pull}79512[#79512] - -Infra/Logging:: -* Remove slowlog level {es-pull}57591[#57591] (issue: {es-issue}56171[#56171]) - -Infra/Plugins:: -* Remove deprecated basic license feature enablement settings {es-pull}56211[#56211] (issue: {es-issue}54745[#54745]) - -Infra/REST API:: -* Remove content type required setting {es-pull}61043[#61043] -* Remove deprecated endpoints containing `_xpack` {es-pull}48170[#48170] (issue: {es-issue}35958[#35958]) -* Remove deprecated endpoints of hot threads API {es-pull}55109[#55109] (issue: {es-issue}52640[#52640]) -* Allow parsing Content-Type and Accept headers with version {es-pull}61427[#61427] - -Infra/Resiliency:: -* Fail node containing ancient closed index {es-pull}44264[#44264] (issues: {es-issue}21830[#21830], {es-issue}41731[#41731], {es-issue}44230[#44230]) - -Infra/Scripting:: -* Consolidate script parsing from object {es-pull}59507[#59507] (issue: {es-issue}59391[#59391]) -* Move `script_cache` into _nodes/stats {es-pull}59265[#59265] (issues: {es-issue}50152[#50152], {es-issue}59262[#59262]) -* Remove general cache settings {es-pull}59262[#59262] (issue: {es-issue}50152[#50152]) - -Infra/Settings:: -* Change default value of `action.destructive_requires_name` to `true` {es-pull}66908[#66908] (issue: {es-issue}61074[#61074]) -* Forbid settings without a namespace {es-pull}45947[#45947] (issues: {es-issue}45905[#45905], {es-issue}45940[#45940]) - -Ingest:: -* Remove default maxmind GeoIP databases from distribution {es-pull}78362[#78362] (issue: {es-issue}68920[#68920]) - -License:: -* Set `xpack.security.enabled` to true for all licenses {es-pull}72300[#72300] -* Enforce license expiration {es-pull}79671[#79671] - -Machine Learning:: -* Remove deprecated `_xpack` endpoints {es-pull}59870[#59870] (issues: {es-issue}35958[#35958], {es-issue}48170[#48170]) -* Remove the ability to update datafeed's `job_id` {es-pull}44752[#44752] (issue: {es-issue}44616[#44616]) -* Remove `allow_no_datafeeds` and `allow_no_jobs` parameters from APIs {es-pull}80048[#80048] (issue: {es-issue}60732[#60732]) - -Mapping:: -* Remove `boost` mapping parameter {es-pull}62639[#62639] (issue: {es-issue}62623[#62623]) -* Remove support for chained multi-fields {es-pull}42333[#42333] (issues: {es-issue}41267[#41267], {es-issue}41926[#41926]) -* Remove support for string in `unmapped_type` {es-pull}45675[#45675] -* Removes typed URLs from mapping APIs {es-pull}41676[#41676] - -Network:: -* Remove client feature tracking {es-pull}44929[#44929] (issues: {es-issue}31020[#31020], {es-issue}42538[#42538], {es-issue}44667[#44667]) -* Remove escape hatch permitting incompatible builds {es-pull}65753[#65753] (issues: {es-issue}65249[#65249], {es-issue}65601[#65601]) - -Packaging:: -* Remove SysV init support {es-pull}51716[#51716] (issue: {es-issue}51480[#51480]) -* Remove support for `JAVA_HOME` {es-pull}69149[#69149] (issue: {es-issue}55820[#55820]) -* Remove no-jdk distributions {es-pull}76896[#76896] (issue: {es-issue}65109[#65109]) -* Require Java 17 for running Elasticsearch {es-pull}79873[#79873] - -Recovery:: -* Remove dangling index auto import functionality {es-pull}59698[#59698] (issue: {es-issue}48366[#48366]) - -Reindex:: -* Reindex from Remote encoding {es-pull}41007[#41007] (issue: {es-issue}40303[#40303]) -* Reindex remove outer level size {es-pull}43373[#43373] (issues: {es-issue}24344[#24344], {es-issue}41894[#41894]) - -Rollup:: -* `RollupStart` endpoint should return OK if job already started {es-pull}41502[#41502] (issues: {es-issue}35928[#35928], {es-issue}39845[#39845]) - -Search:: -* Decouple shard allocation awareness from search and get requests {es-pull}45735[#45735] (issue: {es-issue}43453[#43453]) -* Fix range query on date fields for number inputs {es-pull}63692[#63692] (issue: {es-issue}63680[#63680]) -* Make fuzziness reject illegal values earlier {es-pull}33511[#33511] -* Make remote cluster resolution stricter {es-pull}40419[#40419] (issue: {es-issue}37863[#37863]) -* Parse empty first line in msearch request body as action metadata {es-pull}41011[#41011] (issue: {es-issue}39841[#39841]) -* Remove `CommonTermsQuery` and `cutoff_frequency` param {es-pull}42654[#42654] (issue: {es-issue}37096[#37096]) -* Remove `type` query {es-pull}47207[#47207] (issue: {es-issue}41059[#41059]) -* Remove `use_field_mapping` format option for docvalue fields {es-pull}55622[#55622] -* Remove deprecated `SimpleQueryStringBuilder` parameters {es-pull}57200[#57200] -* Remove deprecated `search.remote` settings {es-pull}42381[#42381] (issues: {es-issue}33413[#33413], {es-issue}38556[#38556]) -* Remove deprecated sort options: `nested_path` and `nested_filter` {es-pull}42809[#42809] (issue: {es-issue}27098[#27098]) -* Remove deprecated vector functions {es-pull}48725[#48725] (issue: {es-issue}48604[#48604]) -* Remove support for `_type` in searches {es-pull}68564[#68564] (issues: {es-issue}41059[#41059], {es-issue}68311[#68311]) -* Remove support for sparse vectors {es-pull}48781[#48781] (issue: {es-issue}48368[#48368]) -* Remove the object format for `indices_boost` {es-pull}55078[#55078] -* Removes type from `TermVectors` APIs {es-pull}42198[#42198] (issue: {es-issue}41059[#41059]) -* Removes typed endpoint from search and related APIs {es-pull}41640[#41640] -* Set max allowed size for stored async response {es-pull}74455[#74455] (issue: {es-issue}67594[#67594]) -* `indices.query.bool.max_clause_count` now limits all query clauses {es-pull}75297[#75297] - -Security:: -* Remove obsolete security settings {es-pull}40496[#40496] -* Remove support of creating CA on the fly when generating certificates {es-pull}65590[#65590] (issue: {es-issue}61884[#61884]) -* Remove the `id` field from the `InvalidateApiKey` API {es-pull}66671[#66671] (issue: {es-issue}66317[#66317]) -* Remove the migrate tool {es-pull}42174[#42174] -* Compress audit logs {es-pull}64472[#64472] (issue: {es-issue}63843[#63843]) -* Remove insecure settings {es-pull}46147[#46147] (issue: {es-issue}45947[#45947]) -* Remove `kibana_dashboard_only_user` reserved role {es-pull}76507[#76507] - -Snapshot/Restore:: -* Blob store compress default to `true` {es-pull}40033[#40033] -* Get snapshots support for multiple repositories {es-pull}42090[#42090] (issue: {es-issue}41210[#41210]) -* Remove repository stats API {es-pull}62309[#62309] (issue: {es-issue}62297[#62297]) -* Remove frozen cache setting leniency {es-pull}71013[#71013] (issue: {es-issue}70341[#70341]) -* Adjust snapshot index resolution behavior to be more intuitive {es-pull}79670[#79670] (issue: {es-issue}78320[#78320]) - -TLS:: -* Reject misconfigured/ambiguous SSL server config {es-pull}45892[#45892] -* Remove support for configurable PKCS#11 keystores {es-pull}75404[#75404] -* Remove the client transport profile filter {es-pull}43236[#43236] - - - -[[breaking-java-8.0.0]] -[float] -=== Breaking Java changes - -Authentication:: -* Mandate x-pack REST handler installed {es-pull}71061[#71061] (issue: {es-issue}70523[#70523]) - -CCR:: -* Remove the `CcrClient` {es-pull}42816[#42816] - -CRUD:: -* Remove types from `BulkRequest` {es-pull}46983[#46983] (issue: {es-issue}41059[#41059]) -* Remove `Client.prepareIndex(index, type, id)` method {es-pull}48443[#48443] - - -Client:: -* Remove `SecurityClient` from x-pack {es-pull}42471[#42471] - -Features/ILM+SLM:: -* Remove the `ILMClient` {es-pull}42817[#42817] - -Features/Monitoring:: -* Remove `MonitoringClient` from x-pack {es-pull}42770[#42770] - -Features/Watcher:: -* Remove `WatcherClient` from x-pack {es-pull}42815[#42815] - -Infra/Core:: -* Remove `XPackClient` from x-pack {es-pull}42729[#42729] -* Remove the transport client {es-pull}42538[#42538] -* Remove transport client from x-pack {es-pull}42202[#42202] - -Infra/REST API:: -* Copy HTTP headers to `ThreadContext` strictly {es-pull}45945[#45945] - -Machine Learning:: -* Remove the `MachineLearningClient` {es-pull}43108[#43108] - -Mapping:: -* Remove type filter from `GetMappings` API {es-pull}47364[#47364] (issue: {es-issue}41059[#41059]) -* Remove `type` parameter from `PutMappingRequest.buildFromSimplifiedDef()` {es-pull}50844[#50844] (issue: {es-issue}41059[#41059]) -* Remove unused parameter from `MetadataFieldMapper.TypeParser#getDefault()` {es-pull}51219[#51219] -* Remove `type` parameter from `CIR.mapping(type, object...)` {es-pull}50739[#50739] (issue: {es-issue}41059[#41059]) - -Search:: -* Removes types from `SearchRequest` and `QueryShardContext` {es-pull}42112[#42112] - -Snapshot/Restore:: -* Remove deprecated repository methods {es-pull}42359[#42359] (issue: {es-issue}42213[#42213]) - - -[[deprecation-8.0.0]] -[float] -=== Deprecations - -Authentication:: -* Deprecate setup-passwords tool {es-pull}76902[#76902] - -CRUD:: -* Remove `indices_segments` 'verbose' parameter {es-pull}78451[#78451] (issue: {es-issue}75955[#75955]) - -Engine:: -* Deprecate setting `max_merge_at_once_explicit` {es-pull}80574[#80574] - -Machine Learning:: -* Deprecate `estimated_heap_memory_usage_bytes` and replace with `model_size_bytes` {es-pull}80554[#80554] - -Monitoring:: -* Add deprecation info API entries for deprecated monitoring settings {es-pull}78799[#78799] -* Automatically install monitoring templates at plugin initialization {es-pull}78350[#78350] -* Remove Monitoring ingest pipelines {es-pull}77459[#77459] (issue: {es-issue}50770[#50770]) - -Search:: -* Configure `IndexSearcher.maxClauseCount()` based on node characteristics {es-pull}81525[#81525] (issue: {es-issue}46433[#46433]) - -Transform:: -* Improve transform deprecation messages {es-pull}81847[#81847] (issues: {es-issue}81521[#81521], {es-issue}81523[#81523]) - -[[feature-8.0.0]] -[float] -=== New features - -Security:: -* Auto-configure TLS for new nodes of new clusters {es-pull}77231[#77231] (issues: {es-issue}75144[#75144], {es-issue}75704[#75704]) - -Snapshot/Restore:: -* Support IAM roles for Kubernetes service accounts {es-pull}81255[#81255] (issue: {es-issue}52625[#52625]) - -Watcher:: -* Use `startsWith` rather than exact matches for Watcher history template names {es-pull}82396[#82396] - - -[[enhancement-8.0.0]] -[float] -=== Enhancements - -Analysis:: -* Move `reload_analyzers` endpoint to x-pack {es-pull}43559[#43559] - -Authentication:: -* Reset elastic password CLI tool {es-pull}74892[#74892] (issues: {es-issue}70113[#70113], {es-issue}74890[#74890]) -* Autogenerate and print elastic password on startup {es-pull}77291[#77291] -* Enroll Kibana API uses Service Accounts {es-pull}76370[#76370] -* Add `reset-kibana-system-user` tool {es-pull}77322[#77322] -* New CLI tool to reset password for built-in users {es-pull}79709[#79709] -* Auto-configure the `elastic` user password {es-pull}78306[#78306] - -Authorization:: -* Granting `kibana_system` reserved role access to "all" privileges to `.internal.preview.alerts*` index {es-pull}80889[#80889] (issues: {es-issue}76624[#76624], {es-issue}80746[#80746], {es-issue}116374[#116374]) -* Granting `kibana_system` reserved role access to "all" privileges to .preview.alerts* index {es-pull}80746[#80746] -* Granting editor and viewer roles access to alerts-as-data indices {es-pull}81285[#81285] - -Cluster Coordination:: -* Prevent downgrades from 8.x to 7.x {es-pull}78586[#78586] (issues: {es-issue}42489[#42489], {es-issue}52414[#52414]) -* Prevent downgrades from 8.x to 7.x {es-pull}78638[#78638] (issues: {es-issue}42489[#42489], {es-issue}52414[#52414]) -* Make `TaskBatcher` less lock-heavy {es-pull}82227[#82227] (issue: {es-issue}77466[#77466]) - -Data streams:: -* Data stream support read and write with custom routing and partition size {es-pull}74394[#74394] (issue: {es-issue}74390[#74390]) - -EQL:: -* Add option for returning results from the tail of the stream {es-pull}64869[#64869] (issue: {es-issue}58646[#58646]) -* Introduce case insensitive variant `in~` {es-pull}68176[#68176] (issue: {es-issue}68172[#68172]) -* Optimize redundant `toString` {es-pull}71070[#71070] (issue: {es-issue}70681[#70681]) - -Engine:: -* Always use soft-deletes in `InternalEngine` {es-pull}50415[#50415] -* Remove translog retention policy {es-pull}51417[#51417] (issue: {es-issue}50775[#50775]) - -Features/CAT APIs:: -* Remove `size` and add `time` params to `_cat/threadpool` {es-pull}55736[#55736] (issue: {es-issue}54478[#54478]) - -Features/ILM+SLM:: -* Allow for setting the total shards per node in the Allocate ILM action {es-pull}76794[#76794] (issue: {es-issue}76775[#76775]) -* Inject migrate action regardless of allocate action {es-pull}79090[#79090] (issue: {es-issue}76147[#76147]) -* Make unchanged ILM policy updates into noop {es-pull}82240[#82240] (issue: {es-issue}82065[#82065]) -* Avoid unnecessary `LifecycleExecutionState` recalculation {es-pull}81558[#81558] (issues: {es-issue}77466[#77466], {es-issue}79692[#79692]) - -Features/Indices APIs:: -* Batch rollover cluster state updates {es-pull}79945[#79945] (issues: {es-issue}77466[#77466], {es-issue}79782[#79782]) -* Reuse `MappingMetadata` instances in Metadata class {es-pull}80348[#80348] (issues: {es-issue}69772[#69772], {es-issue}77466[#77466]) - -Features/Stats:: -* Add bulk stats track the bulk per shard {es-pull}52208[#52208] (issues: {es-issue}47345[#47345], {es-issue}50536[#50536]) - -Features/Watcher:: -* Remove Watcher history clean up from monitoring {es-pull}67154[#67154] - -Infra/Core:: -* Remove aliases exist action {es-pull}43430[#43430] -* Remove indices exists action {es-pull}43164[#43164] -* Remove types exists action {es-pull}43344[#43344] -* Retain reference to stdout for exceptional cases {es-pull}77460[#77460] -* Check whether stdout is a real console {es-pull}79882[#79882] -* Share int, long, float, double, and byte pages {es-pull}75053[#75053] -* Revert "Deprecate resolution loss on date field (#78921)" {es-pull}79914[#79914] (issue: {es-issue}78921[#78921]) -* Add two missing entries to the deprecation information API {es-pull}80290[#80290] (issue: {es-issue}80233[#80233]) -* Prevent upgrades to 8.0 without first upgrading to the last 7.x release {es-pull}82321[#82321] (issue: {es-issue}81865[#81865]) - -Infra/Logging:: -* Make Elasticsearch JSON logs ECS compliant {es-pull}47105[#47105] (issue: {es-issue}46119[#46119]) - -Infra/REST API:: -* Allow for field declaration for future compatible versions {es-pull}69774[#69774] (issue: {es-issue}51816[#51816]) -* Introduce stability description to the REST API specification {es-pull}38413[#38413] -* Parsing: Validate that fields are not registered twice {es-pull}70243[#70243] -* Support response content-type with versioned media type {es-pull}65500[#65500] (issue: {es-issue}51816[#51816]) -* [REST API Compatibility] Typed endpoints for index and get APIs {es-pull}69131[#69131] (issue: {es-issue}54160[#54160]) -* [REST API Compatibility] Typed endpoints for put and get mapping and get field mappings {es-pull}71721[#71721] (issues: {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Allow `copy_settings` flag for resize operations {es-pull}75184[#75184] (issues: {es-issue}38514[#38514], {es-issue}51816[#51816]) -* [REST API Compatibility] Allow for type in geo shape query {es-pull}74553[#74553] (issues: {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Always return `adjust_pure_negative` value {es-pull}75182[#75182] (issues: {es-issue}49543[#49543], {es-issue}51816[#51816]) -* [REST API Compatibility] Clean up x-pack/plugin rest compat tests {es-pull}74701[#74701] (issue: {es-issue}51816[#51816]) -* [REST API Compatibility] Do not return `_doc` for empty mappings in template {es-pull}75448[#75448] (issues: {es-issue}51816[#51816], {es-issue}54160[#54160], {es-issue}70966[#70966], {es-issue}74544[#74544]) -* [REST API Compatibility] Dummy REST action for `indices.upgrade` API {es-pull}75136[#75136] (issue: {es-issue}51816[#51816]) -* [REST API Compatibility] REST Terms vector typed response {es-pull}73117[#73117] -* [REST API Compatibility] Rename `BulkItemResponse.Failure` type field {es-pull}74937[#74937] (issue: {es-issue}51816[#51816]) -* [REST API Compatibility] Type metadata for docs used in simulate request {es-pull}74222[#74222] (issues: {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Typed `TermLookups` {es-pull}74544[#74544] (issues: {es-issue}46943[#46943], {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Typed and x-pack graph explore API {es-pull}74185[#74185] (issues: {es-issue}46935[#46935], {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Typed endpoint for bulk API {es-pull}73571[#73571] (issue: {es-issue}51816[#51816]) -* [REST API Compatibility] Typed endpoint for multi-get API {es-pull}73878[#73878] (issue: {es-issue}51816[#51816]) -* [REST API Compatibility] Typed endpoints for `RestUpdateAction` and `RestDeleteAction` {es-pull}73115[#73115] (issues: {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Typed endpoints for `get_source` API {es-pull}73957[#73957] (issues: {es-issue}46587[#46587], {es-issue}46931[#46931], {es-issue}51816[#51816]) -* [REST API Compatibility] Typed endpoints for explain API {es-pull}73901[#73901] (issue: {es-issue}51816[#51816]) -* [REST API Compatibility] Typed endpoints for search `_count` API {es-pull}73958[#73958] (issues: {es-issue}42112[#42112], {es-issue}51816[#51816]) -* [REST API Compatibility] Typed indexing stats {es-pull}74181[#74181] (issues: {es-issue}47203[#47203], {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Types for percolate query API {es-pull}74698[#74698] (issues: {es-issue}46985[#46985], {es-issue}51816[#51816], {es-issue}54160[#54160], {es-issue}74689[#74689]) -* [REST API Compatibility] Validate query typed API {es-pull}74171[#74171] (issues: {es-issue}46927[#46927], {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Voting config exclusion exception message {es-pull}75406[#75406] (issues: {es-issue}51816[#51816], {es-issue}55291[#55291]) -* [REST API Compatibility] `MoreLikeThisQuery` with types {es-pull}75123[#75123] (issues: {es-issue}42198[#42198], {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Update and delete by query using size field {es-pull}69606[#69606] -* [REST API Compatibility] Indicies boost in object format {es-pull}74422[#74422] (issues: {es-issue}51816[#51816], {es-issue}55078[#55078]) -* [REST API Compatibility] Typed endpoints for search and related endpoints {es-pull}72155[#72155] (issues: {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Allow to use size `-1` {es-pull}75342[#75342] (issues: {es-issue}51816[#51816], {es-issue}69548[#69548], {es-issue}70209[#70209]) -* [REST API Compatibility] Ignore `use_field_mapping` option for docvalue {es-pull}74435[#74435] (issue: {es-issue}55622[#55622]) -* [REST API Compatibility] `_time` and `_term` sort orders {es-pull}74919[#74919] (issues: {es-issue}39450[#39450], {es-issue}51816[#51816]) -* [REST API Compatability] `template` parameter and field on PUT index template {es-pull}71238[#71238] (issues: {es-issue}49460[#49460], {es-issue}51816[#51816], {es-issue}68905[#68905]) -* [REST API Compatibility] Make query registration easier {es-pull}75722[#75722] (issue: {es-issue}51816[#51816]) -* [REST API Compatibility] Typed query {es-pull}75453[#75453] (issues: {es-issue}47207[#47207], {es-issue}51816[#51816], {es-issue}54160[#54160]) -* [REST API Compatibility] Deprecate the use of synced flush {es-pull}75372[#75372] (issues: {es-issue}50882[#50882], {es-issue}51816[#51816]) -* [REST API Compatibility] Licence `accept_enterprise` and response changes {es-pull}75479[#75479] (issues: {es-issue}50067[#50067], {es-issue}50735[#50735], {es-issue}51816[#51816], {es-issue}58217[#58217]) - -Infra/Scripting:: -* Update `DeprecationMap` to `DynamicMap` {es-pull}56149[#56149] (issue: {es-issue}52103[#52103]) -* Add nio Buffers to Painless {es-pull}79870[#79870] (issue: {es-issue}79867[#79867]) -* Restore the scripting general cache {es-pull}79453[#79453] (issue: {es-issue}62899[#62899]) - -Infra/Settings:: -* Fixed inconsistent `Setting.exist()` {es-pull}46603[#46603] (issue: {es-issue}41830[#41830]) -* Remove `index.optimize_auto_generated_id` setting (#27583) {es-pull}27600[#27600] (issue: {es-issue}27583[#27583]) -* Implement setting deduplication via string interning {es-pull}80493[#80493] (issues: {es-issue}77466[#77466], {es-issue}78892[#78892]) - -Ingest:: -* Add support for `_meta` field to ingest pipelines {es-pull}76381[#76381] -* Remove binary field after attachment processor execution {es-pull}79172[#79172] -* Improving cache lookup to reduce recomputing / searches {es-pull}77259[#77259] -* Extract more standard metadata from binary files {es-pull}78754[#78754] (issue: {es-issue}22339[#22339]) - -License:: -* Add deprecated `accept_enterprise` param to `/_xpack` {es-pull}58220[#58220] (issue: {es-issue}58217[#58217]) -* Support `accept_enterprise` param in get license API {es-pull}50067[#50067] (issue: {es-issue}49474[#49474]) -* Enforce Transport TLS check on all licenses {es-pull}79602[#79602] (issue: {es-issue}75292[#75292]) - -Machine Learning:: -* The Windows build platform for the {ml} C++ code now uses Visual Studio 2019 {ml-pull}1352[#1352] -* The macOS build platform for the {ml} C++ code is now Mojave running Xcode 11.3.1, - or Ubuntu 20.04 running clang 8 for cross compilation {ml-pull}1429[#1429] -* Add a new application for evaluating PyTorch models. The app depends on LibTorch - the C++ front end to PyTorch - and performs inference on models stored in the TorchScript format {ml-pull}1902[#1902] -* Adding new PUT trained model vocabulary endpoint {es-pull}77387[#77387] -* Creating new PUT model definition part API {es-pull}76987[#76987] -* Add inference time configuration overrides {es-pull}78441[#78441] (issue: {es-issue}77799[#77799]) -* Optimize source extraction for `categorize_text` aggregation {es-pull}79099[#79099] -* The Linux build platform for the {ml} C++ code is now CentOS 7 running gcc 10.3. {ml-pull}2028[#2028] -* Make ML indices hidden when the node becomes master {es-pull}77416[#77416] (issue: {es-issue}53674[#53674]) -* Add `deployment_stats` to trained model stats {es-pull}80531[#80531] -* The setting `use_auto_machine_memory_percent` now defaults `max_model_memory_limit` {es-pull}80532[#80532] (issue: {es-issue}80415[#80415]) - -Mapping:: -* Sparse vector to throw exception consistently {es-pull}62646[#62646] -* Add support for configuring HNSW parameters {es-pull}79193[#79193] (issue: {es-issue}78473[#78473]) -* Extend `dense_vector` to support indexing vectors {es-pull}78491[#78491] (issue: {es-issue}78473[#78473]) - -Monitoring:: -* Add previously removed Monitoring settings back for 8.0 {es-pull}78784[#78784] -* Change Monitoring plugin cluster alerts to not install by default {es-pull}79657[#79657] -* Adding default templates for Metricbeat ECS data {es-pull}81744[#81744] - -Network:: -* Enable LZ4 transport compression by default {es-pull}76326[#76326] (issue: {es-issue}73497[#73497]) -* Improve slow inbound handling to include response type {es-pull}80425[#80425] - -Packaging:: -* Make the Docker build more re-usable in Cloud {es-pull}50277[#50277] (issues: {es-issue}46166[#46166], {es-issue}49926[#49926]) -* Update docker-compose.yml to fix bootstrap check error {es-pull}47650[#47650] -* Allow total memory to be overridden {es-pull}78750[#78750] (issue: {es-issue}65905[#65905]) -* Convert repository plugins to modules {es-pull}81870[#81870] (issue: {es-issue}81652[#81652]) - -Recovery:: -* Use Lucene index in peer recovery and resync {es-pull}51189[#51189] (issue: {es-issue}50775[#50775]) -* Fix `PendingReplicationActions` submitting lots of `NOOP` tasks to `GENERIC` {es-pull}82092[#82092] (issues: {es-issue}77466[#77466], {es-issue}79837[#79837]) - -Reindex:: -* Make reindexing managed by a persistent task {es-pull}43382[#43382] (issue: {es-issue}42612[#42612]) -* Reindex restart from checkpoint {es-pull}46055[#46055] (issue: {es-issue}42612[#42612]) -* Reindex search resiliency {es-pull}45497[#45497] (issues: {es-issue}42612[#42612], {es-issue}43187[#43187]) -* Reindex v2 rethrottle sliced fix {es-pull}46967[#46967] (issues: {es-issue}42612[#42612], {es-issue}46763[#46763]) -* Do not scroll if max docs is less than scroll size (update/delete by query) {es-pull}81654[#81654] (issue: {es-issue}54270[#54270]) - -Rollup:: -* Adds support for `date_nanos` in Rollup Metric and `DateHistogram` Configs {es-pull}59349[#59349] (issue: {es-issue}44505[#44505]) - -SQL:: -* Add text formatting support for multivalue {es-pull}68606[#68606] -* Add xDBC and CLI support. QA CSV specs {es-pull}68966[#68966] -* Export array values through result sets {es-pull}69512[#69512] -* Improve alias resolution in sub-queries {es-pull}67216[#67216] (issue: {es-issue}56713[#56713]) -* Improve the optimization of null conditionals {es-pull}71192[#71192] -* Push `WHERE` clause inside subqueries {es-pull}71362[#71362] -* Use Java `String` methods for `LTRIM/RTRIM` {es-pull}57594[#57594] -* QL: Make canonical form take into account children {es-pull}71266[#71266] -* QL: Polish optimizer expression rule declaration {es-pull}71396[#71396] -* QL: Propagate nullability constraints across conjunctions {es-pull}71187[#71187] (issue: {es-issue}70683[#70683]) - -Search:: -* Completely disallow setting negative size in search {es-pull}70209[#70209] (issue: {es-issue}69548[#69548]) -* Make `0` as invalid value for `min_children` in `has_child` query {es-pull}41347[#41347] -* Return error when remote indices are locally resolved {es-pull}74556[#74556] (issue: {es-issue}26247[#26247]) -* [REST API Compatibility] Nested path and filter sort options {es-pull}76022[#76022] (issues: {es-issue}42809[#42809], {es-issue}51816[#51816]) -* [REST API Compatibility] `CommonTermsQuery` and `cutoff_frequency` parameter {es-pull}75896[#75896] (issues: {es-issue}42654[#42654], {es-issue}51816[#51816]) -* [REST API Compatibility] Allow first empty line for `_msearch` {es-pull}75886[#75886] (issues: {es-issue}41011[#41011], {es-issue}51816[#51816]) -* Node level can match action {es-pull}78765[#78765] -* TSDB: Add time series information to field caps {es-pull}78790[#78790] (issue: {es-issue}74660[#74660]) -* Add new kNN search endpoint {es-pull}79013[#79013] (issue: {es-issue}78473[#78473]) -* Disallow kNN searches on nested vector fields {es-pull}79403[#79403] (issue: {es-issue}78473[#78473]) -* Ensure kNN search respects authorization {es-pull}79693[#79693] (issue: {es-issue}78473[#78473]) -* Load kNN vectors format with mmapfs {es-pull}78724[#78724] (issue: {es-issue}78473[#78473]) -* Support cosine similarity in kNN search {es-pull}79500[#79500] -* Node level can match action {es-pull}78765[#78765] -* Check nested fields earlier in kNN search {es-pull}80516[#80516] (issue: {es-issue}78473[#78473]) - -Security:: -* Add a tool for creating enrollment tokens {es-pull}74890[#74890] -* Add the Enroll Kibana API {es-pull}72207[#72207] -* Change default hashing algorithm for FIPS 140 {es-pull}55544[#55544] -* Create enrollment token {es-pull}73573[#73573] (issues: {es-issue}71438[#71438], {es-issue}72129[#72129]) -* Enroll node API {es-pull}72129[#72129] -* Configure security for the initial node CLI {es-pull}74868[#74868] -* Generate and store password hash for elastic user {es-pull}76276[#76276] (issue: {es-issue}75310[#75310]) -* Set elastic password and generate enrollment token {es-pull}75816[#75816] (issue: {es-issue}75310[#75310]) -* Add `elasticsearch-enroll-node` tool {es-pull}77292[#77292] -* Default hasher to `PBKDF2_STRETCH` on FIPS mode {es-pull}76274[#76274] -* Add v7 `restCompat` for invalidating API key with the id field {es-pull}78664[#78664] (issue: {es-issue}66671[#66671]) -* Print enrollment token on startup {es-pull}78293[#78293] -* Startup check for security implicit behavior change {es-pull}76879[#76879] -* CLI tool to reconfigure nodes to enroll {es-pull}79690[#79690] (issue: {es-issue}7718[#7718]) -* Security auto-configuration for packaged installations {es-pull}75144[#75144] (issue: {es-issue}78306[#78306]) - -Snapshot/Restore:: -* Introduce searchable snapshots index setting for cascade deletion of snapshots {es-pull}74977[#74977] -* Unify blob store compress setting {es-pull}39346[#39346] (issue: {es-issue}39073[#39073]) -* Add recovery state tracking for searchable snapshots {es-pull}60505[#60505] -* Allow listing older repositories {es-pull}78244[#78244] -* Optimize SLM Policy Queries {es-pull}79341[#79341] (issue: {es-issue}79321[#79321]) - -TLS:: -* Add `ChaCha20` TLS ciphers on Java 12+ {es-pull}42155[#42155] -* Add support for `KeyStore` filters to `ssl-config` {es-pull}75407[#75407] -* Update TLS ciphers and protocols for JDK 11 {es-pull}41808[#41808] (issues: {es-issue}38646[#38646], {es-issue}41385[#41385]) - -Transform:: -* Prevent old beta transforms from starting {es-pull}79712[#79712] - -TSDB:: -* Automatically add timestamp mapper {es-pull}79136[#79136] -* Create a coordinating node level reader for tsdb {es-pull}79197[#79197] -* Fix TSDB shrink test in multi-version cluster {es-pull}79940[#79940] (issue: {es-issue}79936[#79936]) -* Do not allow shadowing metrics or dimensions {es-pull}79757[#79757] - - -[[bug-8.0.0]] -[float] -=== Bug fixes - -Aggregations:: -* Fix BWC issues for `x_pack/usage` {es-pull}55181[#55181] (issue: {es-issue}54847[#54847]) -* Fix `DoubleBounds` null serialization {es-pull}59475[#59475] -* Fix `TopHitsAggregationBuilder` adding duplicate `_score` sort clauses {es-pull}42179[#42179] (issue: {es-issue}42154[#42154]) -* Fix `t_test` usage stats {es-pull}54753[#54753] (issue: {es-issue}54744[#54744]) -* Throw exception if legacy interval cannot be parsed in `DateIntervalWrapper` {es-pull}41972[#41972] (issue: {es-issue}41970[#41970]) - -Autoscaling:: -* Autoscaling use adjusted total memory {es-pull}80528[#80528] (issue: {es-issue}78750[#78750]) - -CCR:: -* Fix `AutoFollow` version checks {es-pull}73776[#73776] (issue: {es-issue}72935[#72935]) - -Cluster Coordination:: -* Apply cluster states in system context {es-pull}53785[#53785] (issue: {es-issue}53751[#53751]) - -Data streams:: -* Prohibit restoring a data stream alias with a conflicting write data stream {es-pull}81217[#81217] (issue: {es-issue}80976[#80976]) - -Distributed:: -* Introduce `?wait_for_active_shards=index-setting` {es-pull}67158[#67158] (issue: {es-issue}66419[#66419]) -* Fixes to task result index mapping {es-pull}50359[#50359] (issue: {es-issue}50248[#50248]) - -Features/CAT APIs:: -* Fix cat recovery display of bytes fields {es-pull}40379[#40379] (issue: {es-issue}40335[#40335]) - -Features/ILM+SLM:: -* Ensuring that the `ShrinkAction` does not hang if total shards per node is too low {es-pull}76732[#76732] (issue: {es-issue}44070[#44070]) -* Less verbose serialization of snapshot failure in SLM metadata {es-pull}80942[#80942] (issue: {es-issue}77466[#77466]) - -Features/Indices APIs:: -* Fix `ComposableIndexTemplate` equals when `composed_of` is null {es-pull}80864[#80864] - -Features/Java High Level REST Client:: -* The Java High Level Rest Client (HLRC) has been removed and replaced by a new -{es} Java client. For migration steps, refer to -{java-api-client}/migrate-hlrc.html[Migrate from the High Level Rest Client]. - -Geo:: -* Preprocess polygon rings before processing it for decomposition {es-pull}59501[#59501] (issues: {es-issue}54441[#54441], {es-issue}59386[#59386]) - -Infra/Core:: -* Add searchable snapshot cache folder to `NodeEnvironment` {es-pull}66297[#66297] (issue: {es-issue}65725[#65725]) -* CLI tools: Write errors to stderr instead of stdout {es-pull}45586[#45586] (issue: {es-issue}43260[#43260]) -* Precompute `ParsedMediaType` for XContentType {es-pull}67409[#67409] -* Prevent stack overflow in rounding {es-pull}80450[#80450] - -Infra/Logging:: -* Fix NPE when logging null values in JSON {es-pull}53715[#53715] (issue: {es-issue}46702[#46702]) -* Fix stats in slow logs to be a escaped JSON {es-pull}44642[#44642] -* Populate data stream fields when `xOpaqueId` not provided {es-pull}62156[#62156] - -Infra/REST API:: -* Do not allow spaces within `MediaType's` parameters {es-pull}64650[#64650] (issue: {es-issue}51816[#51816]) -* Handle incorrect header values {es-pull}64708[#64708] (issues: {es-issue}51816[#51816], {es-issue}64689[#64689]) -* Ignore media ranges when parsing {es-pull}64721[#64721] (issues: {es-issue}51816[#51816], {es-issue}64689[#64689]) -* `RestController` should not consume request content {es-pull}44902[#44902] (issue: {es-issue}37504[#37504]) -* Handle exceptions thrown from `RestCompatibleVersionHelper` {es-pull}80253[#80253] (issues: {es-issue}78214[#78214], {es-issue}79060[#79060]) - -Infra/Scripting:: -* Change compound assignment structure to support string concatenation {es-pull}61825[#61825] -* Fixes casting in constant folding {es-pull}61508[#61508] -* Several minor Painless fixes {es-pull}61594[#61594] -* Fix duplicated allow lists upon script engine creation {es-pull}82820[#82820] (issue: {es-issue}82778[#82778]) - -Infra/Settings:: -* Stricter `UpdateSettingsRequest` parsing on the REST layer {es-pull}79227[#79227] (issue: {es-issue}29268[#29268]) -* Set Auto expand replica on deprecation log data stream {es-pull}79226[#79226] (issue: {es-issue}78991[#78991]) - -Ingest:: -* Adjust default geoip logging to be less verbose {es-pull}81404[#81404] (issue: {es-issue}81356[#81356]) - -Machine Learning:: -* Add timeout parameter for delete trained models API {es-pull}79739[#79739] (issue: {es-issue}77070[#77070]) -* Tone down ML unassigned job notifications {es-pull}79578[#79578] (issue: {es-issue}79270[#79270]) -* Use a new annotations index for future annotations {es-pull}79006[#79006] (issue: {es-issue}78439[#78439]) -* Set model state compatibility version to 8.0.0 {ml-pull}2139[#2139] -* Check that `total_definition_length` is consistent before starting a deployment {es-pull}80553[#80553] -* Fail inference processor more consistently on certain error types {es-pull}81475[#81475] -* Optimize the job stats call to do fewer searches {es-pull}82362[#82362] (issue: {es-issue}82255[#82255]) - -Mapping:: -* Remove assertions that mappings have one top-level key {es-pull}58779[#58779] (issue: {es-issue}58521[#58521]) - -Packaging:: -* Suppress illegal access in plugin install {es-pull}41620[#41620] (issue: {es-issue}41478[#41478]) - -Recovery:: -* Make shard started response handling only return after the cluster state update completes {es-pull}82790[#82790] (issue: {es-issue}81628[#81628]) - -SQL:: -* Introduce dedicated node for `HAVING` declaration {es-pull}71279[#71279] (issue: {es-issue}69758[#69758]) -* Make `RestSqlQueryAction` thread-safe {es-pull}69901[#69901] - -Search:: -* Check for negative `from` values in search request body {es-pull}54953[#54953] (issue: {es-issue}54897[#54897]) -* Fix `VectorsFeatureSetUsage` serialization in BWC mode {es-pull}55399[#55399] (issue: {es-issue}55378[#55378]) -* Handle total hits equal to `track_total_hits` {es-pull}37907[#37907] (issue: {es-issue}37897[#37897]) -* Improve error msg for CCS request on node without remote cluster role {es-pull}60351[#60351] (issue: {es-issue}59683[#59683]) -* Remove unsafe assertion in wildcard field {es-pull}78966[#78966] - -Security:: -* Allow access to restricted system indices for reserved system roles {es-pull}76845[#76845] - -Snapshot/Restore:: -* Fix `GET /_snapshot/_all/_all` if there are no repos {es-pull}43558[#43558] (issue: {es-issue}43547[#43547]) -* Don't fill stack traces in `SnapshotShardFailure` {es-pull}80009[#80009] (issue: {es-issue}79718[#79718]) -* Remove custom metadata if there is nothing to restore {es-pull}81373[#81373] (issues: {es-issue}81247[#81247], {es-issue}82019[#82019]) - -[[regression-8.0.0]] -[float] -=== Regressions - -Search:: -* Disable numeric sort optimization conditionally {es-pull}78103[#78103] - -[[upgrade-8.0.0]] -[float] -=== Upgrades - -Authentication:: -* Upgrade to UnboundID LDAP SDK v6.0.2 {es-pull}79332[#79332] - -Infra/Logging:: -* Upgrade ECS logging layout to latest version {es-pull}80500[#80500] - -Search:: -* Upgrade to Lucene 9 {es-pull}81426[#81426] - -Security:: -* Update to OpenSAML 4 {es-pull}77012[#77012] (issue: {es-issue}71983[#71983]) - -Snapshot/Restore:: -* Upgrade repository-hdfs plugin to Hadoop 3 {es-pull}76897[#76897] diff --git a/docs/reference/release-notes/8.0.1.asciidoc b/docs/reference/release-notes/8.0.1.asciidoc deleted file mode 100644 index b6eb14c78ef00..0000000000000 --- a/docs/reference/release-notes/8.0.1.asciidoc +++ /dev/null @@ -1,81 +0,0 @@ -[[release-notes-8.0.1]] -== {es} version 8.0.1 - -Also see <>. -[[known-issues-8.0.1]] -[float] -=== Known issues - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] -[[bug-8.0.1]] -[float] -=== Bug fixes - -Aggregations:: -* Fix backward compatibility with 7.17.0 {es-pull}83715[#83715] - -Distributed:: -* Correctly handle large zones with 500 or more instances {es-pull}83785[#83785] (issue: {es-issue}83783[#83783]) - -ILM+SLM:: -* Do not allow negative age in explain lifecycle API response {es-pull}84043[#84043] - -Infra/Core:: -* Copy `trace.id` in threadcontext stash {es-pull}83218[#83218] -* Preserve context in `ResultDeduplicator` {es-pull}84038[#84038] (issue: {es-issue}84036[#84036]) -* Update system index mappings if `_meta` is null {es-pull}83896[#83896] (issue: {es-issue}83890[#83890]) - -Ingest:: -* Fix `GeoIpDownloader` startup during rolling upgrade {es-pull}84000[#84000] -* Short circuit date patterns after first match {es-pull}83764[#83764] - -Machine Learning:: -* Retry anomaly detection job recovery during relocation {es-pull}83456[#83456] - -Packaging:: -* Add `log4j-slf4j-impl` to `repository-azure` {es-pull}83661[#83661] (issue: {es-issue}83652[#83652]) - -Recovery:: -* Add missing `indices.recovery.internal_action_retry_timeout` to list of settings {es-pull}83354[#83354] - -SQL:: -* Fix txt format for empty result sets {es-pull}83376[#83376] - -Search:: -* Avoid eagerly loading `StoredFieldsReader` in fetch phase {es-pull}83693[#83693] (issue: {es-issue}82777[#82777]) -* Returns valid PIT when no index matched {es-pull}83424[#83424] - -Security:: -* Upgrade jANSI dependency to 2.4.0 {es-pull}83566[#83566] - -Snapshot/Restore:: -* Move get snapshots serialization to management pool {es-pull}83215[#83215] -* Preserve context in `snapshotDeletionListeners` {es-pull}84089[#84089] (issue: {es-issue}84036[#84036]) - -Transform:: -* Fix condition on which the transform stops processing buckets {es-pull}82852[#82852] - -Watcher:: -* Tolerate empty types array in Watch definitions {es-pull}83524[#83524] (issue: {es-issue}83235[#83235]) - -[[enhancement-8.0.1]] -[float] -=== Enhancements - -Infra/REST API:: -* Update YAML REST tests to check for product header on all responses {es-pull}83290[#83290] - -Recovery:: -* Adjust `indices.recovery.max_bytes_per_sec` according to external settings {es-pull}82819[#82819] - -[[upgrade-8.0.1]] -[float] -=== Upgrades - -Geo:: -* Update vector tiles google protobuf to 3.16.1 {es-pull}83402[#83402] - -Packaging:: -* Bump bundled JDK to 17.0.2+8 {es-pull}83243[#83243] (issue: {es-issue}83242[#83242]) - - diff --git a/docs/reference/release-notes/8.1.0.asciidoc b/docs/reference/release-notes/8.1.0.asciidoc deleted file mode 100644 index 868739665f5fb..0000000000000 --- a/docs/reference/release-notes/8.1.0.asciidoc +++ /dev/null @@ -1,348 +0,0 @@ -[[release-notes-8.1.0]] -== {es} version 8.1.0 - -Also see <>. -[[known-issues-8.1.0]] -[float] -=== Known issues - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] -[[breaking-8.1.0]] -[float] -=== Breaking changes - -Geo:: -* Fields API should return normalize geometries {es-pull}80649[#80649] (issues: {es-issue}79232[#79232], {es-issue}63739[#63739]) - -[[bug-8.1.0]] -[float] -=== Bug fixes - -Aggregations:: -* Reenable `BooleanTermsIT` {es-pull}83421[#83421] (issue: {es-issue}83351[#83351]) -* backward compatibility with version 7.17.0 {es-pull}83715[#83715] -* ip prefix bucket reduction {es-pull}83637[#83637] -* reduce float and half-float values to their stored precision {es-pull}83213[#83213] - -Allocation:: -* Fix `updateMinNode` condition {es-pull}80403[#80403] (issue: {es-issue}41194[#41194]) -* Make `*.routing.allocation.*` list-based setting {es-pull}80420[#80420] (issue: {es-issue}77773[#77773]) -* Permit metadata updates on flood-stage-blocked indices {es-pull}81781[#81781] -* Reroute after cluster recovery {es-pull}82856[#82856] (issue: {es-issue}82456[#82456]) - -Authorization:: -* Capture anonymous roles when creating API keys {es-pull}81427[#81427] (issue: {es-issue}81024[#81024]) -* Extend fleet-server service account privileges {es-pull}82600[#82600] - -Autoscaling:: -* Fix autoscaling of follower data streams {es-pull}83302[#83302] (issue: {es-issue}82857[#82857]) - -Client:: -* Fix for self-suppression in REST client {es-pull}83568[#83568] (issue: {es-issue}42223[#42223]) - -Distributed:: -* [GCE Discovery] Correctly handle large zones with 500 or more instances {es-pull}83785[#83785] (issue: {es-issue}83783[#83783]) - -Engine:: -* Fork to `WRITE` thread when failing shard {es-pull}84606[#84606] (issue: {es-issue}84602[#84602]) - -Geo:: -* Handle bounds properly when grid tiles crosses the dateline {es-pull}83348[#83348] (issue: {es-issue}83299[#83299]) -* `GeometryNormalizer` should not fail if it cannot compute signed area {es-pull}84051[#84051] (issue: {es-issue}83946[#83946]) - -ILM+SLM:: -* Fix `PolicyStepsRegistry`'s `cachedSteps` null handling {es-pull}84588[#84588] - -Indices APIs:: -* Allow removing unreferenced composable data stream templates {es-pull}84376[#84376] (issues: {es-issue}84171[#84171], {es-issue}84188[#84188]) -* Simplify and speed up `ExecutorSelector` {es-pull}83514[#83514] (issue: {es-issue}82450[#82450]) - -Infra/Core:: -* Always re-run Feature migrations which have encountered errors {es-pull}83918[#83918] (issue: {es-issue}83917[#83917]) -* Copy `trace.id` in threadcontext stash {es-pull}83218[#83218] -* Preserve context in `ResultDeduplicator` {es-pull}84038[#84038] (issue: {es-issue}84036[#84036]) -* Registration of `SystemIndexMigrationTask` named xcontent objects {es-pull}84192[#84192] (issue: {es-issue}84115[#84115]) -* Update system index mappings if `_meta` is null {es-pull}83896[#83896] (issue: {es-issue}83890[#83890]) - -Infra/REST API:: -* Do not allow safelisted media types on Content-Type {es-pull}83448[#83448] - -Infra/Scripting:: -* Fix duplicated allow lists upon script engine creation {es-pull}82820[#82820] (issue: {es-issue}82778[#82778]) -* Fix plumbing in double and keyword runtime fields for the scripting fields API {es-pull}83392[#83392] - -Ingest:: -* Fix `GeoIpDownloader` startup during rolling upgrade {es-pull}84000[#84000] -* Short circuit date patterns after first match {es-pull}83764[#83764] - -Machine Learning:: -* Allow autoscaling to work when vertical scaling is possible {es-pull}84242[#84242] (issue: {es-issue}84198[#84198]) -* Correctly capture min stats for `inference.ingest_processors` in ML usage {es-pull}82352[#82352] -* Fail queued inference requests with cause if the process crashes {es-pull}81584[#81584] -* Fix NLP tokenization `never_split` handling around punctuation {es-pull}82982[#82982] -* Fix `ZeroShotClassificationConfig` update mixing fields {es-pull}82848[#82848] -* Fix bug where initial scale from 0->1 could scale too high {es-pull}84244[#84244] -* Fix submit after shutdown in process worker service {es-pull}83645[#83645] (issue: {es-issue}83633[#83633]) -* Fixes `categorize_text` parameter validation to be parse order independent {es-pull}82628[#82628] (issue: {es-issue}82629[#82629]) -* Record node shutdown start time for each node {es-pull}84355[#84355] -* Register the named X-content parser for snapshot upgrade params {es-pull}84420[#84420] (issue: {es-issue}84419[#84419]) -* Retry anomaly detection job recovery during relocation {es-pull}83456[#83456] -* Return `zxx` for `lang_ident_model_1` if no valid text is found for language identification {es-pull}82746[#82746] (issue: {es-issue}81933[#81933]) -* Text structure finder caps exclude lines pattern at 1000 characters {es-pull}84236[#84236] (issue: {es-issue}83434[#83434]) -* Validate vocabulary on model deployment {es-pull}81548[#81548] (issue: {es-issue}81470[#81470]) -* Wait for model process to stop in stop deployment {es-pull}83644[#83644] - -Mapping:: -* Add support for sub-fields to `search_as_you_type` fields {es-pull}82430[#82430] (issue: {es-issue}56326[#56326]) -* Better exception message for `MappingParser.parse` {es-pull}80696[#80696] -* Completion field to support multiple completion multi-fields {es-pull}83595[#83595] (issue: {es-issue}83534[#83534]) - -Network:: -* Throw `NoSeedNodeLeftException` on proxy failure {es-pull}80961[#80961] (issue: {es-issue}80898[#80898]) - -Packaging:: -* Add `log4j-slf4j-impl` to `repository-azure` {es-pull}83661[#83661] (issue: {es-issue}83652[#83652]) -* Restart ES after keystore upgrade in postinst {es-pull}84224[#84224] (issue: {es-issue}82433[#82433]) - -Recovery:: -* Add missing `indices.recovery.internal_action_retry_timeout` to list of settings {es-pull}83354[#83354] -* Add missing max overcommit factor to list of (dynamic) settings {es-pull}83350[#83350] - -SQL:: -* Fix txt format for empty result sets {es-pull}83376[#83376] - -Search:: -* Avoid eagerly loading `StoredFieldsReader` in fetch phase {es-pull}83693[#83693] (issue: {es-issue}82777[#82777]) -* Do not deserialise the document when not needed in the fields fetch phase {es-pull}84184[#84184] -* Returns valid PIT when no index matched {es-pull}83424[#83424] - -Security:: -* Add validation for API key role descriptors {es-pull}82049[#82049] (issue: {es-issue}67311[#67311]) - -Snapshot/Restore:: -* Adjust `LinuxFileSystemNatives.allocatedSizeInBytes` for aarch64 architectures {es-pull}81376[#81376] (issues: {es-issue}80437[#80437], {es-issue}81362[#81362]) -* Distinguish "missing repository" from "missing repository plugin" {es-pull}82457[#82457] (issue: {es-issue}81758[#81758]) -* Fix `DirectBlobContainerIndexInput` cloning method {es-pull}84341[#84341] (issue: {es-issue}84238[#84238]) -* Move get snapshots serialization to management pool {es-pull}83215[#83215] -* Preserve context in `snapshotDeletionListeners` {es-pull}84089[#84089] (issue: {es-issue}84036[#84036]) - -TSDB:: -* Fix time series timestamp meta missing {es-pull}80695[#80695] - -Transform:: -* Fix NPE in transform version check {es-pull}81756[#81756] -* Fix condition on which the transform stops processing buckets {es-pull}82852[#82852] -* Prevent stopping of transforms due to threadpool limitation {es-pull}81912[#81912] (issue: {es-issue}81796[#81796]) - -Watcher:: -* Tolerate empty types array in Watch definitions {es-pull}83524[#83524] (issue: {es-issue}83235[#83235]) - -[[deprecation-8.1.0]] -[float] -=== Deprecations - -CRUD:: -* Bulk actions JSON must be well-formed {es-pull}78876[#78876] (issue: {es-issue}43774[#43774]) - -Cluster Coordination:: -* Remove last few mentions of Zen discovery {es-pull}80410[#80410] - -Search:: -* Deprecate the `indices.query.bool.max_clause_count` node setting {es-pull}81525[#81525] (issue: {es-issue}46433[#46433]) - -SQL:: -* Deprecate `index_include_frozen` request parameter {es-pull}83943[#83943] (issue: {es-issue}81939[#81939]) - -[[enhancement-8.1.0]] -[float] -=== Enhancements - -Aggregations:: -* Add an aggregator for IPv4 and IPv6 subnets {es-pull}82410[#82410] -* Fail shards early when we can detect a type missmatch {es-pull}79869[#79869] (issue: {es-issue}72276[#72276]) -* Optimize `significant_text` aggregation to only parse the field it requires from `_source` {es-pull}79651[#79651] - -Allocation:: -* Identify other node in `SameShardAllocDec` message {es-pull}82890[#82890] (issue: {es-issue}80767[#80767]) -* Make `AllocationService#adaptAutoExpandReplicas` Faster {es-pull}83092[#83092] -* Speed up same host check {es-pull}80767[#80767] - -Analysis:: -* Expose Japanese completion filter to kuromoji analysis plugin {es-pull}81858[#81858] - -Authentication:: -* Enable `run_as` for all authentication schemes {es-pull}79809[#79809] -* Return API key name in `_authentication` response {es-pull}78946[#78946] (issue: {es-issue}70306[#70306]) - -Authorization:: -* Avoid loading authorized indices when requested indices are all concrete names {es-pull}81237[#81237] -* Optimize DLS bitset building for `matchAll` query {es-pull}81030[#81030] (issue: {es-issue}80904[#80904]) - -Cluster Coordination:: -* Add detail to slow cluster state warning message {es-pull}83221[#83221] -* Batch Index Settings Update Requests {es-pull}82896[#82896] (issue: {es-issue}79866[#79866]) -* Improve node-join task descriptions {es-pull}80090[#80090] -* Make `PeerFinder` log messages happier {es-pull}83222[#83222] -* More compact serialization of metadata {es-pull}82608[#82608] (issue: {es-issue}77466[#77466]) -* Paginate persisted cluster state {es-pull}78875[#78875] -* Reduce verbosity-increase timeout to 3 minutes {es-pull}81118[#81118] -* Use network recycler for publications {es-pull}80650[#80650] (issue: {es-issue}80111[#80111]) - -Data streams:: -* Defer reroute when autocreating datastream {es-pull}82412[#82412] (issue: {es-issue}82159[#82159]) - -ILM+SLM:: -* Expose the index age in ILM explain output {es-pull}81273[#81273] (issue: {es-issue}64429[#64429]) - -Indices APIs:: -* Batch auto create index cluster state updates {es-pull}82159[#82159] -* Expose 'features' option in Get Index API {es-pull}83083[#83083] (issue: {es-issue}82948[#82948]) -* Expose index health and status to the `_stats` API {es-pull}81954[#81954] (issue: {es-issue}80413[#80413]) -* Force merge REST API support `wait_for_completion` {es-pull}80463[#80463] (issues: {es-issue}80129[#80129], {es-issue}80129[#80129]) - -Infra/Circuit Breakers:: -* Allow dynamically changing the `use_real_memory` setting {es-pull}78288[#78288] (issue: {es-issue}77324[#77324]) - -Infra/Core:: -* Use `VarHandles` for number conversions {es-pull}80367[#80367] (issue: {es-issue}78823[#78823]) -* Use `VarHandles` in `ByteUtils` {es-pull}80442[#80442] (issue: {es-issue}78823[#78823]) -* `FilterPathBasedFilter` support match fieldname with dot {es-pull}83178[#83178] (issues: {es-issue}83148[#83148], {es-issue}83152[#83152]) - -Infra/REST API:: -* Allow for customised content-type validation {es-pull}80906[#80906] (issue: {es-issue}80482[#80482]) -* Update YAML REST tests to check for product header on all responses {es-pull}83290[#83290] - -Infra/Scripting:: -* Add '$' syntax as a shortcut for 'field' in Painless {es-pull}80518[#80518] -* Add `BinaryDocValuesField` to replace `BytesRef` `(ScriptDocValues)` {es-pull}79760[#79760] -* Add a geo point field for the scripting fields api {es-pull}81395[#81395] -* Add date fields to the scripting fields api {es-pull}81272[#81272] -* Add half float mapping to the scripting fields API {es-pull}82294[#82294] -* Add scaled float to the scripting fields API {es-pull}82275[#82275] -* Add support for `GeoShape` to the scripting fields API {es-pull}81617[#81617] -* Fields API for IP mapped type {es-pull}81396[#81396] -* Fields API for byte, double, float, integer, long, short {es-pull}81126[#81126] (issue: {es-issue}79105[#79105]) -* Fields API for flattened mapped type {es-pull}82590[#82590] -* Fields API for x-pack `constant_keyword` {es-pull}82292[#82292] -* Fields API for x-pack version, doc version, seq no, mumur3 {es-pull}81476[#81476] -* Improve support for joda datetime to java datetime in Painless {es-pull}83099[#83099] -* Keyword fields API support {es-pull}81266[#81266] -* Make wildcard accessible from the scripting field API {es-pull}82763[#82763] -* Ordinal field data plumbing {es-pull}80970[#80970] (issue: {es-issue}79105[#79105]) -* Support boolean fields in Fields API {es-pull}80043[#80043] (issue: {es-issue}79105[#79105]) -* Time series compile and cache evict metrics {es-pull}79078[#79078] (issue: {es-issue}62899[#62899]) - -Infra/Settings:: -* Optimize duplicated code block in `MetadataUpdateSettingsService` {es-pull}82048[#82048] - -Machine Learning:: -* Add ability to update the truncation option at inference {es-pull}80267[#80267] -* Add error counts to trained model stats {es-pull}82705[#82705] -* Add latest search interval to datafeed stats {es-pull}82620[#82620] (issue: {es-issue}82405[#82405]) -* Adds new MPNet tokenization for NLP models {es-pull}82234[#82234] -* Force delete trained models {es-pull}80595[#80595] -* Improve error message on starting scrolling datafeed with no matching indices {es-pull}81069[#81069] (issue: {es-issue}81013[#81013]) -* Report thread settings per node for trained model deployments {es-pull}81723[#81723] (issue: {es-issue}81149[#81149]) -* Set default value of 30 days for model prune window {es-pull}81377[#81377] -* Track token positions and use source string to tag NER entities {es-pull}81275[#81275] -* Warn when creating job with an unusual bucket span {es-pull}82145[#82145] (issue: {es-issue}81645[#81645]) - -Mapping:: -* Allow doc-values only search on geo_point fields {es-pull}83395[#83395] -* Implement all queries on doc-values only keyword fields {es-pull}83404[#83404] -* Optimize source filtering in `SourceFieldMapper` {es-pull}81970[#81970] (issues: {es-issue}77154[#77154], {es-issue}81575[#81575]) - -Monitoring:: -* Add Enterprise Search monitoring index templates {es-pull}82743[#82743] -* Add `beats_stats.metrics.apm-server.sampling.tail` to Stack Monitoring templates {es-pull}82401[#82401] - -Network:: -* Report close connection exceptions at INFO {es-pull}81768[#81768] (issues: {es-issue}51612[#51612], {es-issue}66473[#66473]) -* Serialize outbound messages on netty buffers {es-pull}80111[#80111] -* Track histogram of transport handling times {es-pull}80581[#80581] (issue: {es-issue}80428[#80428]) - -Recovery:: -* Adjust `indices.recovery.max_bytes_per_sec` according to external settings {es-pull}82819[#82819] - -SQL:: -* Compress Cursors {es-pull}83591[#83591] -* Extend Tableau connector to reconnect with catalog {es-pull}81321[#81321] - -Search:: -* Add `scripted_metric` agg context to `unsigned_long` {es-pull}64422[#64422] (issue: {es-issue}64347[#64347]) -* Add field usage support for vectors {es-pull}80608[#80608] -* Allow doc-values only search on boolean fields {es-pull}82925[#82925] (issues: {es-issue}82409[#82409], {es-issue}81210[#81210], {es-issue}52728[#52728]) -* Allow doc-values only search on date types {es-pull}82602[#82602] (issues: {es-issue}82409[#82409], {es-issue}81210[#81210], {es-issue}52728[#52728]) -* Allow doc-values only search on ip fields {es-pull}82929[#82929] (issues: {es-issue}82409[#82409], {es-issue}81210[#81210], {es-issue}52728[#52728]) -* Allow doc-values only search on keyword fields {es-pull}82846[#82846] (issues: {es-issue}82409[#82409], {es-issue}81210[#81210], {es-issue}52728[#52728]) -* Allow doc-values only search on number types {es-pull}82409[#82409] (issues: {es-issue}81210[#81210], {es-issue}52728[#52728]) -* Rewrite `match` and `match_phrase` queries to `term` queries on `keyword` fields {es-pull}82612[#82612] (issue: {es-issue}82515[#82515]) -* Short cut if reader has point values {es-pull}80268[#80268] -* Support combining `_shards` preference param with `` {es-pull}80024[#80024] (issue: {es-issue}80021[#80021]) - -Security:: -* Activate user profile API {es-pull}82400[#82400] -* Add an initial `ProfileService` for user profiles {es-pull}81899[#81899] -* Add new system index for user profile documents {es-pull}81355[#81355] -* Add update user profile data API {es-pull}82772[#82772] -* Add user profile API for get profile by UID {es-pull}81910[#81910] -* Update Kibana system user privileges {es-pull}82781[#82781] - -Snapshot/Restore:: -* Add Linux x86-64bits native method to retrieve the number of allocated bytes on disk for a file {es-pull}80437[#80437] (issue: {es-issue}79698[#79698]) - -Stats:: -* Add index pressure stats in cluster stats {es-pull}80303[#80303] (issue: {es-issue}79788[#79788]) -* Optimize `getIndices` in `IndicesSegmentResponse` {es-pull}80064[#80064] -* Speed up `MappingStats` Computation on Coordinating Node {es-pull}82830[#82830] - -TSDB:: -* Add `_tsid` field to `time_series` indices {es-pull}80276[#80276] -* Make time boundaries settings required in TSDB indices {es-pull}81146[#81146] - -Transform:: -* Introduce `deduce_mappings` transform setting {es-pull}82256[#82256] (issue: {es-issue}82559[#82559]) -* Make it possible to clear retention policy on an existing transform {es-pull}82703[#82703] (issue: {es-issue}82560[#82560]) -* Report transforms without config as erroneous {es-pull}81141[#81141] (issue: {es-issue}80955[#80955]) - -[[feature-8.1.0]] -[float] -=== New features - -Authentication:: -* Initial version of JWT Realm {es-pull}82175[#82175] -* Introduce domain setting to associate realms {es-pull}81968[#81968] - -Distributed:: -* Add desired nodes API {es-pull}82975[#82975] - -Geo:: -* New `GeoHexGrid` aggregation {es-pull}82924[#82924] - -Health:: -* Model for the new health reporting api {es-pull}83398[#83398] - -TSDB:: -* Handle `fields.with.dots` in `routing_path` {es-pull}83148[#83148] - -Transform:: -* Add transform reset API {es-pull}79828[#79828] (issue: {es-issue}75768[#75768]) - -[[upgrade-8.1.0]] -[float] -=== Upgrades - -Geo:: -* Update vector tiles google protobuf to 3.16.1 {es-pull}83402[#83402] - -Network:: -* Upgrade to Netty 4.1.73 {es-pull}82844[#82844] - -Packaging:: -* Bump bundled JDK to 17.0.2+8 {es-pull}83243[#83243] (issue: {es-issue}83242[#83242]) - -Security:: -* Upgrade jANSI dependency to 2.4.0 {es-pull}83566[#83566] - - - diff --git a/docs/reference/release-notes/8.1.1.asciidoc b/docs/reference/release-notes/8.1.1.asciidoc deleted file mode 100644 index d0ad36fd36487..0000000000000 --- a/docs/reference/release-notes/8.1.1.asciidoc +++ /dev/null @@ -1,65 +0,0 @@ -[[release-notes-8.1.1]] -== {es} version 8.1.1 - -Also see <>. -[[known-issues-8.1.1]] -[float] -=== Known issues - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] -[[bug-8.1.1]] -[float] -=== Bug fixes - -Analysis:: -* Fix `min_hash` configuration settings names {es-pull}84753[#84753] (issue: {es-issue}84578[#84578]) - -EQL:: -* Clean any used memory by the sequence matcher and circuit breaker used bytes in case of exception {es-pull}84451[#84451] - -Engine:: -* Increase store ref before snapshotting index commit {es-pull}84776[#84776] - -ILM+SLM:: -* Invoke initial `AsyncActionStep` for newly created indices {es-pull}84541[#84541] (issue: {es-issue}77269[#77269]) - -Indices APIs:: -* Remove existing `indices/datastreams/aliases` before simulating index template {es-pull}84675[#84675] (issue: {es-issue}84256[#84256]) - -Infra/Core:: -* Fix `NullPointerException` in `SystemIndexMetadataUpgradeService` hidden alias handling {es-pull}84780[#84780] (issue: {es-issue}81411[#81411]) -* Require and preserve content type for filtered rest requests {es-pull}84914[#84914] (issue: {es-issue}84784[#84784]) -* Wrap thread creation in `doPrivileged` call {es-pull}85180[#85180] - -Infra/REST API:: -* Correctly return `_type` field for documents in V7 compatiblity mode {es-pull}84873[#84873] (issue: {es-issue}84173[#84173]) - -Ingest:: -* Backport mark `GeoIpDownloaderTask` as completed after cancellation #84028 {es-pull}85014[#85014] (issues: {es-issue}84028[#84028], {es-issue}84652[#84652]) -* `CompoundProcessor` should also catch exceptions when executing a processor {es-pull}84838[#84838] (issue: {es-issue}84781[#84781]) - -Machine Learning:: -* Fix Kibana date format and similar overrides in text structure endpoint {es-pull}84967[#84967] -* Fixes for multi-line start patterns in text structure endpoint {es-pull}85066[#85066] -* Return all datafeeds in get anomaly detection jobs API {es-pull}84759[#84759] - -Packaging:: -* Remove use of Cloudflare zlib {es-pull}84680[#84680] - -Search:: -* Fix point visitor in `DiskUsage` API {es-pull}84909[#84909] -* `DotExpandingXContentParser` to expose the original token location {es-pull}84970[#84970] - -Snapshot/Restore:: -* Don't fail if there no symlink for AWS Web Identity Token {es-pull}84697[#84697] -* Lookup AWS Region for STS Client from STS endpoint {es-pull}84585[#84585] (issue: {es-issue}83826[#83826]) - -[[enhancement-8.1.1]] -[float] -=== Enhancements - -SQL:: -* Forward warning headers to JDBC driver {es-pull}84499[#84499] - -Watcher:: -* Add list of allowed domains for Watcher email action {es-pull}84894[#84894] (issue: {es-issue}84739[#84739]) diff --git a/docs/reference/release-notes/8.1.2.asciidoc b/docs/reference/release-notes/8.1.2.asciidoc deleted file mode 100644 index ed557e41a8a87..0000000000000 --- a/docs/reference/release-notes/8.1.2.asciidoc +++ /dev/null @@ -1,44 +0,0 @@ -[[release-notes-8.1.2]] -== {es} version 8.1.2 - -Also see <>. -[[known-issues-8.1.2]] -[float] -=== Known issues - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] -[[bug-8.1.2]] -[float] -=== Bug fixes - -Authorization:: -* Add delete privilege to `kibana_system` for APM {es-pull}85085[#85085] - -Engine:: -* Increase store ref before snapshotting index commit {es-pull}84776[#84776] - -Infra/Core:: -* Return empty version instead of blowing up if we cannot find it {es-pull}85244[#85244] -* Validate index format agreement for system index descriptors {es-pull}85173[#85173] -* Wrap thread creation in `doPrivileged` call {es-pull}85180[#85180] - -Machine Learning:: -* Do not fetch source when finding index of last state docs {es-pull}85334[#85334] -* Fixes for multi-line start patterns in text structure endpoint {es-pull}85066[#85066] -* Reallocate model deployments on node shutdown events {es-pull}85310[#85310] - -Mapping:: -* Do not fail on duplicated content field filters {es-pull}85382[#85382] - -Search:: -* Increase store ref before analyzing disk usage {es-pull}84774[#84774] -* Limit concurrent shard requests in disk usage API {es-pull}84900[#84900] (issue: {es-issue}84779[#84779]) -* `TransportBroadcastAction` should always set response for each shard {es-pull}84926[#84926] - -Snapshot/Restore:: -* Fix leaking listeners bug on frozen tier {es-pull}85239[#85239] - -Watcher:: -* No longer require master node to install Watcher templates {es-pull}85287[#85287] (issue: {es-issue}85043[#85043]) - - diff --git a/docs/reference/release-notes/8.1.3.asciidoc b/docs/reference/release-notes/8.1.3.asciidoc deleted file mode 100644 index e69e6f4cc0480..0000000000000 --- a/docs/reference/release-notes/8.1.3.asciidoc +++ /dev/null @@ -1,33 +0,0 @@ -[[release-notes-8.1.3]] -== {es} version 8.1.3 - -Also see <>. -[[known-issues-8.1.3]] -[float] -=== Known issues - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] -[[bug-8.1.3]] -[float] -=== Bug fixes - -Authorization:: -* Ignore app priv failures when resolving superuser {es-pull}85519[#85519] - -Machine Learning:: -* Avoid multiple queued quantiles documents in renormalizer {es-pull}85555[#85555] (issue: {es-issue}85539[#85539]) - -Mapping:: -* Do not fail on duplicated content field filters {es-pull}85382[#85382] - -Search:: -* Fix skip caching factor with `indices.queries.cache.all_segments` {es-pull}85510[#85510] - -Snapshot/Restore:: -* Expose proxy settings for GCS repositories {es-pull}85785[#85785] (issue: {es-issue}84569[#84569]) - -Watcher:: -* Avoiding watcher validation errors when a data stream points to more than one index {es-pull}85507[#85507] (issue: {es-issue}85508[#85508]) -* Log at WARN level for Watcher cluster state validation errors {es-pull}85632[#85632] - - diff --git a/docs/reference/release-notes/8.10.0.asciidoc b/docs/reference/release-notes/8.10.0.asciidoc deleted file mode 100644 index 34d1d26e5d69a..0000000000000 --- a/docs/reference/release-notes/8.10.0.asciidoc +++ /dev/null @@ -1,289 +0,0 @@ -[[release-notes-8.10.0]] -== {es} version 8.10.0 - -Also see <>. - -[[known-issues-8.10.0]] -[float] -=== Known issues - -// tag::repositorydata-format-change[] -* Snapshot-based downgrades -+ -The snapshot repository format changed in a manner that prevents earlier -versions of Elasticsearch from reading the repository contents if it contains -snapshots from this version and the last cluster to write to this repository -was in the 8.10 series. This will prevent you from reverting an upgrade to the -8.10 series by restoring a snapshot taken before the upgrade. -+ -Snapshot repositories written by clusters running versions 8.11.0 and later are -compatible with all earlier versions. Moreover, clusters running version 8.11.0 -or later will also automatically repair the repository format the first time -they write to the repository to take or delete a snapshot, making it so that -all earlier versions can read its contents again. -+ -If you wish to downgrade to a version prior to 8.9.0, take or delete a snapshot -using a cluster running version 8.11.0 or later to repair the repository format -first. If you cannot repair the repository in this way, first delete all the -snapshots in the repository taken with version 8.9.0 or later. To do this will -require using a cluster running version 8.10.0 or later. -+ -If you wish to downgrade to a version in the 8.9 series, you must take or delete -a snapshot using a cluster running version 8.11.0 or later to repair the -repository format first. If you cannot repair the repository in this way, first -delete all the snapshots in the repository taken with version 8.10.0 or later -using a cluster running version 8.10.4. -// end::repositorydata-format-change[] - -include::8.7.1.asciidoc[tag=no-preventive-gc-issue] - -[[breaking-8.10.0]] -[float] -=== Breaking changes - -Analysis:: -* Change pre-configured and cached analyzer components to use IndexVersion instead of Version {es-pull}97319[#97319] - -Geo:: -* Remove the unused executor builder for vector tile plugin {es-pull}96577[#96577] - -[[bug-8.10.0]] -[float] -=== Bug fixes - -Aggregations:: -* Cardinality nested in time series doc values bug {es-pull}99007[#99007] -* Skip segment for `MatchNoDocsQuery` filters {es-pull}98295[#98295] (issue: {es-issue}94637[#94637]) - -Allocation:: -* Do not assign ignored shards {es-pull}98265[#98265] -* Remove exception wrapping in `BatchedRerouteService` {es-pull}97224[#97224] - -Application:: -* [Profiling] Abort index creation on outdated index {es-pull}98864[#98864] -* [Profiling] Consider static settings in status {es-pull}97890[#97890] -* [Profiling] Mark executables without a name {es-pull}98884[#98884] - -CRUD:: -* Add missing sync on `indicesThatCannotBeCreated` {es-pull}97869[#97869] - -Cluster Coordination:: -* Fix cluster bootstrap warning for single-node discovery {es-pull}96895[#96895] (issue: {es-issue}96874[#96874]) -* Fix election scheduling after discovery outage {es-pull}98420[#98420] -* Improve reliability of elections with message delays {es-pull}98354[#98354] (issue: {es-issue}97909[#97909]) -* Make `TransportAddVotingConfigExclusionsAction` retryable {es-pull}98386[#98386] -* Release master service task on timeout {es-pull}97711[#97711] - -Data streams:: -* Avoid lifecycle NPE in the data stream lifecycle usage API {es-pull}98260[#98260] - -Distributed:: -* Avoid `transport_worker` thread in `TransportBroadcastAction` {es-pull}98001[#98001] -* Avoid `transport_worker` thread in `TransportBroadcastByNodeAction` {es-pull}97920[#97920] (issue: {es-issue}97914[#97914]) -* Fork response reading in `TransportNodesAction` {es-pull}97899[#97899] - -Downsampling:: -* Copy "index.lifecycle.name" for ILM managed indices {es-pull}97110[#97110] (issue: {es-issue}96732[#96732]) -* Downsampling: copy the `_tier_preference` setting {es-pull}96982[#96982] (issue: {es-issue}96733[#96733]) - -EQL:: -* Fix async missing events {es-pull}97718[#97718] (issue: {es-issue}97644[#97644]) - -Geo:: -* Fix how Maps#flatten handle map values inside a list {es-pull}98828[#98828] -* Fix mvt error when returning partial results {es-pull}98765[#98765] (issue: {es-issue}98730[#98730]) - -Health:: -* `_health_report` SLM indicator should use the policy ID (not the name) {es-pull}99111[#99111] - -Indices APIs:: -* Ensure frozen indices have correct tier preference {es-pull}97967[#97967] - -Infra/REST API:: -* Fix possible NPE when transportversion is null in `MainResponse` {es-pull}97203[#97203] - -Ingest Node:: -* Revert "Add mappings for enrich fields" {es-pull}98683[#98683] - -Machine Learning:: -* Avoid risk of OOM in datafeeds when memory is constrained {es-pull}98324[#98324] (issue: {es-issue}89769[#89769]) -* Detect infinite loop in the WordPiece tokenizer {es-pull}98206[#98206] -* Fix to stop aggregatable subobjects from being considered multi-fields, to support `"subobjects": false` in data frame analytics {es-pull}97705[#97705] (issue: {es-issue}88605[#88605]) -* Fix weird `change_point` bug where all data values are equivalent {es-pull}97588[#97588] -* The model loading service should not notify listeners in a sync block {es-pull}97142[#97142] - -Mapping:: -* Fix `fields` API with `subobjects: false` {es-pull}97092[#97092] (issue: {es-issue}96700[#96700]) - -Network:: -* Fork remote-cluster response handling {es-pull}97922[#97922] - -Search:: -* Fork CCS remote-cluster responses {es-pull}98124[#98124] (issue: {es-issue}97997[#97997]) -* Fork CCS search-shards handling {es-pull}98209[#98209] -* Improve test coverage for CCS search cancellation and fix response bugs {es-pull}97029[#97029] -* Make `terminate_after` early termination friendly {es-pull}97540[#97540] (issue: {es-issue}97269[#97269]) -* Track `max_score` in collapse when requested {es-pull}97703[#97703] (issue: {es-issue}97653[#97653]) - -Security:: -* Fix NPE when `GetUser` with profile uid before profile index exists {es-pull}98961[#98961] - -Snapshot/Restore:: -* Fix `BlobCacheBufferedIndexInput` large read after clone {es-pull}98970[#98970] - -TSDB:: -* Mapped field types searchable with doc values {es-pull}97724[#97724] - -Transform:: -* Fix transform incorrectly calculating date bucket on updating old data {es-pull}97401[#97401] (issue: {es-issue}97101[#97101]) - -Watcher:: -* Changing watcher to disable cookies in shared http client {es-pull}97591[#97591] - -[[deprecation-8.10.0]] -[float] -=== Deprecations - -Authorization:: -* Mark `apm_user` for removal in a future major release {es-pull}87674[#87674] - -[[enhancement-8.10.0]] -[float] -=== Enhancements - -Aggregations:: -* Improve error message when aggregation doesn't support counter field {es-pull}93545[#93545] -* Set default index mode for `TimeSeries` to `null` {es-pull}98808[#98808] (issue: {es-issue}97429[#97429]) - -Allocation:: -* Add `node.roles` to cat allocation API {es-pull}96994[#96994] - -Application:: -* [Profiling] Add initial support for upgrades {es-pull}97380[#97380] -* [Profiling] Support index migrations {es-pull}97773[#97773] - -Authentication:: -* Avoid double get {es-pull}98067[#98067] (issue: {es-issue}97928[#97928]) -* Give all acces to .slo-observability.* indice to kibana user {es-pull}97539[#97539] -* Refresh tokens without search {es-pull}97395[#97395] - -Authorization:: -* Add "operator" field to authenticate response {es-pull}97234[#97234] -* Read operator privs enabled from Env settings {es-pull}98246[#98246] -* [Fleet] Allow `kibana_system` to put datastream lifecycle {es-pull}97732[#97732] - -Data streams:: -* Install data stream template for Kibana reporting {es-pull}97765[#97765] - -Downsampling:: -* Change `MetricFieldProducer#metrics` field type from list to array {es-pull}97344[#97344] -* Improve iterating over many field producers during downsample operation {es-pull}97281[#97281] -* Run downsampling using persistent tasks {es-pull}97557[#97557] (issue: {es-issue}93582[#93582]) - -EQL:: -* EQL to use only the necessary fields in the internal `field_caps` calls {es-pull}98987[#98987] - -Engine:: -* Fix edge case for active flag for flush on idle {es-pull}97332[#97332] (issue: {es-issue}97154[#97154]) - -Health:: -* Adding special logic to the disk health check for search-only nodes {es-pull}98508[#98508] -* Health API Periodic Logging {es-pull}96772[#96772] - -ILM+SLM:: -* Separating SLM from ILM {es-pull}98184[#98184] - -Infra/Core:: -* Infrastructure to report upon document parsing {es-pull}97961[#97961] - -Infra/Node Lifecycle:: -* Check ILM status before reporting node migration STALLED {es-pull}98367[#98367] (issue: {es-issue}89486[#89486]) - -Infra/Plugins:: -* Adding `ApiFilteringActionFilter` {es-pull}97985[#97985] - -Infra/REST API:: -* Enable Serverless API protections dynamically {es-pull}97079[#97079] -* Make `RestController` pluggable {es-pull}98187[#98187] - -Infra/Settings:: -* Mark customer settings for serverless {es-pull}98051[#98051] - -Ingest Node:: -* Allow custom geo ip database files to be downloaded {es-pull}97850[#97850] - -Network:: -* Add request header size limit for RCS transport connections {es-pull}98692[#98692] - -Search:: -* Add `completion_time` time field to `async_search` get and status response {es-pull}97700[#97700] (issue: {es-issue}88640[#88640]) -* Add setting for search parallelism {es-pull}98455[#98455] -* Add support for concurrent collection when size is greater than zero {es-pull}98425[#98425] -* Cross-cluster search provides details about search on each cluster {es-pull}97731[#97731] -* Enable parallel collection in Dfs phase {es-pull}97416[#97416] -* Exclude clusters from a cross-cluster search {es-pull}97865[#97865] -* Improve MatchNoDocsQuery description {es-pull}96069[#96069] (issue: {es-issue}95741[#95741]) -* Improve exists query rewrite {es-pull}97159[#97159] -* Improve match query rewrite {es-pull}97208[#97208] -* Improve prefix query rewrite {es-pull}97209[#97209] -* Improve wildcard query and terms query rewrite {es-pull}97594[#97594] -* Introduce Synonyms Management API used for synonym and synonym_graph filters {es-pull}97962[#97962] (issue: {es-issue}38523[#38523]) -* Introduce a collector manager for `PartialHitCountCollector` {es-pull}97550[#97550] -* Introduce a collector manager for `QueryPhaseCollector` {es-pull}97410[#97410] -* Limit `_terms_enum` prefix size {es-pull}97488[#97488] (issue: {es-issue}96572[#96572]) -* Support minimum_should_match field for terms_set query {es-pull}96082[#96082] -* Support type for simple query string {es-pull}96717[#96717] -* Unwrap IOException in `ContextIndexSearcher` concurrent code-path {es-pull}98459[#98459] -* Use a collector manager in DfsPhase Knn Search {es-pull}96689[#96689] -* Use the Weight#matches mode for highlighting by default {es-pull}96068[#96068] -* Wire `QueryPhaseCollectorManager` into the query phase {es-pull}97726[#97726] -* Wire concurrent top docs collector managers when size is 0 {es-pull}97755[#97755] -* `ProfileCollectorManager` to support child profile collectors {es-pull}97387[#97387] -* cleanup some code NoriTokenizerFactory and KuromojiTokenizerFactory {es-pull}92574[#92574] - -Security:: -* Add an API for managing the settings of Security system indices {es-pull}97630[#97630] -* Support getting active-only API keys via Get API keys API {es-pull}98259[#98259] (issue: {es-issue}97995[#97995]) - -Snapshot/Restore:: -* Add Setting to optionally use mmap for shared cache IO {es-pull}97581[#97581] -* Collect additional object store stats for S3 {es-pull}98083[#98083] -* HDFS plugin add replication_factor param {es-pull}94132[#94132] - -Store:: -* Allow Lucene directory implementations to estimate their size {es-pull}97822[#97822] -* Allow `ByteSizeDirectory` to expose their data set sizes {es-pull}98085[#98085] - -TSDB:: -* Add tsdb metrics builtin component template {es-pull}97602[#97602] -* Include more downsampling status statistics {es-pull}96930[#96930] (issue: {es-issue}96760[#96760]) -* `TimeSeriesIndexSearcher` to offload to the provided executor {es-pull}98414[#98414] - -Transform:: -* Support boxplot aggregation in transform {es-pull}96515[#96515] - -[[feature-8.10.0]] -[float] -=== New features - -Application:: -* Enable Query Rules as technical preview {es-pull}97466[#97466] -* [Enterprise Search] Add connectors indices and ent-search pipeline {es-pull}97463[#97463] - -Data streams:: -* Introduce downsampling configuration for data stream lifecycle {es-pull}97041[#97041] - -Search:: -* Introduce executor for concurrent search {es-pull}98204[#98204] - -Security:: -* Beta release for API key based cross-cluster access {es-pull}98307[#98307] - -[[upgrade-8.10.0]] -[float] -=== Upgrades - -Network:: -* Upgrade Netty to 4.1.94.Final {es-pull}97040[#97040] - - diff --git a/docs/reference/release-notes/8.10.1.asciidoc b/docs/reference/release-notes/8.10.1.asciidoc deleted file mode 100644 index 0cb00699eeac7..0000000000000 --- a/docs/reference/release-notes/8.10.1.asciidoc +++ /dev/null @@ -1,30 +0,0 @@ -[[release-notes-8.10.1]] -== {es} version 8.10.1 - -Also see <>. - -[[known-issues-8.10.1]] -[float] -=== Known issues - -include::8.10.0.asciidoc[tag=repositorydata-format-change] - -include::8.7.1.asciidoc[tag=no-preventive-gc-issue] - -[[bug-8.10.1]] -[float] -=== Bug fixes - -Aggregations:: -* Use long in Centroid count {es-pull}99491[#99491] (issue: {es-issue}80153[#80153]) - -Infra/Core:: -* Fix deadlock between Cache.put and Cache.invalidateAll {es-pull}99480[#99480] (issue: {es-issue}99326[#99326]) - -Infra/Node Lifecycle:: -* Fork computation in `TransportGetShutdownStatusAction` {es-pull}99490[#99490] (issue: {es-issue}99487[#99487]) - -Search:: -* Fix PIT when resolving with deleted indices {es-pull}99281[#99281] - - diff --git a/docs/reference/release-notes/8.10.2.asciidoc b/docs/reference/release-notes/8.10.2.asciidoc deleted file mode 100644 index 911a410104a26..0000000000000 --- a/docs/reference/release-notes/8.10.2.asciidoc +++ /dev/null @@ -1,12 +0,0 @@ -[[release-notes-8.10.2]] -== {es} version 8.10.2 - -[[known-issues-8.10.2]] -[float] -=== Known issues - -include::8.10.0.asciidoc[tag=repositorydata-format-change] - -include::8.7.1.asciidoc[tag=no-preventive-gc-issue] - -Also see <>. diff --git a/docs/reference/release-notes/8.10.3.asciidoc b/docs/reference/release-notes/8.10.3.asciidoc deleted file mode 100644 index 119930058a42e..0000000000000 --- a/docs/reference/release-notes/8.10.3.asciidoc +++ /dev/null @@ -1,87 +0,0 @@ -[[release-notes-8.10.3]] -== {es} version 8.10.3 - -[[known-issues-8.10.3]] -[float] -=== Known issues - -include::8.10.0.asciidoc[tag=repositorydata-format-change] - -// tag::no-preventive-gc-issue[] -* High Memory Pressure due to a GC change in JDK 21 -+ -This version of Elasticsearch is bundled with JDK 21. In JDK 21 -https://bugs.openjdk.org/browse/JDK-8297639[Preventive GC has been removed]. -This may lead to increased memory pressure and an increased number of CircuitBreakerExceptions when retrieving large -documents under some particular load. (issue: {es-issue}99592[#99592]) -+ -If you needed to explicitly <>, we recommend you avoid to upgrade to this version, as the settings to enable Preventive GC have been removed -from JDK 21. -// end::no-preventive-gc-issue[] - -Also see <>. - -[[bug-8.10.3]] -[float] -=== Bug fixes - -Aggregations:: -* Fix cardinality agg for `const_keyword` {es-pull}99814[#99814] (issue: {es-issue}99776[#99776]) - -Distributed:: -* Skip settings validation during desired nodes updates {es-pull}99946[#99946] - -Highlighting:: -* Implement matches() on `SourceConfirmedTextQuery` {es-pull}100252[#100252] - -ILM+SLM:: -* ILM introduce the `check-ts-end-time-passed` step {es-pull}100179[#100179] (issue: {es-issue}99696[#99696]) -* ILM the delete action waits for a TSDS index time/bounds to lapse {es-pull}100207[#100207] - -Ingest Node:: -* Validate enrich index before completing policy execution {es-pull}100106[#100106] - -Machine Learning:: -* Adding retry logic for start model deployment API {es-pull}99673[#99673] -* Using 1 MB chunks for elser model storage {es-pull}99677[#99677] - -Search:: -* Close expired search contexts on SEARCH thread {es-pull}99660[#99660] -* Fix fields API for `geo_point` fields inside other arrays {es-pull}99868[#99868] (issue: {es-issue}99781[#99781]) - -Snapshot/Restore:: -* Support $ and / in restore rename replacements {es-pull}99892[#99892] (issue: {es-issue}99078[#99078]) - -Transform:: -* Do not use PIT in the presence of remote indices in source {es-pull}99803[#99803] -* Ignore "index not found" error when `delete_dest_index` flag is set but the dest index doesn't exist {es-pull}99738[#99738] -* Let `_stats` internally timeout if checkpoint information can not be retrieved {es-pull}99914[#99914] - -Vector Search:: -* Update version range in `jvm.options` for the Panama Vector API {es-pull}99846[#99846] - -[[enhancement-8.10.3]] -[float] -=== Enhancements - -Authorization:: -* Add manage permission for fleet managed threat intel indices {es-pull}99231[#99231] - -Highlighting:: -* Implement matches() on `SourceConfirmedTextQuery` {es-pull}100134[#100134] - -Ingest Node:: -* Show a concrete error when the enrich index does not exist rather than a NullPointerException {es-pull}99604[#99604] - -Search:: -* Add checks in term and terms queries that input terms are not too long {es-pull}99818[#99818] (issue: {es-issue}99802[#99802]) - -[[upgrade-8.10.3]] -[float] -=== Upgrades - -Packaging:: -* Upgrade bundled JDK to Java 21 {es-pull}99724[#99724] - - diff --git a/docs/reference/release-notes/8.10.4.asciidoc b/docs/reference/release-notes/8.10.4.asciidoc deleted file mode 100644 index 6c49bae1e2150..0000000000000 --- a/docs/reference/release-notes/8.10.4.asciidoc +++ /dev/null @@ -1,45 +0,0 @@ -[[release-notes-8.10.4]] -== {es} version 8.10.4 - -[[known-issues-8.10.4]] -[float] -=== Known issues - -* Snapshot-based downgrades -+ -The snapshot repository format changed in a manner that prevents earlier -versions of Elasticsearch from reading the repository contents if it contains -snapshots from this version and the last cluster to write to this repository was -in the 8.10 series. This will prevent you from reverting an upgrade to the 8.10 -series by restoring a snapshot taken before the upgrade. -+ -Snapshot repositories written by clusters running versions 8.11.0 and later are -compatible with all earlier versions. Moreover, clusters running version 8.11.0 -or later will also automatically repair the repository format the first time -they write to the repository to take or delete a snapshot, making it so that all -earlier versions can read its contents again. -+ -If you wish to downgrade to a version prior to 8.10.0, take or delete a snapshot -using a cluster running version 8.11.0 or later to repair the repository format -first. If you cannot repair the repository in this way, first delete all the -snapshots in the repository taken with version 8.10.0 or later using a cluster -running version 8.10.4. - -include::8.10.3.asciidoc[tag=no-preventive-gc-issue] - -Also see <>. - -[[bug-8.10.4]] -[float] -=== Bug fixes - -Search:: -* Search of remote clusters with no shards results in successful status {es-pull}100354[#100354] - -Snapshot/Restore:: -* Improve `RepositoryData` BwC {es-pull}100401[#100401] - -Transform:: -* Shutdown the task immediately when `force` == `true` {es-pull}100203[#100203] - - diff --git a/docs/reference/release-notes/8.11.0.asciidoc b/docs/reference/release-notes/8.11.0.asciidoc deleted file mode 100644 index acb27dc180727..0000000000000 --- a/docs/reference/release-notes/8.11.0.asciidoc +++ /dev/null @@ -1,342 +0,0 @@ -[[release-notes-8.11.0]] -== {es} version 8.11.0 - -Also see <>. - -[[breaking-8.11.0]] -[float] -=== Breaking changes - -Infra/Core:: -* Remove `transport_versions` from cluster state API {es-pull}99223[#99223] - -[[known-issues-8.11.0]] -[float] -=== Known issues -include::8.10.3.asciidoc[tag=no-preventive-gc-issue] - -[[bug-8.11.0]] -[float] -=== Bug fixes - -Aggregations:: -* Adjust `DateHistogram's` bucket accounting to be iteratively {es-pull}101012[#101012] -* Allow parsing on non-string routing fields {es-pull}97729[#97729] -* Support runtime fields in synthetic source {es-pull}99796[#99796] (issue: {es-issue}98287[#98287]) - -Allocation:: -* Consider node shutdown in `DataTierAllocationDecider` {es-pull}98824[#98824] (issue: {es-issue}97207[#97207]) - -Application:: -* Align look-back with client-side cache {es-pull}101264[#101264] -* Increase K/V look-back time interval {es-pull}101205[#101205] -* Provide stable resampling {es-pull}101255[#101255] -* [Profiling] Tighten resource creation check {es-pull}99873[#99873] - -Authorization:: -* Allow `enrich_user` to read/view enrich indices {es-pull}100707[#100707] -* Grant editor and viewer access to profiling {es-pull}100594[#100594] - -CCR:: -* CCR: Use local cluster state request {es-pull}100323[#100323] - -CRUD:: -* Change `GetFromTranslog` to indices action {es-pull}99300[#99300] -* Wait for cluster to recover before resolving index template {es-pull}99797[#99797] - -Cluster Coordination:: -* Reset `GatewayService` flags before reroute {es-pull}98653[#98653] (issue: {es-issue}98606[#98606]) - -Data streams:: -* DSL waits for the tsdb time boundaries to lapse {es-pull}100470[#100470] (issue: {es-issue}99696[#99696]) -* Propagate cancellation in `DataTiersUsageTransportAction` {es-pull}100253[#100253] -* [DSL] skip deleting indices that have in-progress downsampling operations {es-pull}101495[#101495] - -Downsampling:: -* Make downsample target index replicas configurable {es-pull}99712[#99712] - -ES|QL:: -* "params" correctly parses the values including an optional "type" {es-pull}99310[#99310] (issue: {es-issue}99294[#99294]) -* Account for an exception being thrown when building a `BytesRefArrayBlock` {es-pull}99726[#99726] (issue: {es-issue}99472[#99472]) -* Add arithmetic operators {es-pull}98628[#98628] -* Add identity check in Block equality {es-pull}100377[#100377] (issue: {es-issue}100374[#100374]) -* Adds Enrich implicit `match_fields` to `field_caps` call {es-pull}101456[#101456] (issue: {es-issue}101328[#101328]) -* Better management of not stored TEXT fiels with synthetic source {es-pull}99695[#99695] -* Continue resolving attributes for Eval {es-pull}99601[#99601] (issue: {es-issue}99576[#99576]) -* Create a Vector when needed for IN {es-pull}99382[#99382] (issue: {es-issue}99347[#99347]) -* ESQL: Fix unreleased block in topn {es-pull}101648[#101648] (issue: {es-issue}101588[#101588]) -* ESQL: check type before casting {es-pull}101492[#101492] (issue: {es-issue}101489[#101489]) -* Fix NPE when aggregating literals {es-pull}99827[#99827] -* Fix escaping of backslash in LIKE operator {es-pull}101120[#101120] (issue: {es-issue}101106[#101106]) -* Fix eval of functions on foldable literals {es-pull}101438[#101438] (issue: {es-issue}101425[#101425]) -* Fix non-null value being returned for unsupported data types in `ValueSources` {es-pull}100656[#100656] (issue: {es-issue}100048[#100048]) -* Graceful handling of non-bool condition in the filter {es-pull}100645[#100645] (issues: {es-issue}100049[#100049], {es-issue}100409[#100409]) -* Handle queries with non-existing enrich policies and no field {es-pull}100647[#100647] (issue: {es-issue}100593[#100593]) -* Implement serialization of `InvalidMappedField` {es-pull}98972[#98972] (issue: {es-issue}98851[#98851]) -* Improve verifier error for incorrect agg declaration {es-pull}100650[#100650] (issue: {es-issue}100641[#100641]) -* Limit how many bytes `concat()` can process {es-pull}100360[#100360] -* Make DISSECT parameter `append_separator` case insensitive {es-pull}101358[#101358] (issue: {es-issue}101138[#101138]) -* Page shouldn't close a block twice {es-pull}100370[#100370] (issues: {es-issue}100356[#100356], {es-issue}100365[#100365]) -* Preserve intermediate aggregation output in local relation {es-pull}100866[#100866] (issue: {es-issue}100807[#100807]) -* Properly handle multi-values in fold() and date math {es-pull}100766[#100766] (issue: {es-issue}100497[#100497]) -* Remove aliasing inside Eval {es-pull}100238[#100238] (issue: {es-issue}100174[#100174]) -* Resilience to non-indexed fields {es-pull}99588[#99588] (issue: {es-issue}99506[#99506]) -* Skip synthetic attributes when planning the physical fragment {es-pull}99188[#99188] (issue: {es-issue}99170[#99170]) -* Support date and time intervals as input params {es-pull}101001[#101001] (issue: {es-issue}99570[#99570]) -* Support queries that don't return underlying fields {es-pull}98759[#98759] (issue: {es-issue}98404[#98404]) -* Use exact attributes for data source extraction {es-pull}99874[#99874] (issue: {es-issue}99183[#99183]) -* `mv_expand` pushes down limit and project and keep the limit after it untouched {es-pull}100782[#100782] (issues: {es-issue}99971[#99971], {es-issue}100774[#100774]) -* support metric tsdb fields while querying index patterns {es-pull}100351[#100351] (issue: {es-issue}100144[#100144]) - -Geo:: -* Use `NamedWritable` to enable `GeoBoundingBox` serialisation {es-pull}99163[#99163] (issue: {es-issue}99089[#99089]) - -Health:: -* Fix NPE in `StableMasterHealthIndicatorService` {es-pull}98635[#98635] -* Health report infrastructure doesn't trip the circuit breakers {es-pull}101629[#101629] -* Propagate cancellation in `GetHealthAction` {es-pull}100273[#100273] - -Highlighting:: -* Correctly handle `ScriptScoreQuery` in plain highlighter {es-pull}99804[#99804] (issue: {es-issue}99700[#99700]) -* Disable `weight_matches` when kNN query is present {es-pull}101713[#101713] - -ILM+SLM:: -* Compute SLM retention from `RepositoryData` {es-pull}100092[#100092] (issue: {es-issue}99953[#99953]) -* `WaitForSnapshotStep` verifies if the index belongs to the latest snapshot of that SLM policy {es-pull}100911[#100911] - -Infra/Core:: -* Add `java.net.NetPermission` to APM module's permissions {es-pull}99474[#99474] -* Don't update system index mappings in mixed clusters {es-pull}101778[#101778] (issues: {es-issue}101331[#101331], {es-issue}99778[#99778]) -* Revert "Kibana system index does not allow user templates to affect it" {es-pull}98888[#98888] -* Specify correct current `IndexVersion` after 8.10 release {es-pull}98574[#98574] (issue: {es-issue}98555[#98555]) -* Tracing: Use `doPriv` when working with spans, use `SpanId` {es-pull}100232[#100232] - -Infra/Scripting:: -* Improve painless error wrapping {es-pull}100872[#100872] - -Ingest Node:: -* Improving tika handling {es-pull}101486[#101486] -* Update enrich execution to only set index false on fields that support it {es-pull}98038[#98038] (issue: {es-issue}98019[#98019]) - -Machine Learning:: -* Avoid risk of OOM in datafeeds when memory is constrained {es-pull}98915[#98915] (issue: {es-issue}89769[#89769]) -* Fix for inference requests being sent to every node with a model allocation. If there are more nodes than items in the original request then empty requests were sent. {es-pull}100388[#100388] (issue: {es-issue}100180[#100180]) -* Preserve order of inference results when calling the _infer API with multiple inputs on a model deployment with more than one allocation the output results order was not guaranteed to match the input order. The fix ensures the output order matches the input order. {es-pull}100143[#100143] -* Remove noisy 'Could not find trained model' message {es-pull}100760[#100760] -* Safely drain deployment request queues before allowing node to shutdown {es-pull}98406[#98406] -* Use the correct writable name for model assignment metadata in mixed version clusters. Prevents a node failure due to IllegalArgumentException Unknown NamedWriteable [trained_model_assignment] {es-pull}100886[#100886] -* Wait to gracefully stop deployments until alternative allocation exists {es-pull}99107[#99107] - -Mapping:: -* Automatically disable `ignore_malformed` on datastream `@timestamp` fields {es-pull}99346[#99346] -* Correct behaviour of `ContentPath::remove()` {es-pull}98332[#98332] (issue: {es-issue}98327[#98327]) -* Fix merges of mappings with `subobjects: false` for composable index templates {es-pull}97317[#97317] (issue: {es-issue}96768[#96768]) -* Percolator to support parsing script score query with params {es-pull}101051[#101051] (issue: {es-issue}97377[#97377]) - -Network:: -* Do not report failure after connections are made {es-pull}99117[#99117] - -Percolator:: -* Fix percolator query for stored queries that expand on wildcard field names {es-pull}98878[#98878] - -Query Languages:: -* Preserve subfields for unsupported types {es-pull}100875[#100875] (issue: {es-issue}100869[#100869]) - -Recovery:: -* Fix interruption of `markAllocationIdAsInSync` {es-pull}100610[#100610] (issues: {es-issue}96578[#96578], {es-issue}100589[#100589]) - -Search:: -* Consistent scores for multi-term `SourceConfirmedTestQuery` {es-pull}100846[#100846] (issue: {es-issue}98712[#98712]) -* Fix UnsignedLong field range query gt "0" can get the result equal to 0 {es-pull}98843[#98843] -* Fix `advanceExact` for doc values from sources {es-pull}99685[#99685] -* Fork response-sending in `OpenPointInTimeAction` {es-pull}99222[#99222] -* [CI] `SearchResponseTests#testSerialization` failing resolved {es-pull}100020[#100020] (issue: {es-issue}100005[#100005]) -* fix fuzzy query rewrite parameter not work {es-pull}97642[#97642] - -Security:: -* Fix NullPointerException in RotableSecret {es-pull}100779[#100779] (issue: {es-issue}99759[#99759]) - -Snapshot/Restore:: -* Fix race condition in `SnapshotsService` {es-pull}101652[#101652] -* Fix snapshot double finalization {es-pull}101497[#101497] -* Fix thread context in `getRepositoryData` {es-pull}99627[#99627] -* Frozen index input clone copy cache file {es-pull}98930[#98930] -* Make S3 anti-contention delay configurable {es-pull}101245[#101245] -* More robust timeout for repo analysis {es-pull}101184[#101184] (issue: {es-issue}101182[#101182]) -* Register `repository_s3` settings {es-pull}101344[#101344] -* Reinstate `RepositoryData` BwC {es-pull}100447[#100447] - -TSDB:: -* Don't ignore empty index template that have no template definition {es-pull}98840[#98840] (issue: {es-issue}98834[#98834]) -* Fix painless execute api and tsdb issue {es-pull}101212[#101212] (issue: {es-issue}101072[#101072]) -* Make tsdb settings public in Serverless {es-pull}99567[#99567] (issue: {es-issue}99563[#99563]) - -Transform:: -* Fix possible NPE when getting transform stats for failed transforms {es-pull}98061[#98061] (issue: {es-issue}98052[#98052]) -* Ignore `IndexNotFound` error when refreshing destination index {es-pull}101627[#101627] -* Make Transform Feature Reset really wait for all the tasks {es-pull}100624[#100624] -* Make tasks that calculate checkpoints cancellable {es-pull}100808[#100808] - -Watcher:: -* Treating watcher webhook response header names as case-insensitive {es-pull}99717[#99717] - -[[deprecation-8.11.0]] -[float] -=== Deprecations - -Rollup:: -* Rollup functionality is now deprecated {es-pull}101265[#101265] - -[[enhancement-8.11.0]] -[float] -=== Enhancements - -Aggregations:: -* Disable `FilterByFilterAggregator` through `ClusterSettings` {es-pull}99417[#99417] (issue: {es-issue}99335[#99335]) -* Represent histogram value count as long {es-pull}99912[#99912] (issue: {es-issue}99820[#99820]) -* Skip `DisiPriorityQueue` on single filter agg {es-pull}99215[#99215] (issue: {es-issue}99202[#99202]) -* Use a competitive iterator in `FiltersAggregator` {es-pull}98360[#98360] (issue: {es-issue}97544[#97544]) - -Allocation:: -* Report a node's "roles" setting in the /_cluster/allocation/explain response {es-pull}98550[#98550] (issue: {es-issue}97859[#97859]) - -Application:: -* Add flamegraph API {es-pull}99091[#99091] -* [Profiling] Allow to customize the ILM policy {es-pull}99909[#99909] -* [Profiling] Allow to wait until resources created {es-pull}99655[#99655] - -Audit:: -* Reduce verbosity of the bulk indexing audit log {es-pull}98470[#98470] - -Authentication:: -* Allow native users/roles to be disabled via setting {es-pull}98654[#98654] - -CAT APIs:: -* Add 'dataset' size to cat indices and cat shards {es-pull}98622[#98622] (issue: {es-issue}95092[#95092]) - -Data streams:: -* Allow explain data stream lifecycle to accept a data stream {es-pull}98811[#98811] - -ES|QL:: -* Add `CEIL` function {es-pull}98847[#98847] -* Add ability to perform date math {es-pull}98870[#98870] (issue: {es-issue}98402[#98402]) -* Add support for TEXT fields in comparison operators and SORT {es-pull}98528[#98528] (issue: {es-issue}98642[#98642]) -* Compact topn {es-pull}99316[#99316] -* Date math for negatives {es-pull}99711[#99711] -* Enable arithmetics for durations and periods {es-pull}99432[#99432] (issue: {es-issue}99293[#99293]) -* Enhance SHOW FUNCTIONS command {es-pull}99736[#99736] (issue: {es-issue}99507[#99507]) -* Improve log messages {es-pull}99470[#99470] -* Log execution time consistently {es-pull}99286[#99286] -* Log query and execution time {es-pull}99058[#99058] -* Log start and end of queries {es-pull}99746[#99746] -* Lower the implicit limit, if none is user-provided {es-pull}99816[#99816] (issue: {es-issue}99458[#99458]) -* Make settings dynamic {es-pull}101516[#101516] -* Mark counter fields as unsupported {es-pull}99054[#99054] -* Remove the swapped-args check for date_xxx() {es-pull}101362[#101362] (issue: {es-issue}99562[#99562]) -* Serialize the source in expressions {es-pull}99956[#99956] -* Simple check if all blocks get released {es-pull}100199[#100199] -* Support unsigned long in sqrt and log10 {es-pull}98711[#98711] -* Use DEBUG log level to report execution steps {es-pull}99303[#99303] - -Engine:: -* Use `IndexWriter.flushNextBuffer()` to reclaim memory from indexing buffers {es-pull}94607[#94607] - -Health:: -* Avoiding the use of nodes that are no longer in the cluster when computing master stability {es-pull}98809[#98809] (issue: {es-issue}98636[#98636]) -* When a primary is inactive but this is considered expected, the same applies for the replica of this shard. {es-pull}99995[#99995] (issue: {es-issue}99951[#99951]) - -Infra/Core:: -* APM Metering API {es-pull}99832[#99832] -* Update the elastic-apm-agent version {es-pull}100064[#100064] -* Use mappings version to retrieve system index mappings at creation time {es-pull}99555[#99555] - -Infra/Node Lifecycle:: -* Add links to docs from failing bootstrap checks {es-pull}99644[#99644] (issue: {es-issue}99614[#99614]) -* Chunk `SingleNodeShutdownStatus` and `ShutdownShardMigrationStatus` (and related action) response {es-pull}99798[#99798] (issue: {es-issue}99678[#99678]) - -Infra/REST API:: -* Add `IndexVersion` to node info {es-pull}99515[#99515] -* Add component info versions to node info in a pluggable way {es-pull}99631[#99631] -* Return a 410 (Gone) status code for unavailable API endpoints {es-pull}97397[#97397] - -Machine Learning:: -* Add new _inference API {es-pull}99224[#99224] -* Adding an option for trained models to be platform specific {es-pull}99584[#99584] -* Log warnings for jobs unassigned for a long time {es-pull}100154[#100154] -* Simplify the Inference Ingest Processor configuration {es-pull}100205[#100205] - -Mapping:: -* Automatically flatten objects when subobjects:false {es-pull}97972[#97972] (issue: {es-issue}88934[#88934]) -* Explicit parsing object capabilities of `FieldMappers` {es-pull}98684[#98684] (issue: {es-issue}98537[#98537]) -* Reintroduce `sparse_vector` mapping {es-pull}98996[#98996] - -Network:: -* Chunk the cluster allocation explain response {es-pull}99641[#99641] (issue: {es-issue}97803[#97803]) - -Recovery:: -* Wait for cluster state in recovery {es-pull}99193[#99193] - -Search:: -* Add additional counters to `_clusters` response for all Cluster search states {es-pull}99566[#99566] (issue: {es-issue}98927[#98927]) -* Adding support for exist queries to `sparse_vector` fields {es-pull}99775[#99775] (issue: {es-issue}99319[#99319]) -* Make `_index` optional for pinned query docs {es-pull}97450[#97450] -* Reduce copying when creating scroll/PIT ids {es-pull}99219[#99219] -* Refactor `SearchResponseClusters` to use CHM {es-pull}100129[#100129] (issue: {es-issue}99101[#99101]) -* Support cluster/details for CCS minimize_roundtrips=false {es-pull}98457[#98457] - -Security:: -* Support rotatating the JWT shared secret {es-pull}99278[#99278] - -Snapshot/Restore:: -* Remove shard data files when they fail to write for snapshot {es-pull}99694[#99694] - -Stats:: -* Prune unnecessary information from TransportNodesInfoAction.NodeInfoRequest {es-pull}99938[#99938] (issue: {es-issue}99744[#99744]) - -TSDB:: -* Add `index.look_back_time` setting for tsdb data streams {es-pull}98518[#98518] (issue: {es-issue}98463[#98463]) -* Improve time-series error and documentation {es-pull}100018[#100018] -* Trim stored fields for `_id` field in tsdb {es-pull}97409[#97409] - -Transform:: -* Add accessors required to recreate `TransformStats` object from the fields {es-pull}98844[#98844] - -Vector Search:: -* Add new max_inner_product vector similarity function {es-pull}99527[#99527] -* Adds `nested` support for indexed `dense_vector` fields {es-pull}99763[#99763] -* Dense vector field types are indexed by default {es-pull}98268[#98268] -* Increase the max vector dims to 4096 {es-pull}99682[#99682] - -[[feature-8.11.0]] -[float] -=== New features - -Analysis:: -* Add support for Persian language stemmer {es-pull}99106[#99106] (issue: {es-issue}98911[#98911]) - -Application:: -* Automatically map float arrays of lengths 128 - 2048 as dense_vector {es-pull}98512[#98512] (issue: {es-issue}97532[#97532]) - -Data streams:: -* GA the data stream lifecycle {es-pull}100187[#100187] -* GET `_data_stream` displays both ILM and DSL information {es-pull}99947[#99947] - -ES|QL:: -* Integrate Elasticsearch Query Language, ES|QL {es-pull}98309[#98309] -* LEAST and GREATEST functions {es-pull}98630[#98630] -* LEFT function {es-pull}98942[#98942] -* LTRIM, RTRIM and fix unicode whitespace {es-pull}98590[#98590] -* RIGHT function {es-pull}98974[#98974] -* TopN sorting with min and max for multi-value fields {es-pull}98337[#98337] - -[[upgrade-8.11.0]] -[float] -=== Upgrades - -Packaging:: -* Update bundled JDK to 21.0.1 {es-pull}101133[#101133] - -Search:: -* Upgrade main to Lucene 9.8.0 {es-pull}100138[#100138] - - diff --git a/docs/reference/release-notes/8.11.1.asciidoc b/docs/reference/release-notes/8.11.1.asciidoc deleted file mode 100644 index b1dbc4a95c963..0000000000000 --- a/docs/reference/release-notes/8.11.1.asciidoc +++ /dev/null @@ -1,43 +0,0 @@ -[[release-notes-8.11.1]] -== {es} version 8.11.1 - -Also see <>. - -[[known-issues-8.11.1]] -[float] -=== Known issues -include::8.10.3.asciidoc[tag=no-preventive-gc-issue] - -[[bug-8.11.1]] -[float] -=== Bug fixes - -Allocation:: -* Avoid negative `DesiredBalanceStats#lastConvergedIndex` {es-pull}101998[#101998] - -Authentication:: -* Fix memory leak from JWT cache (and fix the usage of the JWT auth cache) {es-pull}101799[#101799] - -Machine Learning:: -* Fix inference timeout from the Inference Ingest Processor {es-pull}101971[#101971] - -Mapping:: -* Fix incorrect dynamic mapping for non-numeric-value arrays #101965 {es-pull}101967[#101967] - -Network:: -* Fail listener on exception in `TcpTransport#openConnection` {es-pull}101907[#101907] (issue: {es-issue}100510[#100510]) - -Search:: -* Dry up `AsyncTaskIndexService` memory management and fix inefficient circuit breaker use {es-pull}101892[#101892] - -Snapshot/Restore:: -* Respect regional AWS STS endpoints {es-pull}101705[#101705] (issue: {es-issue}89175[#89175]) - -[[enhancement-8.11.1]] -[float] -=== Enhancements - -Machine Learning:: -* Add inference counts by model to the machine learning usage stats {es-pull}101915[#101915] - - diff --git a/docs/reference/release-notes/8.11.2.asciidoc b/docs/reference/release-notes/8.11.2.asciidoc deleted file mode 100644 index 75987ce6139a6..0000000000000 --- a/docs/reference/release-notes/8.11.2.asciidoc +++ /dev/null @@ -1,89 +0,0 @@ -[[release-notes-8.11.2]] -== {es} version 8.11.2 - -Also see <>. - -[[known-issues-8.11.2]] -[float] -=== Known issues -include::8.10.3.asciidoc[tag=no-preventive-gc-issue] - -[float] -[[security-updates-8.11.2]] -=== Security updates - -* The 8.11.2 patch release contains a fix for a potential security vulnerability. https://discuss.elastic.co/c/announcements/security-announcements/31[Please see our security advisory for more details]. - -[[bug-8.11.2]] -[float] -=== Bug fixes - -Allocation:: -* Improve failure handling in `ContinuousComputation` {es-pull}102281[#102281] - -Application:: -* Default `run_ml_inference` should be true {es-pull}102151[#102151] -* [Query Rules] Fix bug where combining the same metadata with text/numeric values leads to error {es-pull}102891[#102891] (issue: {es-issue}102827[#102827]) - -Cluster Coordination:: -* Synchronize Coordinator#onClusterStateApplied {es-pull}100986[#100986] (issue: {es-issue}99023[#99023]) - -Data streams:: -* [Usage API] Count all the data streams that have lifecycle {es-pull}102259[#102259] - -ES|QL:: -* ES|QL: Fix drop of renamed grouping {es-pull}102282[#102282] (issue: {es-issue}102121[#102121]) -* ES|QL: Fix layout management for Project {es-pull}102399[#102399] (issue: {es-issue}102120[#102120]) -* Fix DISSECT with empty patterns {es-pull}102580[#102580] (issue: {es-issue}102577[#102577]) -* Fix leaking blocks in TopN {es-pull}102715[#102715] (issue: {es-issue}102646[#102646]) -* Fix leaking blocks in `BlockUtils` {es-pull}102716[#102716] -* Fix memory tracking in TopN.Row {es-pull}102831[#102831] (issues: {es-issue}100640[#100640], {es-issue}102784[#102784], {es-issue}102790[#102790], {es-issue}102683[#102683]) - -ILM+SLM:: -* [ILM] Fix downsample to skip already downsampled indices {es-pull}102250[#102250] (issue: {es-issue}102249[#102249]) - -Infra/Circuit Breakers:: -* Add more logging to the real memory circuit breaker and lower minimum interval {es-pull}102396[#102396] - -Ingest Node:: -* Better processor stat merge {es-pull}102821[#102821] - -Machine Learning:: -* Ensure datafeed previews with no start or end time don't search the cold or frozen tiers {es-pull}102492[#102492] -* Recreate the Elasticsearch private temporary directory if it doesn't exist when an ML job is opened {es-pull}102599[#102599] - -Mapping:: -* Fix dense_vector cluster stats indexed_vector_dim_min/max values {es-pull}102467[#102467] (issue: {es-issue}102416[#102416]) - -Search:: -* Allow mismatched sort-by field types if there are no docs to sort {es-pull}102779[#102779] - -Security:: -* Fix double-completion in `SecurityUsageTransportAction` {es-pull}102114[#102114] (issue: {es-issue}102111[#102111]) - -Snapshot/Restore:: -* Set region for the STS client via privileged calls in AWS SDK {es-pull}102230[#102230] (issue: {es-issue}102173[#102173]) -* Simplify `BlobStoreRepository` idle check {es-pull}102057[#102057] (issue: {es-issue}101948[#101948]) - -Transform:: -* Ensure transform updates only modify the expected transform task {es-pull}102934[#102934] (issue: {es-issue}102933[#102933]) -* Exclude stack traces from transform audit messages and health {es-pull}102240[#102240] - -[[enhancement-8.11.2]] -[float] -=== Enhancements - -Machine Learning:: -* Add inference counts by model to the machine learning usage stats {es-pull}101915[#101915] - -Security:: -* Upgrade xmlsec to 2.3.4 {es-pull}102220[#102220] - -[[upgrade-8.11.2]] -[float] -=== Upgrades - -Snapshot/Restore:: -* Upgrade reactor netty http version {es-pull}102311[#102311] - - diff --git a/docs/reference/release-notes/8.11.3.asciidoc b/docs/reference/release-notes/8.11.3.asciidoc deleted file mode 100644 index ddeb50dad1f75..0000000000000 --- a/docs/reference/release-notes/8.11.3.asciidoc +++ /dev/null @@ -1,28 +0,0 @@ -[[release-notes-8.11.3]] -== {es} version 8.11.3 - -coming[8.11.3] - -Also see <>. - -[[bug-8.11.3]] -[float] -=== Bug fixes - -Application:: -* Use latest version of entsearch ingestion pipeline {es-pull}103087[#103087] - -ES|QL:: -* Allow match field in enrich fields {es-pull}102734[#102734] -* Collect warnings in compute service {es-pull}103031[#103031] (issues: {es-issue}100163[#100163], {es-issue}103028[#103028], {es-issue}102871[#102871], {es-issue}102982[#102982]) - -ILM+SLM:: -* [ILM] More resilient when a policy is added to searchable snapshot {es-pull}102741[#102741] (issue: {es-issue}101958[#101958]) - -Mapping:: -* Ensure `dynamicMapping` updates are handled in insertion order {es-pull}103047[#103047] - -Transform:: -* Ensure transform `_schedule_now` API only triggers the expected transform task {es-pull}102958[#102958] (issue: {es-issue}102956[#102956]) - - diff --git a/docs/reference/release-notes/8.11.4.asciidoc b/docs/reference/release-notes/8.11.4.asciidoc deleted file mode 100644 index 0fd57c97b1a89..0000000000000 --- a/docs/reference/release-notes/8.11.4.asciidoc +++ /dev/null @@ -1,31 +0,0 @@ -[[release-notes-8.11.4]] -== {es} version 8.11.4 - -Also see <>. - -[[bug-8.11.4]] -[float] -=== Bug fixes - -EQL:: -* Fix NPE on missing event queries {es-pull}103611[#103611] (issue: {es-issue}103608[#103608]) - -ES|QL:: -* Fix now in millis for ESQL search contexts {es-pull}103474[#103474] (issue: {es-issue}103455[#103455]) -* Fix the transport version of `PlanStreamOutput` {es-pull}103758[#103758] -* `AsyncOperator#isFinished` must never return true on failure {es-pull}104029[#104029] - -Infra/Scripting:: -* Wrap painless explain error {es-pull}103151[#103151] (issue: {es-issue}103018[#103018]) - -Mapping:: -* Revert change {es-pull}103865[#103865] - -Snapshot/Restore:: -* Decref `SharedBytes.IO` after read is done not before {es-pull}102848[#102848] -* Restore `SharedBytes.IO` refcounting on reads & writes {es-pull}102843[#102843] - -Watcher:: -* Fix: Watcher REST API `GET /_watcher/settings` now includes product header {es-pull}103003[#103003] (issue: {es-issue}102928[#102928]) - - diff --git a/docs/reference/release-notes/8.12.0.asciidoc b/docs/reference/release-notes/8.12.0.asciidoc deleted file mode 100644 index bd0ae032ef0b9..0000000000000 --- a/docs/reference/release-notes/8.12.0.asciidoc +++ /dev/null @@ -1,435 +0,0 @@ -[[release-notes-8.12.0]] -== {es} version 8.12.0 - -Also see <>. - -[[known-issues-8.12.0]] -[float] -=== Known issues - -* `int8_hnsw` vector index format may fail to merge segments and prevent from indexing documents (issue: {es-issue}104617[#104617]) -+ -When using `int8_hnsw` and the default `confidence_interval` (or any `confidence_interval` less than `1.0`) and when -there are deleted documents in the segments, quantiles may fail to build and prevent merging. -+ -This issue is fixed in 8.12.1. - -* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, -information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. -If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. -To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. This issue is fixed in 8.15.0. - -[[breaking-8.12.0]] -[float] -=== Breaking changes -There are no breaking changes in 8.12 - -[[notable-8.12.0]] -[float] -=== Notable changes -There are notable changes in 8.12 that you need to be aware of but that we do not consider breaking, items that we may consider as notable changes are - -* Changes to features that are in Technical Preview. -* Changes to log formats. -* Changes to non-public APIs. -* Behaviour changes that repair critical bugs. - -Authorization:: -* Fixed JWT principal from claims {es-pull}101333[#101333] - -ES|QL:: -* [ES|QL] pow function always returns double {es-pull}102183[#102183] (issue: {es-issue}99055[#99055]) - -Infra/Plugins:: -* Remove Plugin.createComponents method in favour of overload with a PluginServices object {es-pull}101457[#101457] - -[[bug-8.12.0]] -[float] -=== Bug fixes - -Aggregations:: -* Adjust Histogram's bucket accounting to be iteratively {es-pull}102172[#102172] -* Aggs error codes part 1 {es-pull}99963[#99963] -* Skip global ordinals loading if query does not match after rewrite {es-pull}102844[#102844] -* Trigger parent circuit breaker when building scorers in filters aggregation {es-pull}102511[#102511] -* Unwrap `ExecutionException` when loading from cache in `AbstractIndexOrdinalsFieldData` {es-pull}102476[#102476] - -Application:: -* [Connector API] Fix bug with nullable tooltip field in parser {es-pull}103427[#103427] -* [Connectors API] Fix `ClassCastException` when creating a new sync job {es-pull}103508[#103508] -* [Connectors API] Fix bug with missing TEXT `DisplayType` enum {es-pull}103430[#103430] -* [Connectors API] Handle nullable fields correctly in the `ConnectorSyncJob` parser {es-pull}103183[#103183] -* [Profiling] Query in parallel only if beneficial {es-pull}103061[#103061] -* [Search Applications] Return 400 response when template rendering produces invalid JSON {es-pull}101474[#101474] - -Authentication:: -* Fall through malformed JWTs to subsequent realms in the chain {es-pull}101660[#101660] (issue: {es-issue}101367[#101367]) - -Authorization:: -* Fix cache invalidation on privilege modification {es-pull}102193[#102193] - -Data streams:: -* Use dataset size instead of on-disk size for data stream stats {es-pull}103342[#103342] - -Distributed:: -* Active shards message corrected for search shards {es-pull}102808[#102808] (issue: {es-issue}101896[#101896]) -* Dispatch `ClusterStateAction#buildResponse` to executor {es-pull}103435[#103435] -* Fix listeners in `SharedBlobCacheService.readMultiRegions` {es-pull}101727[#101727] - -Downsampling:: -* Copy counter field properties to downsampled index {es-pull}103580[#103580] (issue: {es-issue}103569[#103569]) -* Fix downsample api by returning a failure in case one or more downsample persistent tasks failed {es-pull}103615[#103615] - -EQL:: -* Cover head/tail commands edge cases and data types coverage {es-pull}101859[#101859] (issue: {es-issue}101724[#101724]) -* Fix NPE on missing event queries {es-pull}103611[#103611] (issue: {es-issue}103608[#103608]) -* Samples should check if the aggregations result is empty or null {es-pull}103574[#103574] - -ES|QL:: -* ESQL: Fix `to_degrees()` returning infinity {es-pull}103209[#103209] (issue: {es-issue}102987[#102987]) -* ESQL: Fix planning of MV_EXPAND with foldable expressions {es-pull}101385[#101385] (issue: {es-issue}101118[#101118]) -* ESQL: Fix rare bug with empty string {es-pull}102350[#102350] (issue: {es-issue}101969[#101969]) -* ESQL: Fix resolution of MV_EXPAND after KEEP * {es-pull}103339[#103339] (issue: {es-issue}103331[#103331]) -* ESQL: Fix single value query {es-pull}102317[#102317] (issue: {es-issue}102298[#102298]) -* ESQL: Improve local folding of aggregates {es-pull}103670[#103670] -* ESQL: Improve pushdown of certain filters {es-pull}103671[#103671] -* ESQL: Narrow catch in convert functions {es-pull}101788[#101788] (issue: {es-issue}100820[#100820]) -* ESQL: Update the use of some user-caused exceptions {es-pull}104046[#104046] -* ESQL: remove `time_zone` request parameter {es-pull}102767[#102767] (issue: {es-issue}102159[#102159]) -* ES|QL: Fix NPE on single value detection {es-pull}103150[#103150] (issue: {es-issue}103141[#103141]) -* ES|QL: Improve resolution error management in `mv_expand` {es-pull}102967[#102967] (issue: {es-issue}102964[#102964]) -* Fix layout for MV_EXPAND {es-pull}102916[#102916] (issue: {es-issue}102912[#102912]) -* Fix now in millis for ESQL search contexts {es-pull}103474[#103474] (issue: {es-issue}103455[#103455]) -* Fix planning of duplicate aggs {es-pull}102165[#102165] (issue: {es-issue}102083[#102083]) -* Fix the transport version of `PlanStreamOutput` {es-pull}103758[#103758] -* `AsyncOperator#isFinished` must never return true on failure {es-pull}104029[#104029] - -Engine:: -* Fix `lastUnsafeSegmentGenerationForGets` for realtime get {es-pull}101700[#101700] - -Geo:: -* Fix geo tile bounding boxes to be consistent with arithmetic method {es-pull}100826[#100826] (issues: {es-issue}92611[#92611], {es-issue}95574[#95574]) - -ILM+SLM:: -* Collect data tiers usage stats more efficiently {es-pull}102140[#102140] (issue: {es-issue}100230[#100230]) - -Indices APIs:: -* Fix template simulate setting application ordering {es-pull}103024[#103024] (issue: {es-issue}103008[#103008]) - -Infra/Core:: -* Cache component versions {es-pull}103408[#103408] (issue: {es-issue}102103[#102103]) -* Fix metric gauge creation model {es-pull}100609[#100609] - -Infra/Node Lifecycle:: -* Wait for reroute before acking put-shutdown {es-pull}103251[#103251] - -Infra/Plugins:: -* Making classname optional in Transport protocol {es-pull}99702[#99702] (issue: {es-issue}98584[#98584]) - -Infra/Scripting:: -* Make IPAddress writeable {es-pull}101093[#101093] (issue: {es-issue}101082[#101082]) -* Wrap painless explain error {es-pull}103151[#103151] (issue: {es-issue}103018[#103018]) - -Infra/Settings:: -* Report full stack trace for non-state file settings transforms {es-pull}101346[#101346] - -Ingest Node:: -* Sending an index name to `DocumentParsingObserver` that is not ever null {es-pull}100862[#100862] - -License:: -* Error log when license verification fails locally {es-pull}102919[#102919] - -Machine Learning:: -* Catch exceptions during `pytorch_inference` startup {es-pull}103873[#103873] -* Exclude quantiles when fetching model snapshots where possible {es-pull}103530[#103530] -* Fix `frequent_item_sets` aggregation on empty index {es-pull}103116[#103116] (issue: {es-issue}103067[#103067]) -* If trained model download task is in progress, wait for it to finish before executing start trained model deployment {es-pull}102944[#102944] -* Persist data counts on job close before results index refresh {es-pull}101147[#101147] -* Preserve response headers in Datafeed preview {es-pull}103923[#103923] -* Prevent attempts to access non-existent node information during rebalancing {es-pull}103361[#103361] -* Prevent resource over-subscription in model allocation planner {es-pull}100392[#100392] -* Start a new trace context before loading a trained model {es-pull}103124[#103124] -* Wait for the model results on graceful shutdown {es-pull}103591[#103591] (issue: {es-issue}103414[#103414]) - -Mapping:: -* Revert change {es-pull}103865[#103865] - -Monitoring:: -* [Monitoring] Dont get cluster state until recovery {es-pull}100565[#100565] - -Network:: -* Ensure the correct `threadContext` for `RemoteClusterNodesAction` {es-pull}101050[#101050] - -Ranking:: -* Add an additional tiebreaker to RRF {es-pull}101847[#101847] (issue: {es-issue}101232[#101232]) - -Reindex:: -* Allow prefix index naming while reindexing from remote {es-pull}96968[#96968] (issue: {es-issue}89120[#89120]) - -Search:: -* Add JIT compiler excludes for `computeCommonPrefixLengthAndBuildHistogram` {es-pull}103112[#103112] -* Check that scripts produce correct json in render template action {es-pull}101518[#101518] (issue: {es-issue}101477[#101477]) -* Fix NPE & empty result handling in `CountOnlyQueryPhaseResultConsumer` {es-pull}103203[#103203] -* Fix format string in `OldLuceneVersions` {es-pull}103185[#103185] -* Handle timeout on standalone rewrite calls {es-pull}103546[#103546] -* Introduce Elasticsearch `PostingFormat` based on Lucene 90 positing format using PFOR {es-pull}103601[#103601] (issue: {es-issue}103002[#103002]) -* Restore inter-segment search concurrency with synthetic source is enabled {es-pull}103690[#103690] -* Support complex datemath expressions in index and index alias names {es-pull}100646[#100646] - -Snapshot/Restore:: -* Decref `SharedBytes.IO` after read is done not before {es-pull}102848[#102848] -* More consistent logging messages for snapshot deletion {es-pull}101024[#101024] -* Reroute on shard snapshot completion {es-pull}101585[#101585] (issue: {es-issue}101514[#101514]) -* Restore `SharedBytes.IO` refcounting on reads & writes {es-pull}102843[#102843] - -TSDB:: -* Throw when wrapping rate agg in `DeferableBucketAggregator` {es-pull}101032[#101032] - -Transform:: -* Add an assertion to the testTransformFeatureReset test case {es-pull}100287[#100287] -* Consider search context missing exceptions as recoverable {es-pull}102602[#102602] -* Consider task cancelled exceptions as recoverable {es-pull}100828[#100828] -* Fix NPE that is thrown by `_update` API {es-pull}104051[#104051] (issue: {es-issue}104048[#104048]) -* Log stacktrace together with log message in order to help debugging {es-pull}101607[#101607] -* Split comma-separated source index strings into separate indices {es-pull}102811[#102811] (issue: {es-issue}99564[#99564]) - -Vector Search:: -* Disallow vectors whose magnitudes will not fit in a float {es-pull}100519[#100519] - -Watcher:: -* Correctly logging watcher history write failures {es-pull}101802[#101802] -* Fix: Watcher REST API `GET /_watcher/settings` now includes product header {es-pull}103003[#103003] (issue: {es-issue}102928[#102928]) - -[[enhancement-8.12.0]] -[float] -=== Enhancements - -Aggregations:: -* Check the real memory circuit breaker when building global ordinals {es-pull}102462[#102462] -* Disable concurrency for sampler and diversified sampler {es-pull}102832[#102832] -* Disable parallelism for composite agg against high cardinality fields {es-pull}102644[#102644] -* Enable concurrency for multi terms agg {es-pull}102710[#102710] -* Enable concurrency for scripted metric agg {es-pull}102461[#102461] -* Enable inter-segment concurrency for terms aggs {es-pull}101390[#101390] -* Export circuit breaker trip count as a counter metric {es-pull}101423[#101423] -* Introduce fielddata cache ttl {es-pull}102682[#102682] -* Status codes for Aggregation errors, part 2 {es-pull}100368[#100368] -* Support keyed histograms {es-pull}101826[#101826] (issue: {es-issue}100242[#100242]) - -Allocation:: -* Add more desired balance stats {es-pull}102065[#102065] -* Add undesired shard count {es-pull}101426[#101426] -* Expose reconciliation metrics via APM {es-pull}102244[#102244] - -Application:: -* Calculate CO2 and emmission and costs {es-pull}101979[#101979] -* Consider duplicate stacktraces in custom index {es-pull}102292[#102292] -* Enable Universal Profiling as Enterprise feature {es-pull}100333[#100333] -* Include totals in flamegraph response {es-pull}101126[#101126] -* Retrieve stacktrace events from a custom index {es-pull}102020[#102020] -* [Profiling] Notify early about task cancellation {es-pull}102740[#102740] -* [Profiling] Report in status API if docs exist {es-pull}102735[#102735] - -Authentication:: -* Add ldap user metadata mappings for full name and email {es-pull}102925[#102925] -* Add manage_enrich cluster privilege to kibana_system role {es-pull}101682[#101682] - -Authorization:: -* Remove `auto_configure` privilege for profiling {es-pull}101026[#101026] -* Use `BulkRequest` to store Application Privileges {es-pull}102056[#102056] -* Use non-deprecated SAML callback URL in SAML smoketests {es-pull}99983[#99983] (issue: {es-issue}99986[#99986]) -* Use non-deprecated SAML callback URL in tests {es-pull}99983[#99983] (issue: {es-issue}99985[#99985]) - -CAT APIs:: -* Expose roles by default in cat allocation API {es-pull}101753[#101753] - -CRUD:: -* Cache resolved index for mgets {es-pull}101311[#101311] - -Data streams:: -* Introduce new endpoint to expose data stream lifecycle stats {es-pull}101845[#101845] -* Switch logs data streams to search all fields by default {es-pull}102456[#102456] (issue: {es-issue}99872[#99872]) - -Distributed:: -* Add support for configuring proxy scheme in S3 client settings and EC2 discovery plugin {es-pull}102495[#102495] (issue: {es-issue}101873[#101873]) -* Introduce a `StreamOutput` that counts how many bytes are written to the stream {es-pull}102906[#102906] -* Push s3 requests count via metrics API {es-pull}100383[#100383] -* Record operation purpose for s3 stats collection {es-pull}100236[#100236] - -EQL:: -* Add error logging for *QL {es-pull}101057[#101057] -* Use the eql query filter for the open-pit request {es-pull}103212[#103212] - -ES|QL:: -* ESQL: Add `profile` option {es-pull}102713[#102713] -* ESQL: Alias duplicated aggregations in a stats {es-pull}100642[#100642] (issue: {es-issue}100544[#100544]) -* ESQL: Load more than one field at once {es-pull}102192[#102192] -* ESQL: Load stored fields sequentially {es-pull}102727[#102727] -* ESQL: Load text field from parent keyword field {es-pull}102490[#102490] (issue: {es-issue}102473[#102473]) -* ESQL: Make blocks ref counted {es-pull}100408[#100408] -* ESQL: Make fieldcaps calls lighter {es-pull}102510[#102510] (issues: {es-issue}101763[#101763], {es-issue}102393[#102393]) -* ESQL: More tracking in `BlockHash` impls {es-pull}101488[#101488] -* ESQL: New telemetry commands {es-pull}102937[#102937] -* ESQL: Share constant null Blocks {es-pull}102673[#102673] -* ESQL: Short circuit loading empty doc values {es-pull}102434[#102434] -* ESQL: Support the `_source` metadata field {es-pull}102391[#102391] -* ESQL: Track blocks emitted from lucene {es-pull}101396[#101396] -* ESQL: Track memory from values loaded from lucene {es-pull}101383[#101383] -* Fast path for reading single doc with ordinals {es-pull}102902[#102902] -* Introduce local block factory {es-pull}102901[#102901] -* Load different way {es-pull}101235[#101235] -* Track ESQL enrich memory {es-pull}102184[#102184] -* Track blocks in `AsyncOperator` {es-pull}102188[#102188] -* Track blocks of intermediate state of aggs {es-pull}102562[#102562] -* Track blocks when hashing single multi-valued field {es-pull}102612[#102612] -* Track pages in ESQL enrich request/response {es-pull}102190[#102190] - -Engine:: -* Add static node settings to set default values for max merged segment sizes {es-pull}102208[#102208] - -Geo:: -* Add runtime field of type `geo_shape` {es-pull}100492[#100492] (issue: {es-issue}61299[#61299]) - -Health:: -* Add message field to `HealthPeriodicLogger` and `S3RequestRetryStats` {es-pull}101989[#101989] -* Add non-green indicator names to `HealthPeriodicLogger` message {es-pull}102245[#102245] - -ILM+SLM:: -* Health Report API should not return RED for unassigned cold/frozen shards when data is available {es-pull}100776[#100776] -* Switch fleet's built-in ILM policies to use .actions.rollover.max_primary_shard_size {es-pull}99984[#99984] (issue: {es-issue}99983[#99983]) - -Indices APIs:: -* Add executed pipelines to bulk api response {es-pull}100031[#100031] -* Add support for marking component templates as deprecated {es-pull}101148[#101148] (issue: {es-issue}100992[#100992]) -* Allowing non-dynamic index settings to be updated by automatically unassigning shards {es-pull}101723[#101723] -* Rename component templates and pipelines according to the new naming conventions {es-pull}99975[#99975] -* Run `TransportGetAliasesAction` on local node {es-pull}101815[#101815] - -Infra/CLI:: -* Set `ActiveProcessorCount` when `node.processors` is set {es-pull}101846[#101846] - -Infra/Core:: -* Add apm api for asynchronous counters (always increasing) {es-pull}102598[#102598] -* Log errors in `RestResponse` regardless of `error_trace` parameter {es-pull}101066[#101066] (issue: {es-issue}100884[#100884]) - -Infra/Logging:: -* Add status code to `rest.suppressed` log output {es-pull}100990[#100990] - -Ingest Node:: -* Deprecate the unused `elasticsearch_version` field of enrich policy json {es-pull}103013[#103013] -* Optimize `MurmurHash3` {es-pull}101202[#101202] - -Machine Learning:: -* Accept a single or multiple inputs to `_inference` {es-pull}102075[#102075] -* Add basic telelemetry for the inference feature {es-pull}102877[#102877] -* Add internal inference action for ml models an services {es-pull}102731[#102731] -* Add prefix strings option to trained models {es-pull}102089[#102089] -* Estimate the memory required to deploy trained models more accurately {es-pull}98874[#98874] -* Improve stability of spike and dip detection for the change point aggregation {es-pull}102637[#102637] -* Include ML processor limits in `_ml/info` response {es-pull}101392[#101392] -* Read scores from downloaded vocabulary for XLM Roberta tokenizers {es-pull}101868[#101868] -* Support for GET all models and by task type in the `_inference` API {es-pull}102806[#102806] - -Mapping:: -* Improve analyzer reload log message {es-pull}102273[#102273] - -Monitoring:: -* Add memory utilization Kibana metric to the monitoring index templates {es-pull}102810[#102810] -* Added `beat.stats.libbeat.pipeline.queue.max_events` {es-pull}102570[#102570] - -Network:: -* Record more detailed HTTP stats {es-pull}99852[#99852] - -Search:: -* Add metrics to the shared blob cache {es-pull}101577[#101577] -* Add support for Serbian Language Analyzer {es-pull}100921[#100921] -* Add support for `index_filter` to open pit {es-pull}102388[#102388] (issue: {es-issue}99740[#99740]) -* Added metric for cache eviction of entries with non zero frequency {es-pull}100570[#100570] -* Disable inter-segment concurrency when sorting by field {es-pull}101535[#101535] -* Enable query phase parallelism within a single shard {es-pull}101230[#101230] (issue: {es-issue}80693[#80693]) -* Node stats as metrics {es-pull}102248[#102248] -* Optimize `_count` type API requests {es-pull}102888[#102888] - -Security:: -* Expose the `invalidation` field in Get/Query `ApiKey` APIs {es-pull}102472[#102472] -* Make `api_key.delete.interval` a dynamic setting {es-pull}102680[#102680] - -Snapshot/Restore:: -* Fail S3 repository analysis on partial reads {es-pull}102840[#102840] -* Parallelize stale index deletion {es-pull}100316[#100316] (issue: {es-issue}61513[#61513]) -* Repo analysis of uncontended register behaviour {es-pull}101185[#101185] -* Repo analysis: allow configuration of register ops {es-pull}102051[#102051] -* Repo analysis: verify empty register {es-pull}102048[#102048] - -Stats:: -* Introduce includeShardsStats in the stats request to indicate that we only fetch a summary {es-pull}100466[#100466] (issue: {es-issue}99744[#99744]) -* Set includeShardsStats = false in NodesStatsRequest where the caller does not use shards-level statistics {es-pull}100938[#100938] - -Store:: -* Add methods for adding generation listeners with primary term {es-pull}100899[#100899] -* Allow executing multiple periodic flushes while they are being made durable {es-pull}102571[#102571] -* Pass shard's primary term to Engine#addSegmentGenerationListener {es-pull}99752[#99752] - -Transform:: -* Implement exponential backoff for transform state persistence retrying {es-pull}102512[#102512] (issue: {es-issue}102528[#102528]) -* Make tasks that calculate checkpoints time out {es-pull}101055[#101055] -* Pass source query to `_field_caps` (as `index_filter`) when deducing destination index mappings for better performance {es-pull}102379[#102379] -* Pass transform source query as `index_filter` to `open_point_in_time` request {es-pull}102447[#102447] (issue: {es-issue}101049[#101049]) -* Skip shards that don't match the source query during checkpointing {es-pull}102138[#102138] - -Vector Search:: -* Add vector_operation_count in profile output for knn searches {es-pull}102032[#102032] -* Make cosine similarity faster by storing magnitude and normalizing vectors {es-pull}99445[#99445] - -[[feature-8.12.0]] -[float] -=== New features - -Application:: -* Enable Connectors API as technical preview {es-pull}102994[#102994] -* [Behavioral Analytics] Analytics collections use Data Stream Lifecycle (DSL) instead of Index Lifecycle Management (ILM) for data retention management. Behavioral analytics has traditionally used ILM to manage data retention. Starting with 8.12.0, this will change. Analytics collections created prior to 8.12.0 will continue to use their existing ILM policies, but new analytics collections will be managed using DSL. {es-pull}100033[#100033] - -Authentication:: -* Patterns support for allowed subjects by the JWT realm {es-pull}102426[#102426] - -Cluster Coordination:: -* Add a node feature join barrier. This prevents nodes from joining clusters that do not have all the features already present in the cluster. This ensures that once a features is supported by all the nodes in a cluster, that feature will never then not be supported in the future. This is the corresponding functionality for the version join barrier, but for features - {es-pull}101609[#101609] - -Data streams:: -* Add ability to create a data stream failure store {es-pull}99134[#99134] - -ES|QL:: -* ESQL: emit warnings from single-value functions processing multi-values {es-pull}102417[#102417] (issue: {es-issue}98743[#98743]) -* GEO_POINT and CARTESIAN_POINT type support {es-pull}102177[#102177] - -Infra/Core:: -* Create new cluster state API for querying features present on a cluster {es-pull}100974[#100974] - -Ingest Node:: -* Adding a simulate ingest api {es-pull}101409[#101409] - -Security:: -* Allow granting API keys with JWT as the access_token {es-pull}101904[#101904] - -Vector Search:: -* Add byte quantization for float vectors in HNSW {es-pull}102093[#102093] -* Make knn search a query {es-pull}98916[#98916] - -[[regression-8.12.0]] -[float] -=== Regressions - -Infra/Core:: -* Revert non-semantic `NodeInfo` {es-pull}102636[#102636] - -[[upgrade-8.12.0]] -[float] -=== Upgrades - -Search:: -* Upgrade to Lucene 9.9.1 {es-pull}103549[#103549] - - diff --git a/docs/reference/release-notes/8.12.1.asciidoc b/docs/reference/release-notes/8.12.1.asciidoc deleted file mode 100644 index 8ebe5cbac3852..0000000000000 --- a/docs/reference/release-notes/8.12.1.asciidoc +++ /dev/null @@ -1,83 +0,0 @@ -[[release-notes-8.12.1]] -== {es} version 8.12.1 - -Also see <>. - -[[known-issues-8.12.1]] -[float] -=== Known issues -* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, -information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. -If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. -To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. This issue is fixed in 8.15.0. - -[[bug-8.12.1]] -[float] -=== Bug fixes - -Allocation:: -* Improve `CANNOT_REBALANCE_CAN_ALLOCATE` explanation {es-pull}104904[#104904] - -Application:: -* [Connector API] Fix bug in configuration validation parser {es-pull}104198[#104198] -* [Connector API] Fix bug when triggering a sync job via API {es-pull}104802[#104802] -* [Profiling] Query in parallel on content nodes {es-pull}104600[#104600] - -Data streams:: -* Data streams fix failure store delete {es-pull}104281[#104281] -* Fix _alias/ returning non-matching data streams {es-pull}104145[#104145] (issue: {es-issue}96589[#96589]) - -Downsampling:: -* Downsampling supports `date_histogram` with tz {es-pull}103511[#103511] (issue: {es-issue}101309[#101309]) - -ES|QL:: -* Avoid execute ESQL planning on refresh thread {es-pull}104591[#104591] -* ESQL: Allow grouping by null blocks {es-pull}104523[#104523] -* ESQL: Fix `SearchStats#count(String)` to count values not rows {es-pull}104891[#104891] (issue: {es-issue}104795[#104795]) -* Limit concurrent shards per node for ESQL {es-pull}104832[#104832] (issue: {es-issue}103666[#103666]) -* Reduce the number of Evals `ReplaceMissingFieldWithNull` creates {es-pull}104586[#104586] (issue: {es-issue}104583[#104583]) - -Infra/Resiliency:: -* Limit nesting depth in Exception XContent {es-pull}103741[#103741] - -Ingest Node:: -* Better handling of async processor failures {es-pull}104289[#104289] (issue: {es-issue}101921[#101921]) -* Ingest correctly handle upsert operations and drop processors together {es-pull}104585[#104585] (issue: {es-issue}36746[#36746]) - -Machine Learning:: -* Add retry logic for 500 and 503 errors for OpenAI {es-pull}103819[#103819] -* Avoid possible datafeed infinite loop with filtering aggregations {es-pull}104722[#104722] (issue: {es-issue}104699[#104699]) -* [LTR] `FieldValueExtrator` - Checking if fetched values is empty {es-pull}104314[#104314] - -Network:: -* Fix lost headers with chunked responses {es-pull}104808[#104808] - -Search:: -* Don't throw error for remote shards that open PIT filtered out {es-pull}104288[#104288] (issue: {es-issue}102596[#102596]) - -Snapshot/Restore:: -* Fix deleting index during snapshot finalization {es-pull}103817[#103817] (issue: {es-issue}101029[#101029]) - -TSDB:: -* Fix `routing_path` when template has multiple `path_match` and multi-fields {es-pull}104418[#104418] (issue: {es-issue}104400[#104400]) - -Transform:: -* Fix bug when `latest` transform is used together with `from` parameter {es-pull}104606[#104606] (issue: {es-issue}104543[#104543]) - -[[deprecation-8.12.1]] -[float] -=== Deprecations - -Machine Learning:: -* Deprecate machine learning on Intel macOS {es-pull}104087[#104087] - -[[upgrade-8.12.1]] -[float] -=== Upgrades - -Search:: -* [8.12.1] Upgrade to Lucene 9.9.2 {es-pull}104761[#104761] (issue: {es-issue}104617[#104617]) - - diff --git a/docs/reference/release-notes/8.12.2.asciidoc b/docs/reference/release-notes/8.12.2.asciidoc deleted file mode 100644 index 44202ee8226eb..0000000000000 --- a/docs/reference/release-notes/8.12.2.asciidoc +++ /dev/null @@ -1,68 +0,0 @@ -[[release-notes-8.12.2]] -== {es} version 8.12.2 - -Also see <>. - -[[known-issues-8.12.2]] -[float] -=== Known issues -* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, -information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. -If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. -To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. This issue is fixed in 8.15.0. - -[[bug-8.12.2]] -[float] -=== Bug fixes - -Application:: -* Fix bug in `rule_query` where `text_expansion` errored because it was not rewritten {es-pull}105365[#105365] -* [Connectors API] Fix bug with crawler configuration parsing and `sync_now` flag {es-pull}105024[#105024] - -Authentication:: -* Validate settings before reloading JWT shared secret {es-pull}105070[#105070] - -Downsampling:: -* Downsampling better handle if source index isn't allocated and fix bug in retrieving last processed tsid {es-pull}105228[#105228] - -ES|QL:: -* ESQL: Push CIDR_MATCH to Lucene if possible {es-pull}105061[#105061] (issue: {es-issue}105042[#105042]) -* ES|QL: Fix exception handling on `date_parse` with wrong date pattern {es-pull}105048[#105048] (issue: {es-issue}104124[#104124]) - -Indices APIs:: -* Always show `composed_of` field for composable index templates {es-pull}105315[#105315] (issue: {es-issue}104627[#104627]) - -Ingest Node:: -* Backport stable `ThreadPool` constructor from `LogstashInternalBridge` {es-pull}105165[#105165] -* Harden index mapping parameter check in enrich runner {es-pull}105096[#105096] - -Machine Learning:: -* Fix handling of `ml.config_version` node attribute for nodes with machine learning disabled {es-pull}105066[#105066] -* Fix handling surrogate pairs in the XLM Roberta tokenizer {es-pull}105183[#105183] (issues: {es-issue}104626[#104626], {es-issue}104981[#104981]) -* Inference service should reject tasks during shutdown {es-pull}105213[#105213] - -Network:: -* Close `currentChunkedWrite` on client cancel {es-pull}105258[#105258] -* Fix leaked HTTP response sent after close {es-pull}105293[#105293] (issue: {es-issue}104651[#104651]) -* Fix race in HTTP response shutdown handling {es-pull}105306[#105306] - -Search:: -* Field-caps should read fields from up-to-dated shards {es-pull}105153[#105153] (issue: {es-issue}104809[#104809]) - -Snapshot/Restore:: -* Finalize all snapshots completed by shard snapshot updates {es-pull}105245[#105245] (issue: {es-issue}104939[#104939]) - -Transform:: -* Do not log warning when triggering an `ABORTING` transform {es-pull}105234[#105234] (issue: {es-issue}105233[#105233]) -* Make `_reset` action stop transforms without force first {es-pull}104870[#104870] (issues: {es-issue}100596[#100596], {es-issue}104825[#104825]) - -[[enhancement-8.12.2]] -[float] -=== Enhancements - -IdentityProvider:: -* Include user's privileges actions in IdP plugin `_has_privileges` request {es-pull}104026[#104026] - - diff --git a/docs/reference/release-notes/8.13.0.asciidoc b/docs/reference/release-notes/8.13.0.asciidoc deleted file mode 100644 index 75e2341f33766..0000000000000 --- a/docs/reference/release-notes/8.13.0.asciidoc +++ /dev/null @@ -1,472 +0,0 @@ -[[release-notes-8.13.0]] -== {es} version 8.13.0 - -Also see <>. - -[[known-issues-8.13.0]] -[float] -=== Known issues - -* Searches involving nodes upgraded to 8.13.0 and a coordinator node that is running on version - 8.12 or earlier can produce duplicate buckets when running `date_histogram` or `histogram` - aggregations. This can happen during a rolling upgrade to 8.13 or while running cross-cluster - searches. (issue: {es-issue}108181[#108181]). - -* Due to a bug in the bundled JDK 22 nodes might crash abruptly under high memory pressure. - We recommend <> asap to mitigate the issue. - -* Nodes upgraded to 8.13.0 fail to load downsampling persistent tasks. This prevents them from joining the cluster, blocking its upgrade (issue: {es-issue}106880[#106880]) -+ -This affects clusters running version 8.10 or later, with an active downsampling -https://www.elastic.co/guide/en/elasticsearch/reference/current/downsampling-ilm.html[configuration] -or a configuration that was activated at some point since upgrading to version 8.10 or later. - -* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, -information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. -If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. -To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. This issue is fixed in 8.15.0. - -* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. -In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` -process to reclaim memory. This can cause inference requests to fail. -Elasticsearch will automatically restart the `pytorch_inference` process -after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) - -[[breaking-8.13.0]] -[float] -=== Breaking changes - -ES|QL:: -* ESQL: Grammar - FROM METADATA no longer requires [] {es-pull}105221[#105221] -* ES|QL: remove PROJECT keyword from the grammar {es-pull}105064[#105064] -* [ESQL] Remove is_nan, is_finite, and `is_infinite` {es-pull}104091[#104091] - -TSDB:: -* Change `index.look_ahead_time` index setting's default value from 2 hours to 30 minutes. {es-pull}103898[#103898] -* Lower the `look_ahead_time` index setting's max value from 7 days to 2 hours. {es-pull}103434[#103434] - -[[bug-8.13.0]] -[float] -=== Bug fixes - -Aggregations:: -* Disable parallel collection for terms aggregation with `min_doc_count` equals to 0 {es-pull}106156[#106156] -* `GlobalOrdCardinalityAggregator` should use `HyperLogLogPlusPlus` instead of `HyperLogLogPlusPlusSparse` {es-pull}105546[#105546] - -Allocation:: -* Fix disk computation when initializing new shards {es-pull}102879[#102879] -* Fix disk computation when initializing unassigned shards in desired balance computation {es-pull}102207[#102207] - -Application:: -* Fix Search Applications bug where deleting an alias before deleting an application intermittently caused errors {es-pull}106329[#106329] -* Use search to determine if cluster contains data {es-pull}103920[#103920] -* [Connector API] Bugfix: support list type in filtering advenced snippet value {es-pull}105633[#105633] -* [Connector API] Fix default ordering in `SyncJob` list endpoint {es-pull}105945[#105945] -* [Connector API] Fix serialisation of script params in connector index service {es-pull}106060[#106060] - -Authentication:: -* Execute SAML authentication on the generic threadpool {es-pull}105232[#105232] (issue: {es-issue}104962[#104962]) - -Authorization:: -* Adjust interception of requests for specific shard IDs {es-pull}101656[#101656] - -Client:: -* Validate settings in `ReloadSecureSettings` API {es-pull}103176[#103176] - -Data streams:: -* Apm-data: fix `@custom` component templates {es-pull}104182[#104182] -* Avoid false-positive matches on intermediate objects in `ecs@mappings` {es-pull}105440[#105440] (issue: {es-issue}102794[#102794]) -* Execute lazy rollover with an internal dedicated user #104732 {es-pull}104905[#104905] (issue: {es-issue}104732[#104732]) -* Fix write index resolution when an alias is pointing to a TSDS {es-pull}104440[#104440] (issue: {es-issue}104189[#104189]) -* x-pack/plugin/core: add `match_mapping_type` to `ecs@mappings` dynamic templates {es-pull}103035[#103035] - -Distributed:: -* Fix logger Strings.format calls {es-pull}104573[#104573] -* Request indexing memory pressure in APM node metrics publisher {es-pull}103520[#103520] - -ES|QL:: -* ESQL: Add single value checks on LIKE/RLIKE pushdown {es-pull}103807[#103807] (issue: {es-issue}103806[#103806]) -* ESQL: Correct out-of-range filter pushdowns {es-pull}99961[#99961] (issue: {es-issue}99960[#99960]) -* ESQL: Fix Analyzer to not interpret escaped * as a pattern {es-pull}105325[#105325] (issue: {es-issue}104955[#104955]) -* ESQL: Fix a bug loading unindexed text fields {es-pull}104553[#104553] -* ESQL: Fix bug in grammar that allowed spaces inside id pattern {es-pull}105476[#105476] (issue: {es-issue}105441[#105441]) -* ESQL: Fix replacement of nested expressions in aggs with multiple parameters {es-pull}104718[#104718] (issue: {es-issue}104706[#104706]) -* ESQL: Fix wrong attribute shadowing in pushdown rules {es-pull}105650[#105650] (issue: {es-issue}105434[#105434]) -* ESQL: Improve pushdown of certain filters {es-pull}103538[#103538] (issue: {es-issue}103536[#103536]) -* ESQL: allow `null` in date math {es-pull}103610[#103610] (issue: {es-issue}103085[#103085]) -* ESQL: make `cidr_match` foldable {es-pull}105403[#105403] (issue: {es-issue}105376[#105376]) -* ES|QL: Disable optimizations that rely on Expression.nullable() {es-pull}105691[#105691] -* ES|QL: Improve type validation in aggs for UNSIGNED_LONG better support for VERSION {es-pull}104911[#104911] (issue: {es-issue}102961[#102961]) -* ES|QL: better management of exact subfields for TEXT fields {es-pull}103510[#103510] (issue: {es-issue}99899[#99899]) -* Fix error on sorting unsortable `geo_point` and `cartesian_point` {es-pull}106351[#106351] (issue: {es-issue}106007[#106007]) -* For empty mappings use a `LocalRelation` {es-pull}105081[#105081] (issue: {es-issue}104809[#104809]) -* Resume driver when failing to fetch pages {es-pull}106392[#106392] (issue: {es-issue}106262[#106262]) -* Review KEEP logic to prevent duplicate column names {es-pull}103316[#103316] -* `ProjectOperator` should not retain references to released blocks {es-pull}105848[#105848] - -Engine:: -* Consider currently refreshing data in the memory usage of refresh {es-pull}104122[#104122] -* Release `TranslogSnapshot` buffer after iteration {es-pull}106398[#106398] (issue: {es-issue}106390[#106390]) - -Health:: -* Make Health API more resilient to multi-version clusters {es-pull}105789[#105789] (issue: {es-issue}90183[#90183]) -* Stop the periodic health logger when es is stopping {es-pull}105272[#105272] - -ILM+SLM:: -* Remove `hashCode` and `equals` from `OperationModeUpdateTask` {es-pull}104265[#104265] (issue: {es-issue}100871[#100871]) -* [ILM] Delete step deletes data stream with only one index {es-pull}105772[#105772] - -Indices APIs:: -* Fix `require_alias` implicit true value on presence {es-pull}104099[#104099] (issue: {es-issue}103945[#103945]) - -Infra/CLI:: -* Fix server cli to always pass through exit code {es-pull}104943[#104943] - -Infra/Core:: -* Do not enable APM agent 'instrument', it's not required for manual tracing {es-pull}105055[#105055] -* Fix bogus assertion tripped by force-executed tasks {es-pull}104581[#104581] (issue: {es-issue}104580[#104580]) -* Metrics: Allow `AsyncCounters` to switch providers {es-pull}103025[#103025] -* Metrics: Handle null observations in observers {es-pull}103091[#103091] - -Infra/Node Lifecycle:: -* Close rather than stop `HttpServerTransport` on shutdown {es-pull}102759[#102759] (issue: {es-issue}102501[#102501]) - -Ingest Node:: -* Add stable `ThreadPool` constructor to `LogstashInternalBridge` {es-pull}105163[#105163] -* Adding `executedPipelines` to the `IngestDocument` copy constructor {es-pull}105427[#105427] -* Revert "x-pack/plugin/apm-data: download geoip DB on pipeline creation" {es-pull}104505[#104505] -* X-pack/plugin/apm-data: fix `@custom` pipeline support {es-pull}104113[#104113] - -Machine Learning:: -* Allow GET inference models by user a with read only permission {es-pull}105346[#105346] -* Avoid computing `currentInferenceProcessors` on every cluster state {es-pull}106057[#106057] -* Catch all the potential exceptions in the ingest processor code {es-pull}105391[#105391] -* Changed system auditor to use levels {es-pull}105429[#105429] -* During ML maintenance, reset jobs in the reset state without a corresponding task {es-pull}106062[#106062] -* Fix `categorize_text` aggregation nested under empty buckets {es-pull}105987[#105987] (issue: {es-issue}105836[#105836]) -* Fix resetting a job if the original reset task no longer exists. {es-pull}106020[#106020] -* Retry updates to model snapshot ID on job config {es-pull}104077[#104077] -* The OpenAI model parameter should be in service settings not task settings. Move the configuration field to service settings {es-pull}105458[#105458] -* Undeploy elser when inference model deleted {es-pull}104230[#104230] - -Mapping:: -* Fix parsing of flattened fields within subobjects: false {es-pull}105373[#105373] - -Network:: -* Fix use-after-free at event-loop shutdown {es-pull}105486[#105486] - -Search:: -* Correct profiled rewrite time for knn with a pre-filter {es-pull}104150[#104150] -* Force execution of `SearchService.Reaper` {es-pull}106544[#106544] (issue: {es-issue}106543[#106543]) -* Move `TransportTermsEnumAction` coordination off transport threads {es-pull}104408[#104408] -* Remove `SearchException` usages without a proper status code {es-pull}105150[#105150] -* Require the name field for `inner_hits` for collapse {es-pull}104666[#104666] -* add validation on _id field when upsert new doc {es-pull}103399[#103399] (issue: {es-issue}102981[#102981]) - -Security:: -* Revert "Validate settings in `ReloadSecureSettings` API" {es-pull}103310[#103310] - -Snapshot/Restore:: -* Do not record s3 http request time when it is not available {es-pull}105103[#105103] -* `URLRepository` should not block shutdown {es-pull}105588[#105588] - -TLS:: -* Respect --pass option in certutil csr mode {es-pull}106105[#106105] - -Transform:: -* Fix `_reset` API when called with `force=true` on a failed transform {es-pull}106574[#106574] (issue: {es-issue}106573[#106573]) -* Fix a bug where destination index aliases are not set up for an unattended transform {es-pull}105499[#105499] -* Remove duplicate checkpoint audits {es-pull}105164[#105164] (issue: {es-issue}105106[#105106]) -* Return results in order {es-pull}105089[#105089] (issue: {es-issue}104847[#104847]) -* Use deduced mappings for determining proper fields' format even if `deduce_mappings==false` {es-pull}103682[#103682] (issue: {es-issue}103115[#103115]) - -Vector Search:: -* Fix bug when nested knn pre-filter might match nested docs {es-pull}105994[#105994] - -Watcher:: -* Handling exceptions on watcher reload {es-pull}105442[#105442] (issue: {es-issue}69842[#69842]) - -[[deprecation-8.13.0]] -[float] -=== Deprecations - -Distributed:: -* `DesiredNode:` deprecate `node_version` field and make it optional (unused) in current parser {es-pull}104209[#104209] - -Infra/Core:: -* Deprecate `client.type` {es-pull}104574[#104574] - -[[enhancement-8.13.0]] -[float] -=== Enhancements - -Aggregations:: -* Add index mapping parameter for `counted_keyword` {es-pull}103646[#103646] -* Introduce an `AggregatorReducer` to reduce the footprint of aggregations in the coordinating node {es-pull}105207[#105207] -* Release resources in `BestBucketsDeferringCollector` earlier {es-pull}104893[#104893] -* Support sampling in `counted_terms` aggregation {es-pull}103846[#103846] - -Allocation:: -* Account for reserved disk size {es-pull}103903[#103903] -* Derive expected replica size from primary {es-pull}102078[#102078] - -Application:: -* Add serverless scopes for Connector APIs {es-pull}104063[#104063] -* [Connector API] Change required privileges to indices:data/read(write) {es-pull}105289[#105289] -* [Connector API] Implement update `index_name` action {es-pull}104648[#104648] -* [Connector API] Support filtering by name, index name in list action {es-pull}105131[#105131] -* [Connector API] Support filtering connectors by service type and a query {es-pull}105178[#105178] -* [Connector API] Support updating configuration values only {es-pull}105249[#105249] -* [Connectors API] Add new field `api_key_secret_id` to Connector {es-pull}104982[#104982] -* [Connectors API] Implement connector status update action {es-pull}104750[#104750] -* [Connectors API] Implement update native action endpoint {es-pull}104654[#104654] -* [Connectors API] Implement update service type action {es-pull}104643[#104643] -* [Connectors API] Relax strict response parsing for get/list operations {es-pull}104909[#104909] -* [Profiling] Extract properties faster from source {es-pull}104356[#104356] -* [Profiling] Mark all templates as managed {es-pull}103783[#103783] -* [Profiling] Speed up processing of stacktraces {es-pull}104674[#104674] -* [Profiling] Support downsampling of generic events {es-pull}104730[#104730] -* [Profiling] Use shard request cache consistently {es-pull}103643[#103643] - -Authentication:: -* Expose API key authentication metrics {es-pull}103178[#103178] -* Expose realms authentication metrics {es-pull}104200[#104200] -* Expose service account authentication metrics {es-pull}104043[#104043] -* Expose token authentication metrics {es-pull}104142[#104142] -* Hot-reloadable LDAP bind password {es-pull}104320[#104320] -* Support of `match` for the Query API Key API {es-pull}104594[#104594] - -Authorization:: -* [Security Solution] Allow write permission for `kibana_system` role on endpoint response index {es-pull}103555[#103555] - -CRUD:: -* Avoid wrapping searchers multiple times in mget {es-pull}104227[#104227] (issue: {es-issue}85069[#85069]) - -Client:: -* Add rest spec for Query User API {es-pull}104529[#104529] - -Cluster Coordination:: -* Add troubleshooting docs link to `PeerFinder` logs {es-pull}104787[#104787] -* Report current master in `PeerFinder` {es-pull}104396[#104396] - -Data streams:: -* Introduce lazy rollover for mapping updates in data streams {es-pull}103309[#103309] (issue: {es-issue}89346[#89346]) -* Use new `ignore_dynamic_beyond_limit` in logs and metric data streams {es-pull}105180[#105180] -* X-pack/plugin/apm-data: add dynamic setting for enabling template registry {es-pull}104386[#104386] (issue: {es-issue}104385[#104385]) -* X-pack/plugin/core: rename `double_metrics` template {es-pull}103033[#103033] -* x-pack/plugin/apm-data: Add a new field transaction.profiler_stack_trace_ids to traces-apm@mappings.yaml {es-pull}105223[#105223] -* x-pack/plugin/apm-data: Map some APM fields as flattened and fix error.grouping_name script {es-pull}103032[#103032] -* x-pack/plugin/core: make automatic rollovers lazy {es-pull}105273[#105273] (issue: {es-issue}104083[#104083]) - -Discovery-Plugins:: -* Set read timeout for fetching IMDSv2 token {es-pull}104407[#104407] (issue: {es-issue}104244[#104244]) - -Downsampling:: -* Support patch transport version from 8.12 {es-pull}104406[#104406] - -ES|QL:: -* Add ES|QL async delete API {es-pull}103628[#103628] -* Avoid humongous blocks {es-pull}103340[#103340] -* ESQL: Add TO_UPPER and TO_LOWER functions {es-pull}104309[#104309] -* ESQL: Add option to drop null fields {es-pull}102428[#102428] -* ESQL: Add plan consistency verification after each optimizer {es-pull}105371[#105371] -* ESQL: Check field exists before load from `_source` {es-pull}103632[#103632] -* ESQL: Delay finding field load infrastructure {es-pull}103821[#103821] -* ESQL: Expand shallow copy with vecs {es-pull}103681[#103681] (issue: {es-issue}100528[#100528]) -* ESQL: Extend STATS command to support aggregate expressions {es-pull}104958[#104958] -* ESQL: Infer not null for aggregated fields {es-pull}103673[#103673] (issue: {es-issue}102787[#102787]) -* ESQL: Nested expressions inside stats command {es-pull}104387[#104387] (issue: {es-issue}99828[#99828]) -* ESQL: Pre-allocate rows in TopNOperator {es-pull}104796[#104796] -* ESQL: Referencing expressions that contain backticks requires <>. {es-pull}100740[#100740] (issue: {es-issue}100312[#100312]) -* ESQL: Simpify IS NULL/IS NOT NULL evaluation {es-pull}103099[#103099] (issue: {es-issue}103097[#103097]) -* ESQL: Speed up reading many nulls {es-pull}105088[#105088] -* ESQL: Support loading shapes from source into WKB blocks {es-pull}104269[#104269] -* ESQL: Track the rest of `DocVector` {es-pull}103727[#103727] -* ESQL: `MV_FIRST` and `MV_LAST` {es-pull}103928[#103928] -* ESQL: add `date_diff` function {es-pull}104118[#104118] (issue: {es-issue}101942[#101942]) -* ESQL: push down "[text_field] is not null" {es-pull}105593[#105593] -* ES|QL Async Query API {es-pull}103398[#103398] -* Prepare enrich plan to support multi clusters {es-pull}104355[#104355] -* Reading points from source to reduce precision loss {es-pull}103698[#103698] -* Remove deprecated Block APIs {es-pull}103592[#103592] -* Reserve bytes before serializing page {es-pull}105269[#105269] -* Support ST_CENTROID over spatial points {es-pull}104218[#104218] (issue: {es-issue}104656[#104656]) -* Support cross clusters query in ESQL {es-pull}101640[#101640] -* Support enrich ANY mode in cross clusters query {es-pull}104840[#104840] -* Support enrich coordinator mode {es-pull}104936[#104936] -* Support enrich remote mode {es-pull}104993[#104993] - -Geo:: -* Add support for Well Known Binary (WKB) in the fields API for spatial fields {es-pull}103461[#103461] -* Add the possibility to transform WKT to WKB directly {es-pull}104030[#104030] - -Health:: -* Add APM metrics to `HealthPeriodicLogger` {es-pull}102765[#102765] -* Extend `repository_integrity` health indicator for unknown and invalid repos {es-pull}104614[#104614] (issue: {es-issue}103784[#103784]) - -ILM+SLM:: -* Add "step":"ERROR" to ILM explain response for missing policy {es-pull}103720[#103720] (issue: {es-issue}99030[#99030]) -* Add default rollover conditions to ILM explain API response {es-pull}104721[#104721] (issue: {es-issue}103395[#103395]) -* ILM/SLM history policies forcemerge in hot and dsl configuration {es-pull}103190[#103190] - -Infra/CLI:: -* Add replay diagnostic dir to system jvm options {es-pull}103535[#103535] - -Infra/Circuit Breakers:: -* Lower G1 minimum full GC interval {es-pull}105259[#105259] - -Infra/Core:: -* Adding threadpool metrics {es-pull}102371[#102371] -* ES - document observing with rejections {es-pull}104859[#104859] -* Thread pool metrics {es-pull}104500[#104500] - -Infra/Metrics:: -* Modify name of threadpool metric for rejected {es-pull}105015[#105015] - -Infra/Node Lifecycle:: -* Wait for async searches to finish when shutting down {es-pull}101487[#101487] - -Infra/Transport API:: -* Make `ParentTaskAssigningClient.getRemoteClusterClient` method also return `ParentTaskAssigningClient` {es-pull}100813[#100813] - -Ingest Node:: -* Adding `ActionRequestLazyBuilder` implementation of `RequestBuilder` {es-pull}104927[#104927] -* Adding a `RequestBuilder` interface {es-pull}104778[#104778] -* Adding a custom exception for problems with the graph of pipelines to be applied to a document {es-pull}105196[#105196] -* Improving the performance of the ingest simulate verbose API {es-pull}105265[#105265] -* Ingest geoip processor cache 'no results' from the database {es-pull}104092[#104092] -* Limiting the number of nested pipelines that can be executed {es-pull}105428[#105428] -* Modifying request builders {es-pull}104636[#104636] - -Java Low Level REST Client:: -* Set thread name used by REST client {es-pull}103160[#103160] - -Machine Learning:: -* Add optional pruning configuration (weighted terms scoring) to text expansion query {es-pull}102862[#102862] -* Add text_embedding inference service with multilingual-e5 and custom eland models {es-pull}104949[#104949] -* Add 3 automatic restarts for `pytorch_inference` processes that stop unexpectedly {es-pull}104433[#104433] -* Add support for Cohere inference service {es-pull}104559[#104559] -* Always test for spikes and dips as well as changes in the change point aggregation {es-pull}103922[#103922] -* Apply windowing and chunking to long documents {es-pull}104363[#104363] -* Automatically download the ELSER model when PUT in `_inference` {es-pull}104334[#104334] -* Better handling of number of allocations in pytorch_inference in the case that hardware_concurrency fails {ml-pull}2607[#2607] -* Change detection aggregation improvements {es-pull}102824[#102824] -* Conditionally send the dimensions field as part of the openai requests {es-pull}105299[#105299] (issue: {es-issue}105005[#105005]) -* Endpoint to find positions of Grok pattern matches {es-pull}104394[#104394] -* Ensure unique IDs between inference models and trained model deployments {es-pull}103996[#103996] -* Expose some ML metrics via APM {es-pull}102584[#102584] -* Make `task_type` optional in `_inference` APIs {es-pull}104483[#104483] -* Update `missingTrainedModel` message to include: you may need to create it {es-pull}104155[#104155] -* Upgrade MKL to version 2024.0 on Linux x86_64 {ml-pull}2619[#2619] -* Upgrade PyTorch to version 2.1.2. {ml-pull}2588[#2588] -* Upgrade zlib to version 1.2.13 on Windows {ml-pull}2588[#2588] -* Use Boost.JSON for JSON processing {ml-pull}2614[#2614] -* Validate inference model ids {es-pull}103669[#103669] - - -Mapping:: -* Add `index.mapping.total_fields.ignore_dynamic_beyond_limit` setting to ignore dynamic fields when field limit is reached {es-pull}96235[#96235] -* Make field limit more predictable {es-pull}102885[#102885] - -Network:: -* Prune unnecessary information from TransportNodesStatsAction.NodeStatsRequest {es-pull}102559[#102559] (issue: {es-issue}100878[#100878]) - -Percolator:: -* Return `matched_queries` in Percolator {es-pull}103084[#103084] (issue: {es-issue}10163[#10163]) - -Query Languages:: -* Introduce Alias.unwrap method {es-pull}104575[#104575] - -Search:: -* Dyamically adjust node metrics cache expire {es-pull}104460[#104460] -* Enhancement: Metrics for Search Took Times using Action Listeners {es-pull}104996[#104996] -* Field caps performance pt2 {es-pull}105941[#105941] -* Field-caps field has value lookup use map instead of looping array {es-pull}105770[#105770] -* Flag in `_field_caps` to return only fields with values in index {es-pull}103651[#103651] -* Include better output in profiling & `toString` for automaton based queries {es-pull}105468[#105468] -* Metrics for search latencies {es-pull}102557[#102557] -* Ref count search response bytes {es-pull}103763[#103763] (issue: {es-issue}102657[#102657]) -* Remove leniency in msearch parsing {es-pull}103232[#103232] -* Resolve Cluster API {es-pull}102726[#102726] -* Reuse number field mapper tests in other modules {es-pull}99142[#99142] (issue: {es-issue}92947[#92947]) -* S3 first byte latency metric {es-pull}102435[#102435] -* Update s3 latency metric to use micros {es-pull}103633[#103633] -* Upgrade to Lucene 9.10.0 {es-pull}105578[#105578] - -Security:: -* Add Query Users API {es-pull}104033[#104033] -* Add `ApiKey` expiration time to audit log {es-pull}103959[#103959] -* Add expiration time to update api key api {es-pull}103453[#103453] -* Add stricter validation for api key expiration time {es-pull}103973[#103973] -* Add support for the `simple_query_string` to the Query API Key API {es-pull}104132[#104132] -* Add support for the `type` parameter, for sorting, to the Query API Key API {es-pull}104625[#104625] -* Aggs support for Query API Key Information API {es-pull}104895[#104895] -* Hot-reloadable remote cluster credentials {es-pull}102798[#102798] - -Snapshot/Restore:: -* Add s3 `HeadObject` request to request stats {es-pull}105105[#105105] -* Expose `OperationPurpose` in S3 access logs using a https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html#LogFormatCustom[custom query-string parameter] {es-pull}105044[#105044] -* Fix blob cache race, decay, time dependency {es-pull}104784[#104784] -* Pause shard snapshots on graceful shutdown {es-pull}101717[#101717] -* Retry indefinitely for s3 indices blob read errors {es-pull}103300[#103300] - -Store:: -* List hidden shard stores by default {es-pull}103710[#103710] - -TLS:: -* 'elasticsearch-certutil cert' now verifies the issuing chain of the generated certificate {es-pull}103948[#103948] - -TSDB:: -* Improve storage efficiency for non-metric fields in TSDB {es-pull}99747[#99747] -* Introduce experimental pass-through field type {es-pull}103648[#103648] -* Nest pass-through objects within objects {es-pull}105062[#105062] -* Restrict usage of certain aggregations when in sort order execution is required {es-pull}104665[#104665] -* Small time series agg improvement {es-pull}106288[#106288] - -Transform:: -* Allow transforms to use PIT with remote clusters again {es-pull}105192[#105192] (issue: {es-issue}104518[#104518]) -* Transforms: Adding basic stats API param {es-pull}104878[#104878] - -Vector Search:: -* Add new int8_flat and flat vector index types {es-pull}104872[#104872] -* Add support for more than one `inner_hit` when searching nested vectors {es-pull}104006[#104006] -* Making `k` and `num_candidates` optional for knn search {es-pull}101209[#101209] (issue: {es-issue}97533[#97533]) - -[[feature-8.13.0]] -[float] -=== New features - -Data streams:: -* Add `require_data_stream` parameter to indexing requests to enforce indexing operations target a data stream {es-pull}101872[#101872] (issue: {es-issue}97032[#97032]) -* Redirect failed ingest node operations to a failure store when available {es-pull}103481[#103481] - -ES|QL:: -* ESQL: Introduce mode setting for ENRICH {es-pull}103949[#103949] -* ESQL: add =~ operator (case insensitive equality) {es-pull}103656[#103656] - -Health:: -* Create a DSL health indicator as part of the health API {es-pull}103130[#103130] - -Infra/Core:: -* Add gradle tasks and code to modify and access mappings between version ids and release versions {es-pull}103627[#103627] - -Mapping:: -* Add `unmatch_mapping_type`, and support array of types {es-pull}103171[#103171] (issues: {es-issue}102807[#102807], {es-issue}102795[#102795]) - -Search:: -* Added Duplicate Word Check Feature to Analysis Nori {es-pull}103325[#103325] (issue: {es-issue}103321[#103321]) -* [Synonyms] Mark Synonyms as GA {es-pull}103223[#103223] - -[[upgrade-8.13.0]] -[float] -=== Upgrades - -Query Languages:: -* Upgrade ANTLR4 to 4.13.1 {es-pull}105334[#105334] (issue: {es-issue}102953[#102953]) - -Search:: -* Upgrade to Lucene 9.9.0 {es-pull}102782[#102782] -* Upgrade to Lucene 9.9.1 {es-pull}103387[#103387] -* Upgrade to Lucene 9.9.2 {es-pull}104753[#104753] diff --git a/docs/reference/release-notes/8.13.1.asciidoc b/docs/reference/release-notes/8.13.1.asciidoc deleted file mode 100644 index c654af3dd5cc0..0000000000000 --- a/docs/reference/release-notes/8.13.1.asciidoc +++ /dev/null @@ -1,53 +0,0 @@ -[[release-notes-8.13.1]] -== {es} version 8.13.1 - -Also see <>. - -[[known-issues-8.13.1]] -[float] -=== Known issues -* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, -information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. -If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. -To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. This issue is fixed in 8.15.0. - -* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. -In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` -process to reclaim memory. This can cause inference requests to fail. -Elasticsearch will automatically restart the `pytorch_inference` process -after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) - -[[bug-8.13.1]] -[float] - -* Searches involving nodes upgraded to 8.13.0 and a coordinator node that is running on version - 8.12 or earlier can produce duplicate buckets when running `date_histogram` or `histogram` - aggregations. This can happen during a rolling upgrade to 8.13 or while running cross-cluster - searches. (issue: {es-issue}108181[#108181]). - -=== Bug fixes - -Aggregations:: -* Add test to exercise reduction of terms aggregation order by key {es-pull}106799[#106799] - -Downsampling:: -* Gate reading of optional string array for bwc {es-pull}106878[#106878] - -Machine Learning:: -* Fix Array out of bounds exception in the XLM Roberta tokenizer {es-pull}106655[#106655] - -Search:: -* Fix concurrency bug in `AbstractStringScriptFieldAutomatonQuery` {es-pull}106678[#106678] (issue: {es-issue}105911[#105911]) -* Fix the background set of significant terms aggregations in case the data is in different shards than the foreground set {es-pull}106564[#106564] - -Transform:: -* Fail checkpoint on missing clusters {es-pull}106793[#106793] (issues: {es-issue}104533[#104533], {es-issue}106790[#106790]) - -[[enhancement-8.13.1]] -[float] -=== Enhancements - -Transform:: -* Raise loglevel of events related to transform lifecycle from DEBUG to INFO {es-pull}106602[#106602] diff --git a/docs/reference/release-notes/8.13.2.asciidoc b/docs/reference/release-notes/8.13.2.asciidoc deleted file mode 100644 index f4540343ca9ea..0000000000000 --- a/docs/reference/release-notes/8.13.2.asciidoc +++ /dev/null @@ -1,54 +0,0 @@ -[[release-notes-8.13.2]] -== {es} version 8.13.2 - -Also see <>. - -[[known-issues-8.13.2]] -[float] -=== Known issues -* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, -information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. -If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. -To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. This issue is fixed in 8.15.0. - -* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. -In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` -process to reclaim memory. This can cause inference requests to fail. -Elasticsearch will automatically restart the `pytorch_inference` process -after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) - -[[bug-8.13.2]] -[float] - -* Searches involving nodes upgraded to 8.13.0 and a coordinator node that is running on version - 8.12 or earlier can produce duplicate buckets when running `date_histogram` or `histogram` - aggregations. This can happen during a rolling upgrade to 8.13 or while running cross-cluster - searches. (issue: {es-issue}108181[#108181]). - -=== Bug fixes - -Aggregations:: -* Address concurrency issue in top hits aggregation {es-pull}106990[#106990] - -Application:: -* [Connector API] Support numeric for configuration select option value type {es-pull}107059[#107059] - -Downsampling:: -* Fix a downsample persistent task assignment bug {es-pull}106247[#106247] -* Fix downsample action request serialization {es-pull}106920[#106920] - -ES|QL:: -* ESQL: Fix fully pruned aggregates {es-pull}106673[#106673] (issue: {es-issue}106427[#106427]) - -Packaging:: -* Downgrade JDK to JDK 21.0.2 {es-pull}107137[#107137] (issue: {es-issue}106987[#106987]) - -[[enhancement-8.13.2]] -[float] -=== Enhancements - -Security:: -* Query API Key Information API support for the `typed_keys` request parameter {es-pull}106873[#106873] (issue: {es-issue}106817[#106817]) -* Query API Keys support for both `aggs` and `aggregations` keywords {es-pull}107054[#107054] (issue: {es-issue}106839[#106839]) diff --git a/docs/reference/release-notes/8.13.3.asciidoc b/docs/reference/release-notes/8.13.3.asciidoc deleted file mode 100644 index f1bb4211f4676..0000000000000 --- a/docs/reference/release-notes/8.13.3.asciidoc +++ /dev/null @@ -1,60 +0,0 @@ -[[release-notes-8.13.3]] -== {es} version 8.13.3 - -Also see <>. - -[[breaking-8.13.3]] -[float] -=== Breaking changes - -SQL:: -* Limit how much space some string functions can use {es-pull}107333[#107333] - -[[known-issues-8.13.3]] -[float] -=== Known issues -* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, -information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. -If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. -To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. This issue is fixed in 8.15.0. - -* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. -In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` -process to reclaim memory. This can cause inference requests to fail. -Elasticsearch will automatically restart the `pytorch_inference` process -after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) - -[[bug-8.13.3]] -[float] -=== Bug fixes - -Data streams:: -* Avoid unintentionally clearing the `DataStream.rolloverOnWrite` flag {es-pull}107122[#107122] - -ES|QL:: -* ESQL: Fix bug when combining projections {es-pull}107131[#107131] (issue: {es-issue}107083[#107083]) -* ESQL: Fix missing refs due to pruning renamed grouping columns {es-pull}107328[#107328] (issues: {es-issue}107083[#107083], {es-issue}107166[#107166]) - -Indices APIs:: -* GET /_all should return hidden indices with visible aliases {es-pull}106975[#106975] - -Mapping:: -* Fix merging component templates with a mix of dotted and nested object mapper definitions {es-pull}106077[#106077] (issue: {es-issue}105482[#105482]) - -Network:: -* Handle exceptions thrown by HTTP header validation {es-pull}107355[#107355] (issue: {es-issue}107338[#107338]) - -Percolator:: -* Percolator named queries: rewrite for matched info {es-pull}107432[#107432] (issue: {es-issue}107176[#107176]) - -Search:: -* Fix `minimized_round_trips` in lookup runtime fields {es-pull}107785[#107785] - -[[enhancement-8.13.3]] -[float] -=== Enhancements - -ES|QL:: -* ESQL: Introduce language versioning to REST API {es-pull}106824[#106824] diff --git a/docs/reference/release-notes/8.13.4.asciidoc b/docs/reference/release-notes/8.13.4.asciidoc deleted file mode 100644 index 446aae048945b..0000000000000 --- a/docs/reference/release-notes/8.13.4.asciidoc +++ /dev/null @@ -1,36 +0,0 @@ -[[release-notes-8.13.4]] -== {es} version 8.13.4 - -Also see <>. - -[[known-issues-8.13.4]] -[float] -=== Known issues -* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, -information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. -If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. -To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. This issue is fixed in 8.15.0. - -* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. -In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` -process to reclaim memory. This can cause inference requests to fail. -Elasticsearch will automatically restart the `pytorch_inference` process -after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) - -[[bug-8.13.4]] -[float] -=== Bug fixes - -Aggregations:: -* Fix Bucket ordering for partial reduction in date histogram and histogram aggregation {es-pull}108184[#108184] (issue: {es-issue}108181[#108181]) - -ES|QL:: -* Fix `BlockHash` `DirectEncoder` {es-pull}108283[#108283] (issue: {es-issue}108268[#108268]) - -Snapshot/Restore:: -* Ensure necessary security context for s3 bulk deletions {es-pull}108280[#108280] (issue: {es-issue}108049[#108049]) - -TSDB:: -* Fix tsdb codec when doc-values spread in two blocks {es-pull}108276[#108276] diff --git a/docs/reference/release-notes/8.14.0.asciidoc b/docs/reference/release-notes/8.14.0.asciidoc deleted file mode 100644 index c2fee6ecaa07a..0000000000000 --- a/docs/reference/release-notes/8.14.0.asciidoc +++ /dev/null @@ -1,364 +0,0 @@ -[[release-notes-8.14.0]] -== {es} version 8.14.0 - -Also see <>. - -[[breaking-8.14.0]] -[float] -=== Breaking changes - -Security:: -* Prevent DLS/FLS if `replication` is assigned {es-pull}108600[#108600] -* Apply stricter Document Level Security (DLS) rules for the validate query API with the rewrite parameter {es-pull}105709[#105709] -* Apply stricter Document Level Security (DLS) rules for terms aggregations when min_doc_count is set to 0 {es-pull}105714[#105714] - -[[known-issues-8.14.0]] -[float] -=== Known issues -* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, -information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. -If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. -To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. This issue is fixed in 8.15.0. - -* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. -In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` -process to reclaim memory. This can cause inference requests to fail. -Elasticsearch will automatically restart the `pytorch_inference` process -after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) - -[[bug-8.14.0]] -[float] -=== Bug fixes - -Aggregations:: -* Cross check livedocs for terms aggs when index access control list is non-null {es-pull}105714[#105714] -* ESQL: Enable VALUES agg for datetime {es-pull}107016[#107016] -* Fix IOOBE in TTest aggregation when using filters {es-pull}109034[#109034] -* Validate stats formatting in standard `InternalStats` constructor {es-pull}107678[#107678] (issue: {es-issue}107671[#107671]) - -Application:: -* [Bugfix] Connector API - fix status serialisation issue in termquery {es-pull}108365[#108365] -* [Connector API] Fix bug with filtering validation toXContent {es-pull}107467[#107467] -* [Connector API] Fix bug with parsing *_doc_count nullable fields {es-pull}108854[#108854] -* [Connector API] Fix bug with with wrong target index for access control sync {es-pull}109097[#109097] - -Authorization:: -* Users with monitor privileges can access async_search/status endpoint even when setting keep_alive {es-pull}107383[#107383] - -CAT APIs:: -* Fix numeric sorts in `_cat/nodes` {es-pull}106189[#106189] (issue: {es-issue}48070[#48070]) - -CCR:: -* Add ?master_timeout query parameter to ccr apis {es-pull}105168[#105168] - -CRUD:: -* Fix `noop_update_total` is not being updated when using the `_bulk` {es-pull}105745[#105745] (issue: {es-issue}105742[#105742]) -* Use correct system index bulk executor {es-pull}106150[#106150] - -Cluster Coordination:: -* Fix support for infinite `?master_timeout` {es-pull}107050[#107050] - -Data streams:: -* Add non-indexed fields to ecs templates {es-pull}106714[#106714] -* Fix bulk NPE when retrying failure redirect after cluster block {es-pull}107598[#107598] -* Improve error message when rolling over DS alias {es-pull}106708[#106708] (issue: {es-issue}106137[#106137]) -* Only skip deleting a downsampled index if downsampling is in progress as part of DSL retention {es-pull}109020[#109020] - -Downsampling:: -* Fix downsample action request serialization {es-pull}106919[#106919] (issue: {es-issue}106917[#106917]) - -EQL:: -* Use #addWithoutBreaking when adding a negative number of bytes to the circuit breaker in `SequenceMatcher` {es-pull}107655[#107655] - -ES|QL:: -* ESQL: Allow reusing BUCKET grouping expressions in aggs {es-pull}107578[#107578] -* ESQL: Disable quoting in FROM command {es-pull}108431[#108431] -* ESQL: Fix MV_DEDUPE when using data from an index {es-pull}107577[#107577] (issue: {es-issue}104745[#104745]) -* ESQL: Fix error message when failing to resolve aggregate groupings {es-pull}108101[#108101] (issue: {es-issue}108053[#108053]) -* ESQL: Fix treating all fields as MV in COUNT pushdown {es-pull}106720[#106720] -* ESQL: Re-enable logical dependency check {es-pull}105860[#105860] -* ESQL: median, count and `count_distinct` over constants {es-pull}107414[#107414] (issues: {es-issue}105248[#105248], {es-issue}104900[#104900]) -* ES|QL fix no-length substring with supplementary (4-byte) character {es-pull}107183[#107183] -* ES|QL: Fix usage of IN operator with TEXT fields {es-pull}106654[#106654] (issue: {es-issue}105379[#105379]) -* ES|QL: Improve support for TEXT fields in functions {es-pull}106810[#106810] -* Fix docs generation of signatures for variadic functions {es-pull}107865[#107865] -* [ESQL] Mark `date_diff` as requiring all three arguments {es-pull}108834[#108834] (issue: {es-issue}108383[#108383]) - -Health:: -* Don't stop checking if the `HealthNode` persistent task is present {es-pull}105449[#105449] (issue: {es-issue}98926[#98926]) -* Health monitor concurrency fixes {es-pull}105674[#105674] (issue: {es-issue}105065[#105065]) - -Highlighting:: -* Check preTags and postTags params for empty values {es-pull}106396[#106396] (issue: {es-issue}69009[#69009]) -* added fix for inconsistent text trimming in Unified Highlighter {es-pull}99961[#99961] (issue: {es-issue}101803[#101803]) - -Infra/CLI:: -* Workaround G1 bug for JDK 22 and 22.0.1 {es-pull}108571[#108571] - -Infra/Core:: -* Add a check for the same feature being declared regular and historical {es-pull}106285[#106285] -* Fix `AffixSetting.exists` to include secure settings {es-pull}106745[#106745] -* Fix regression in get index settings (human=true) where the version was not displayed in human-readable format {es-pull}107447[#107447] -* Nativeaccess: try to load all located libsystemds {es-pull}108238[#108238] (issue: {es-issue}107878[#107878]) -* Update several references to `IndexVersion.toString` to use `toReleaseVersion` {es-pull}107828[#107828] (issue: {es-issue}107821[#107821]) -* Update several references to `TransportVersion.toString` to use `toReleaseVersion` {es-pull}107902[#107902] - -Infra/Logging:: -* Log when update AffixSetting using addAffixMapUpdateConsumer {es-pull}97072[#97072] - -Infra/Node Lifecycle:: -* Consider `ShardRouting` roles when calculating shard copies in shutdown status {es-pull}106063[#106063] -* Wait indefintely for http connections on shutdown by default {es-pull}106511[#106511] - -Infra/Scripting:: -* Guard against a null scorer in painless execute {es-pull}109048[#109048] (issue: {es-issue}43541[#43541]) -* Painless: Apply true regex limit factor with FIND and MATCH operation {es-pull}105670[#105670] - -Ingest Node:: -* Catching `StackOverflowErrors` from bad regexes in `GsubProcessor` {es-pull}106851[#106851] -* Fix `uri_parts` processor behaviour for missing extensions {es-pull}105689[#105689] (issue: {es-issue}105612[#105612]) -* Remove leading is_ prefix from Enterprise geoip docs {es-pull}108518[#108518] -* Slightly better geoip `databaseType` validation {es-pull}106889[#106889] - -License:: -* Fix lingering license warning header {es-pull}108031[#108031] (issue: {es-issue}107573[#107573]) - -Machine Learning:: -* Fix NPE in ML assignment notifier {es-pull}107312[#107312] -* Fix `startOffset` must be non-negative error in XLMRoBERTa tokenizer {es-pull}107891[#107891] (issue: {es-issue}104626[#104626]) -* Fix the position of spike, dip and distribution changes bucket when the sibling aggregation includes empty buckets {es-pull}106472[#106472] -* Make OpenAI embeddings parser more flexible {es-pull}106808[#106808] - -Mapping:: -* Dedupe terms in terms queries {es-pull}106381[#106381] -* Extend support of `allowedFields` to `getMatchingFieldNames` and `getAllFields` {es-pull}106862[#106862] -* Fix for raw mapping merge of fields named "properties" {es-pull}108867[#108867] (issue: {es-issue}108866[#108866]) -* Handle infinity during synthetic source construction for scaled float field {es-pull}107494[#107494] (issue: {es-issue}107101[#107101]) -* Handle pass-through subfields with deep nesting {es-pull}106767[#106767] -* Wrap "Pattern too complex" exception into an `IllegalArgumentException` {es-pull}109173[#109173] - -Network:: -* Fix HTTP corner-case response leaks {es-pull}105617[#105617] - -Search:: -* Add `internalClusterTest` for and fix leak in `ExpandSearchPhase` {es-pull}108562[#108562] (issue: {es-issue}108369[#108369]) -* Avoid attempting to load the same empty field twice in fetch phase {es-pull}107551[#107551] -* Bugfix: Disable eager loading `BitSetFilterCache` on Indexing Nodes {es-pull}105791[#105791] -* Cross-cluster painless/execute actions should check permissions only on target remote cluster {es-pull}105360[#105360] -* Fix error 500 on invalid `ParentIdQuery` {es-pull}105693[#105693] (issue: {es-issue}105366[#105366]) -* Fix range queries for float/half_float fields when bounds are out of type's range {es-pull}106691[#106691] -* Fixing NPE when requesting [_none_] for `stored_fields` {es-pull}104711[#104711] -* Fork when handling remote field-caps responses {es-pull}107370[#107370] -* Handle parallel calls to `createWeight` when profiling is on {es-pull}108041[#108041] (issues: {es-issue}104131[#104131], {es-issue}104235[#104235]) -* Harden field-caps request dispatcher {es-pull}108736[#108736] -* Replace `UnsupportedOperationException` with `IllegalArgumentException` for non-existing columns {es-pull}107038[#107038] -* Unable to retrieve multiple stored field values {es-pull}106575[#106575] -* Validate `model_id` is required when using the `learning_to_rank` rescorer {es-pull}107743[#107743] - -Security:: -* Disable validate when rewrite parameter is sent and the index access control list is non-null {es-pull}105709[#105709] -* Fix field caps and field level security {es-pull}106731[#106731] - -Snapshot/Restore:: -* Fix double-pausing shard snapshot {es-pull}109148[#109148] (issue: {es-issue}109143[#109143]) -* Treat 404 as empty register in `AzureBlobStore` {es-pull}108900[#108900] (issue: {es-issue}108504[#108504]) -* `SharedBlobCacheService.maybeFetchRegion` should use `computeCacheFileRegionSize` {es-pull}106685[#106685] - -TSDB:: -* Flip dynamic mapping condition when create tsid {es-pull}105636[#105636] - -Transform:: -* Consolidate permissions checks {es-pull}106413[#106413] (issue: {es-issue}105794[#105794]) -* Disable PIT for remote clusters {es-pull}107969[#107969] -* Make force-stopping the transform always remove persistent task from cluster state {es-pull}106989[#106989] (issue: {es-issue}106811[#106811]) -* Only trigger action once per thread {es-pull}107232[#107232] (issue: {es-issue}107215[#107215]) -* [Transform] Auto retry Transform start {es-pull}106243[#106243] - -Vector Search:: -* Fix multithreading copies in lib vec {es-pull}108802[#108802] -* [8.14] Fix multithreading copies in lib vec {es-pull}108810[#108810] - -[[deprecation-8.14.0]] -[float] -=== Deprecations - -Mapping:: -* Deprecate allowing `fields` in scenarios where it is ignored {es-pull}106031[#106031] - -[[enhancement-8.14.0]] -[float] -=== Enhancements - -Aggregations:: -* Add a `PriorityQueue` backed by `BigArrays` {es-pull}106361[#106361] -* All new `shard_seed` parameter for `random_sampler` agg {es-pull}104830[#104830] - -Allocation:: -* Add allocation stats {es-pull}105894[#105894] -* Add index forecasts to /_cat/allocation output {es-pull}97561[#97561] - -Application:: -* [Profiling] Add TopN Functions API {es-pull}106860[#106860] -* [Profiling] Allow to override index settings {es-pull}106172[#106172] -* [Profiling] Speed up serialization of flamegraph {es-pull}105779[#105779] - -Authentication:: -* Support Profile Activate with JWTs with client authn {es-pull}105439[#105439] (issue: {es-issue}105342[#105342]) - -Authorization:: -* Allow users to get status of own async search tasks {es-pull}106638[#106638] -* [Security Solution] Add `read` permission for third party agent indices for `kibana_system` {es-pull}107046[#107046] - -Data streams:: -* Add data stream lifecycle to kibana reporting template {es-pull}106259[#106259] - -ES|QL:: -* Add ES|QL Locate function {es-pull}106899[#106899] (issue: {es-issue}106818[#106818]) -* Add ES|QL signum function {es-pull}106866[#106866] -* Add status for enrich operator {es-pull}106036[#106036] -* Add two new OGC functions ST_X and ST_Y {es-pull}105768[#105768] -* Adjust array resizing in block builder {es-pull}106934[#106934] -* Bulk loading enrich fields in ESQL {es-pull}106796[#106796] -* ENRICH support for TEXT fields {es-pull}106435[#106435] (issue: {es-issue}105384[#105384]) -* ESQL: Add timers to many status results {es-pull}105421[#105421] -* ESQL: Allow grouping key inside stats expressions {es-pull}106579[#106579] -* ESQL: Introduce expression validation phase {es-pull}105477[#105477] (issue: {es-issue}105425[#105425]) -* ESQL: Log queries at debug level {es-pull}108257[#108257] -* ESQL: Regex improvements {es-pull}106429[#106429] -* ESQL: Sum of constants {es-pull}105454[#105454] -* ESQL: Support ST_DISJOINT {es-pull}107007[#107007] -* ESQL: Support partially folding CASE {es-pull}106094[#106094] -* ESQL: Use faster field caps {es-pull}105067[#105067] -* ESQL: extend BUCKET with spans {es-pull}107272[#107272] -* ESQL: perform a reduction on the data node {es-pull}106516[#106516] -* Expand support for ENRICH to full set supported by ES ingest processors {es-pull}106186[#106186] (issue: {es-issue}106162[#106162]) -* Introduce ordinal bytesref block {es-pull}106852[#106852] (issue: {es-issue}106387[#106387]) -* Leverage ordinals in enrich lookup {es-pull}107449[#107449] -* Serialize big array blocks {es-pull}106373[#106373] -* Serialize big array vectors {es-pull}106327[#106327] -* Specialize serialization for `ArrayVectors` {es-pull}105893[#105893] -* Specialize serialization of array blocks {es-pull}106102[#106102] -* Speed up serialization of `BytesRefArray` {es-pull}106053[#106053] -* Support ST_CONTAINS and ST_WITHIN {es-pull}106503[#106503] -* Support ST_INTERSECTS between geometry column and other geometry or string {es-pull}104907[#104907] (issue: {es-issue}104874[#104874]) - -Engine:: -* Add metric for calculating index flush time excluding waiting on locks {es-pull}107196[#107196] - -Highlighting:: -* Enable 'encoder' and 'tags_schema' highlighting settings at field level {es-pull}107224[#107224] (issue: {es-issue}94028[#94028]) - -ILM+SLM:: -* Add a flag to re-enable writes on the final index after an ILM shrink action. {es-pull}107121[#107121] (issue: {es-issue}106599[#106599]) - -Indices APIs:: -* Wait forever for `IndexTemplateRegistry` asset installation {es-pull}105985[#105985] - -Infra/CLI:: -* Enhance search tier GC options {es-pull}106526[#106526] -* Increase KDF iteration count in `KeyStoreWrapper` {es-pull}107107[#107107] - -Infra/Core:: -* Add pluggable `BuildVersion` in `NodeMetadata` {es-pull}105757[#105757] - -Infra/Metrics:: -* Infrastructure for metering the update requests {es-pull}105063[#105063] -* `DocumentParsingObserver` to accept an `indexName` to allow skipping system indices {es-pull}107041[#107041] - -Infra/Scripting:: -* String sha512() painless function {es-pull}99048[#99048] (issue: {es-issue}97691[#97691]) - -Ingest Node:: -* Add support for the 'Anonymous IP' database to the geoip processor {es-pull}107287[#107287] (issue: {es-issue}90789[#90789]) -* Add support for the 'Enterprise' database to the geoip processor {es-pull}107377[#107377] -* Adding `cache_stats` to geoip stats API {es-pull}107334[#107334] -* Support data streams in enrich policy indices {es-pull}107291[#107291] (issue: {es-issue}98836[#98836]) - -Machine Learning:: -* Add GET `_inference` for all inference endpoints {es-pull}107517[#107517] -* Added a timeout parameter to the inference API {es-pull}107242[#107242] -* Enable retrying on 500 error response from Cohere text embedding API {es-pull}105797[#105797] - -Mapping:: -* Make int8_hnsw our default index for new dense-vector fields {es-pull}106836[#106836] - -Ranking:: -* Add retrievers using the parser-only approach {es-pull}105470[#105470] - -Search:: -* Add Lucene spanish plural stemmer {es-pull}106952[#106952] -* Add `modelId` and `modelText` to `KnnVectorQueryBuilder` {es-pull}106068[#106068] -* Add a SIMD (Neon) optimised vector distance function for int8 {es-pull}106133[#106133] -* Add transport version for search load autoscaling {es-pull}106377[#106377] -* CCS with `minimize_roundtrips` performs incremental merges of each `SearchResponse` {es-pull}105781[#105781] -* Track ongoing search tasks {es-pull}107129[#107129] - -Security:: -* Invalidating cross cluster API keys requires `manage_security` {es-pull}107411[#107411] -* Show owner `realm_type` for returned API keys {es-pull}105629[#105629] - -Snapshot/Restore:: -* Add setting for max connections to S3 {es-pull}107533[#107533] -* Distinguish different snapshot failures by log level {es-pull}105622[#105622] - -Stats:: -* (API+) CAT Nodes alias for shard header to match CAT Allocation {es-pull}105847[#105847] -* Add total size in bytes to doc stats {es-pull}106840[#106840] (issue: {es-issue}97670[#97670]) - -TSDB:: -* Improve short-circuiting downsample execution {es-pull}106563[#106563] -* Support non-keyword dimensions as routing fields in TSDB {es-pull}105501[#105501] -* Text fields are stored by default in TSDB indices {es-pull}106338[#106338] (issue: {es-issue}97039[#97039]) - -Transform:: -* Check node shutdown before fail {es-pull}107358[#107358] (issue: {es-issue}100891[#100891]) -* Do not log error on node restart when the transform is already failed {es-pull}106171[#106171] (issue: {es-issue}106168[#106168]) - -[[feature-8.14.0]] -[float] -=== New features - -Application:: -* Allow `typed_keys` for search application Search API {es-pull}108007[#108007] -* [Connector API] Support cleaning up sync jobs when deleting a connector {es-pull}107253[#107253] - -ES|QL:: -* ESQL: Values aggregation function {es-pull}106065[#106065] (issue: {es-issue}103600[#103600]) -* ESQL: allow sorting by expressions and not only regular fields {es-pull}107158[#107158] -* Support ES|QL requests through the `NodeClient::execute` {es-pull}106244[#106244] - -Indices APIs:: -* Add granular error list to alias action response {es-pull}106514[#106514] (issue: {es-issue}94478[#94478]) - -Machine Learning:: -* Add Cohere rerank to `_inference` service {es-pull}106378[#106378] -* Add support for Azure OpenAI embeddings to inference service {es-pull}107178[#107178] -* Create default word based chunker {es-pull}107303[#107303] -* Text structure endpoints to determine the structure of a list of messages and of an indexed field {es-pull}105660[#105660] - -Mapping:: -* Flatten object mappings when subobjects is false {es-pull}103542[#103542] (issues: {es-issue}99860[#99860], {es-issue}103497[#103497]) - -Security:: -* Get and Query API Key with profile uid {es-pull}106531[#106531] - -Vector Search:: -* Adding support for hex-encoded byte vectors on knn-search {es-pull}105393[#105393] - -[[upgrade-8.14.0]] -[float] -=== Upgrades - -Infra/Core:: -* Upgrade jna to 5.12.1 {es-pull}105717[#105717] - -Ingest Node:: -* Updating the tika version to 2.9.1 in the ingest attachment plugin {es-pull}106315[#106315] - -Network:: -* Upgrade to Netty 4.1.107 {es-pull}105517[#105517] - -Packaging:: -* Update bundled JDK to Java 22 (again) {es-pull}108654[#108654] diff --git a/docs/reference/release-notes/8.14.1.asciidoc b/docs/reference/release-notes/8.14.1.asciidoc deleted file mode 100644 index de3ecd210b488..0000000000000 --- a/docs/reference/release-notes/8.14.1.asciidoc +++ /dev/null @@ -1,50 +0,0 @@ -[[release-notes-8.14.1]] -== {es} version 8.14.1 - - -Also see <>. - -[[known-issues-8.14.1]] -[float] -=== Known issues -* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, -information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. -If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. -To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. This issue is fixed in 8.15.0. - -* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. -In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` -process to reclaim memory. This can cause inference requests to fail. -Elasticsearch will automatically restart the `pytorch_inference` process -after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) - -[[bug-8.14.1]] -[float] -=== Bug fixes - -Authorization:: -* Fix task cancellation authz on fulfilling cluster {es-pull}109357[#109357] - -Infra/Core:: -* Guard systemd library lookup from unreadable directories {es-pull}108931[#108931] - -Machine Learning:: -* Reset retryable index requests after failures {es-pull}109320[#109320] - -Network:: -* Fix task cancellation on remote cluster when original request fails {es-pull}109440[#109440] - -Transform:: -* Reset max page size to settings value {es-pull}109532[#109532] (issue: {es-issue}109308[#109308]) - -Vector Search:: -* Correct how hex strings are handled when dynamically updating vector dims {es-pull}109423[#109423] - -[[enhancement-8.14.1]] -[float] -=== Enhancements - -Infra/Settings:: -* Add remove index setting command {es-pull}109276[#109276] diff --git a/docs/reference/release-notes/8.14.2.asciidoc b/docs/reference/release-notes/8.14.2.asciidoc deleted file mode 100644 index f3f0651508dca..0000000000000 --- a/docs/reference/release-notes/8.14.2.asciidoc +++ /dev/null @@ -1,52 +0,0 @@ -[[release-notes-8.14.2]] -== {es} version 8.14.2 - -Also see <>. - -[[known-issues-8.14.2]] -[float] -=== Known issues -* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, -information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. -If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. -To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. This issue is fixed in 8.15.0. - -* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. -In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` -process to reclaim memory. This can cause inference requests to fail. -Elasticsearch will automatically restart the `pytorch_inference` process -after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) - -[[bug-8.14.2]] -[float] -=== Bug fixes - -Data streams:: -* Ensure a lazy rollover request will rollover the target data stream once. {es-pull}109636[#109636] -* [Data streams] Fix the description of the lazy rollover task {es-pull}109629[#109629] - -ES|QL:: -* Fix ESQL cancellation for exchange requests {es-pull}109695[#109695] -* Fix equals and hashcode for `SingleValueQuery.LuceneQuery` {es-pull}110035[#110035] -* Force execute inactive sink reaper {es-pull}109632[#109632] - -Infra/Scripting:: -* Check array size before returning array item in script doc values {es-pull}109824[#109824] (issue: {es-issue}104998[#104998]) - -Infra/Settings:: -* Guard file settings readiness on file settings support {es-pull}109500[#109500] - -Machine Learning:: -* Fix IndexOutOfBoundsException during inference {es-pull}109533[#109533] - -Mapping:: -* Re-define `index.mapper.dynamic` setting in 8.x for a better 7.x to 8.x upgrade if this setting is used. {es-pull}109341[#109341] - -Ranking:: -* Fix for from parameter when using `sub_searches` and rank {es-pull}106253[#106253] (issue: {es-issue}99011[#99011]) - -Search:: -* Add hexstring support byte painless scorers {es-pull}109492[#109492] -* Fix automatic tracking of collapse with `docvalue_fields` {es-pull}110103[#110103] diff --git a/docs/reference/release-notes/8.14.3.asciidoc b/docs/reference/release-notes/8.14.3.asciidoc deleted file mode 100644 index 17c53faa4a37f..0000000000000 --- a/docs/reference/release-notes/8.14.3.asciidoc +++ /dev/null @@ -1,32 +0,0 @@ -[[release-notes-8.14.3]] -== {es} version 8.14.3 - -Also see <>. - -[[known-issues-8.14.3]] -[float] -=== Known issues -* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, -information about the new functionality of these upgraded nodes may not be registered properly with the master node. -This can lead to some new functionality added since 8.12.0 not being accessible on the upgraded cluster. -If your cluster is running on ECK 2.12.1 and above, this may cause problems with finalizing the upgrade. -To resolve this issue, perform a rolling restart on the non-master-eligible nodes once all Elasticsearch nodes -are upgraded. This issue is fixed in 8.15.0. - -[[bug-8.14.3]] -[float] -=== Bug fixes - -Cluster Coordination:: -* Ensure tasks preserve versions in `MasterService` {es-pull}109850[#109850] - -ES|QL:: -* Introduce compute listener {es-pull}110400[#110400] - -Mapping:: -* Automatically adjust `ignore_malformed` only for the @timestamp {es-pull}109948[#109948] - -TSDB:: -* Disallow index.time_series.end_time setting from being set or updated in normal indices {es-pull}110268[#110268] (issue: {es-issue}110265[#110265]) - - diff --git a/docs/reference/release-notes/8.15.0.asciidoc b/docs/reference/release-notes/8.15.0.asciidoc deleted file mode 100644 index 80c86c7079f0c..0000000000000 --- a/docs/reference/release-notes/8.15.0.asciidoc +++ /dev/null @@ -1,558 +0,0 @@ -[[release-notes-8.15.0]] -== {es} version 8.15.0 - -Also see <>. - - -[[known-issues-8.15.0]] -[float] -=== Known issues -* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. -In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` -process to reclaim memory. This can cause inference requests to fail. -Elasticsearch will automatically restart the `pytorch_inference` process -after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) - -* Pipeline aggregations under `time_series` and `categorize_text` aggregations are never -returned (issue: {es-issue}111679[#111679]) - -* Elasticsearch will not start on Windows machines if -[`bootstrap.memory_lock` is set to `true`](https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration-memory.html#bootstrap-memory_lock). -Either downgrade to an earlier version, upgrade to 8.15.1, or else follow the -recommendation in the manual to entirely disable swap instead of using the -memory lock feature (issue: {es-issue}111847[#111847]) - -* The `took` field of the response to the <> API is incorrect and may be rather large. Clients which -<> assume that this value will be within a particular range (e.g. that it fits into a 32-bit -signed integer) may encounter errors (issue: {es-issue}111854[#111854]) - -* Elasticsearch will not start if custom role mappings are configured using the -`xpack.security.authc.realms.*.files.role_mapping` configuration option. As a workaround, custom role mappings -can be configured using the https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-put-role-mapping.html[REST API] (issue: {es-issue}112503[#112503]) - -* ES|QL queries can lead to node crashes due to Out Of Memory errors when: -** Multiple indices match the query pattern -** These indices have many conflicting field mappings -** Many of those fields are included in the request -These issues deplete heap memory, increasing the likelihood of OOM errors. (issue: {es-issue}111964[#111964], {es-issue}111358[#111358]). -In Kibana, you might indirectly execute these queries when using Discover, or adding a Field Statistics panel to a dashboard. -+ -To work around this issue, you have a number of options: -** Downgrade to an earlier version -** Upgrade to 8.15.2 upon release -** Follow the instructions to -<> -** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts. - -* Synthetic source bug. Synthetic source may fail generating the _source at runtime, causing failures in get APIs or -partial failures in the search APIs. The result is that for the affected documents the _source can't be retrieved. -There is no workaround and the only option to is to upgrade to 8.15.2 when released. -+ -If you use synthetic source then you may be affected by this bug if the following is true: -** If you have more fields then the `index.mapping.total_fields.limit` setting allows. -** If you use dynamic mappings and the `index.mapping.total_fields.ignore_dynamic_beyond_limit` setting is enabled. - -[[breaking-8.15.0]] -[float] -=== Breaking changes - -Cluster Coordination:: -* Interpret `?timeout=-1` as infinite ack timeout {es-pull}107675[#107675] - -Inference API:: -* Replace `model_id` with `inference_id` in GET inference API {es-pull}111366[#111366] - -Rollup:: -* Disallow new rollup jobs in clusters with no rollup usage {es-pull}108624[#108624] (issue: {es-issue}108381[#108381]) - -Search:: -* Change `skip_unavailable` remote cluster setting default value to true {es-pull}105792[#105792] - -[[bug-8.15.0]] -[float] -=== Bug fixes - -Aggregations:: -* Don't sample calls to `ReduceContext#consumeBucketsAndMaybeBreak` ins `InternalDateHistogram` and `InternalHistogram` during reduction {es-pull}110186[#110186] -* Fix `ClassCastException` in Significant Terms {es-pull}108429[#108429] (issue: {es-issue}108427[#108427]) -* Run terms concurrently when cardinality is only lower than shard size {es-pull}110369[#110369] (issue: {es-issue}105505[#105505]) - -Allocation:: -* Fix misc trappy allocation API timeouts {es-pull}109241[#109241] -* Fix trappy timeout in allocation explain API {es-pull}109240[#109240] - -Analysis:: -* Correct positioning for unique token filter {es-pull}109395[#109395] - -Authentication:: -* Add comma before charset parameter in WWW-Authenticate response header {es-pull}110906[#110906] -* Avoid NPE if `users_roles` file does not exist {es-pull}109606[#109606] -* Improve security-crypto threadpool overflow handling {es-pull}111369[#111369] - -Authorization:: -* Fix trailing slash in `security.put_privileges` specification {es-pull}110177[#110177] -* Fixes cluster state-based role mappings not recovered from disk {es-pull}109167[#109167] -* Handle unmatching remote cluster wildcards properly for `IndicesRequest.SingleIndexNoWildcards` requests {es-pull}109185[#109185] - -Autoscaling:: -* Expose `?master_timeout` in autoscaling APIs {es-pull}108759[#108759] - -CRUD:: -* Update checkpoints after post-replication actions, even on failure {es-pull}109908[#109908] - -Cluster Coordination:: -* Deserialize publish requests on generic thread-pool {es-pull}108814[#108814] (issue: {es-issue}106352[#106352]) -* Fail cluster state API if blocked {es-pull}109618[#109618] (issue: {es-issue}107503[#107503]) -* Use `scheduleUnlessShuttingDown` in `LeaderChecker` {es-pull}108643[#108643] (issue: {es-issue}108642[#108642]) - -Data streams:: -* Apm-data: set concrete values for `metricset.interval` {es-pull}109043[#109043] -* Ecs@mappings: reduce scope for `ecs_geo_point` {es-pull}108349[#108349] (issue: {es-issue}108338[#108338]) -* Include component templates in retention validaiton {es-pull}109779[#109779] - -Distributed:: -* Associate restore snapshot task to parent mount task {es-pull}108705[#108705] (issue: {es-issue}105830[#105830]) -* Don't detect `PlainActionFuture` deadlock on concurrent complete {es-pull}110361[#110361] (issues: {es-issue}110181[#110181], {es-issue}110360[#110360]) -* Handle nullable `DocsStats` and `StoresStats` {es-pull}109196[#109196] - -Downsampling:: -* Support flattened fields and multi-fields as dimensions in downsampling {es-pull}110066[#110066] (issue: {es-issue}99297[#99297]) - -ES|QL:: -* ESQL: Change "substring" function to not return null on empty string {es-pull}109174[#109174] -* ESQL: Fix Join references {es-pull}109989[#109989] -* ESQL: Fix LOOKUP attribute shadowing {es-pull}109807[#109807] (issue: {es-issue}109392[#109392]) -* ESQL: Fix Max doubles bug with negatives and add tests for Max and Min {es-pull}110586[#110586] -* ESQL: Fix `IpPrefix` function not handling correctly `ByteRefs` {es-pull}109205[#109205] (issue: {es-issue}109198[#109198]) -* ESQL: Fix equals `hashCode` for functions {es-pull}107947[#107947] (issue: {es-issue}104393[#104393]) -* ESQL: Fix variable shadowing when pushing down past Project {es-pull}108360[#108360] (issue: {es-issue}108008[#108008]) -* ESQL: Validate unique plan attribute names {es-pull}110488[#110488] (issue: {es-issue}110541[#110541]) -* ESQL: change from quoting from backtick to quote {es-pull}108395[#108395] -* ESQL: make named params objects truly per request {es-pull}110046[#110046] (issue: {es-issue}110028[#110028]) -* ES|QL: Fix DISSECT that overwrites input {es-pull}110201[#110201] (issue: {es-issue}110184[#110184]) -* ES|QL: limit query depth to 500 levels {es-pull}108089[#108089] (issue: {es-issue}107752[#107752]) -* ES|QL: reduce max expression depth to 400 {es-pull}111186[#111186] (issue: {es-issue}109846[#109846]) -* Fix ST_DISTANCE Lucene push-down for complex predicates {es-pull}110391[#110391] (issue: {es-issue}110349[#110349]) -* Fix `ClassCastException` with MV_EXPAND on missing field {es-pull}110096[#110096] (issue: {es-issue}109974[#109974]) -* Fix bug in union-types with type-casting in grouping key of STATS {es-pull}110476[#110476] (issues: {es-issue}109922[#109922], {es-issue}110477[#110477]) -* Fix for union-types for multiple columns with the same name {es-pull}110793[#110793] (issues: {es-issue}110490[#110490], {es-issue}109916[#109916]) -* [ESQL] Count_distinct(_source) should return a 400 {es-pull}110824[#110824] -* [ESQL] Fix parsing of large magnitude negative numbers {es-pull}110665[#110665] (issue: {es-issue}104323[#104323]) -* [ESQL] Migrate `SimplifyComparisonArithmetics` optimization {es-pull}109256[#109256] (issues: {es-issue}108388[#108388], {es-issue}108743[#108743]) - -Engine:: -* Async close of `IndexShard` {es-pull}108145[#108145] - -Highlighting:: -* Fix issue with returning incomplete fragment for plain highlighter {es-pull}110707[#110707] - -ILM+SLM:: -* Allow `read_slm` to call GET /_slm/status {es-pull}108333[#108333] - -Indices APIs:: -* Create a new `NodeRequest` for every `NodesDataTiersUsageTransport` use {es-pull}108379[#108379] - -Infra/Core:: -* Add a cluster listener to fix missing node features after upgrading from a version prior to 8.13 {es-pull}110710[#110710] (issue: {es-issue}109254[#109254]) -* Add bounds checking to parsing ISO8601 timezone offset values {es-pull}108672[#108672] -* Fix native preallocate to actually run {es-pull}110851[#110851] -* Ignore additional cpu.stat fields {es-pull}108019[#108019] (issue: {es-issue}107983[#107983]) -* Specify parse index when error occurs on multiple datetime parses {es-pull}108607[#108607] - -Infra/Metrics:: -* Provide document size reporter with `MapperService` {es-pull}109794[#109794] - -Infra/Node Lifecycle:: -* Expose `?master_timeout` on get-shutdown API {es-pull}108886[#108886] -* Fix serialization of put-shutdown request {es-pull}107862[#107862] (issue: {es-issue}107857[#107857]) -* Support wait indefinitely for search tasks to complete on node shutdown {es-pull}107426[#107426] - -Infra/REST API:: -* Add some missing timeout params to REST API specs {es-pull}108761[#108761] -* Consider `error_trace` supported by all endpoints {es-pull}109613[#109613] (issue: {es-issue}109612[#109612]) - -Ingest Node:: -* Fix Dissect with leading non-ascii characters {es-pull}111184[#111184] -* Fix enrich policy runner exception handling on empty segments response {es-pull}111290[#111290] -* GeoIP tasks should wait longer for master {es-pull}108410[#108410] -* Removing the use of Stream::peek from `GeoIpDownloader::cleanDatabases` {es-pull}110666[#110666] -* Simulate should succeed if `ignore_missing_pipeline` {es-pull}108106[#108106] (issue: {es-issue}107314[#107314]) - -Machine Learning:: -* Allow deletion of the ELSER inference service when reference in ingest {es-pull}108146[#108146] -* Avoid `InferenceRunner` deadlock {es-pull}109551[#109551] -* Correctly handle duplicate model ids for the `_cat` trained models api and usage statistics {es-pull}109126[#109126] -* Do not use global ordinals strategy if the leaf reader context cannot be obtained {es-pull}108459[#108459] -* Fix NPE in trained model assignment updater {es-pull}108942[#108942] -* Fix serialising inference delete response {es-pull}109384[#109384] -* Fix "stack use after scope" memory error {ml-pull}2673[#2673] -* Fix trailing slash in `ml.get_categories` specification {es-pull}110146[#110146] -* Handle any exception thrown by inference {ml-pull}2680[#2680] -* Increase response size limit for batched requests {es-pull}110112[#110112] -* Offload request to generic threadpool {es-pull}109104[#109104] (issue: {es-issue}109100[#109100]) -* Propagate accurate deployment timeout {es-pull}109534[#109534] (issue: {es-issue}109407[#109407]) -* Refactor TextEmbeddingResults to use primitives rather than objects {es-pull}108161[#108161] -* Require question to be non-null in `QuestionAnsweringConfig` {es-pull}107972[#107972] -* Start Trained Model Deployment API request query params now override body params {es-pull}109487[#109487] -* Suppress deprecation warnings from ingest pipelines when deleting trained model {es-pull}108679[#108679] (issue: {es-issue}105004[#105004]) -* Use default translog durability on AD results index {es-pull}108999[#108999] -* Use the multi node routing action for internal inference services {es-pull}109358[#109358] -* [Inference API] Extract optional long instead of integer in `RateLimitSettings#of` {es-pull}108602[#108602] -* [Inference API] Fix serialization for inference delete endpoint response {es-pull}110431[#110431] -* [Inference API] Replace `model_id` with `inference_id` in inference API except when stored {es-pull}111366[#111366] - -Mapping:: -* Fix off by one error when handling null values in range fields {es-pull}107977[#107977] (issue: {es-issue}107282[#107282]) -* Limit number of synonym rules that can be created {es-pull}109981[#109981] (issue: {es-issue}108785[#108785]) -* Propagate mapper builder context flags across nested mapper builder context creation {es-pull}109963[#109963] -* `DenseVectorFieldMapper` fixed typo {es-pull}108065[#108065] - -Network:: -* Use proper executor for failing requests when connection closes {es-pull}109236[#109236] (issue: {es-issue}109225[#109225]) -* `NoSuchRemoteClusterException` should not be thrown when a remote is configured {es-pull}107435[#107435] (issue: {es-issue}107381[#107381]) - -Packaging:: -* Adding override for lintian false positive on `libvec.so` {es-pull}108521[#108521] (issue: {es-issue}108514[#108514]) - -Ranking:: -* Fix score count validation in reranker response {es-pull}111424[#111424] (issue: {es-issue}111202[#111202]) - -Rollup:: -* Fix trailing slash in two rollup specifications {es-pull}110176[#110176] - -Search:: -* Adding score from `RankDoc` to `SearchHit` {es-pull}108870[#108870] -* Better handling of multiple rescorers clauses with LTR {es-pull}109071[#109071] -* Correct query profiling for conjunctions {es-pull}108122[#108122] (issue: {es-issue}108116[#108116]) -* Fix `DecayFunctions'` `toString` {es-pull}107415[#107415] (issue: {es-issue}100870[#100870]) -* Fix leak in collapsing search results {es-pull}110927[#110927] -* Fork freeing search/scroll contexts to GENERIC pool {es-pull}109481[#109481] - -Security:: -* Add permission to secure access to certain config files {es-pull}107827[#107827] -* Add permission to secure access to certain config files specified by settings {es-pull}108895[#108895] -* Fix trappy timeouts in security settings APIs {es-pull}109233[#109233] - -Snapshot/Restore:: -* Stricter failure handling in multi-repo get-snapshots request handling {es-pull}107191[#107191] - -TSDB:: -* Sort time series indices by time range in `GetDataStreams` API {es-pull}107967[#107967] (issue: {es-issue}102088[#102088]) - -Transform:: -* Always pick the user `maxPageSize` value {es-pull}109876[#109876] (issue: {es-issue}109844[#109844]) -* Exit gracefully when deleted {es-pull}107917[#107917] (issue: {es-issue}107266[#107266]) -* Fix NPE during destination index creation {es-pull}108891[#108891] (issue: {es-issue}108890[#108890]) -* Forward `indexServiceSafe` exception to listener {es-pull}108517[#108517] (issue: {es-issue}108418[#108418]) -* Halt Indexer on Stop/Abort API {es-pull}107792[#107792] -* Handle `IndexNotFoundException` {es-pull}108394[#108394] (issue: {es-issue}107263[#107263]) -* Prevent concurrent jobs during cleanup {es-pull}109047[#109047] -* Redirect `VersionConflict` to reset code {es-pull}108070[#108070] -* Reset max page size to settings value {es-pull}109449[#109449] (issue: {es-issue}109308[#109308]) - -Vector Search:: -* Ensure vector similarity correctly limits `inner_hits` returned for nested kNN {es-pull}111363[#111363] (issue: {es-issue}111093[#111093]) -* Ensure we return non-negative scores when scoring scalar dot-products {es-pull}108522[#108522] - -Watcher:: -* Avoiding running watch jobs in TickerScheduleTriggerEngine if it is paused {es-pull}110061[#110061] (issue: {es-issue}105933[#105933]) - -[[deprecation-8.15.0]] -[float] -=== Deprecations - -ILM+SLM:: -* Deprecate using slm privileges to access ilm {es-pull}110540[#110540] - -Infra/Settings:: -* `ParseHeapRatioOrDeprecatedByteSizeValue` for `indices.breaker.total.limit` {es-pull}110236[#110236] - -Machine Learning:: -* Deprecate `text_expansion` and `weighted_tokens` queries {es-pull}109880[#109880] - -[[enhancement-8.15.0]] -[float] -=== Enhancements - -Aggregations:: -* Aggs: Scripted metric allow list {es-pull}109444[#109444] -* Enable inter-segment concurrency for low cardinality numeric terms aggs {es-pull}108306[#108306] -* Increase size of big arrays only when there is an actual value in the aggregators {es-pull}107764[#107764] -* Increase size of big arrays only when there is an actual value in the aggregators (Analytics module) {es-pull}107813[#107813] -* Optimise `BinaryRangeAggregator` for single value fields {es-pull}108016[#108016] -* Optimise cardinality aggregations for single value fields {es-pull}107892[#107892] -* Optimise composite aggregations for single value fields {es-pull}107897[#107897] -* Optimise few metric aggregations for single value fields {es-pull}107832[#107832] -* Optimise histogram aggregations for single value fields {es-pull}107893[#107893] -* Optimise multiterms aggregation for single value fields {es-pull}107937[#107937] -* Optimise terms aggregations for single value fields {es-pull}107930[#107930] -* Speed up collecting zero document string terms {es-pull}110922[#110922] - -Allocation:: -* Log shard movements {es-pull}105829[#105829] -* Support effective watermark thresholds in node stats API {es-pull}107244[#107244] (issue: {es-issue}106676[#106676]) - -Application:: -* Add Create or update query rule API call {es-pull}109042[#109042] -* Rename rule query and add support for multiple rulesets {es-pull}108831[#108831] -* Support multiple associated groups for TopN {es-pull}108409[#108409] (issue: {es-issue}108018[#108018]) -* [Connector API] Change `UpdateConnectorFiltering` API to have better defaults {es-pull}108612[#108612] - -Authentication:: -* Expose API Key cache metrics {es-pull}109078[#109078] - -Authorization:: -* Cluster state role mapper file settings service {es-pull}107886[#107886] -* Cluster-state based Security role mapper {es-pull}107410[#107410] -* Introduce role description field {es-pull}107088[#107088] -* [Osquery] Extend `kibana_system` role with an access to new `osquery_manager` index {es-pull}108849[#108849] - -Data streams:: -* Add metrics@custom component template to metrics-*-* index template {es-pull}109540[#109540] (issue: {es-issue}109475[#109475]) -* Apm-data: enable plugin by default {es-pull}108860[#108860] -* Apm-data: ignore malformed fields, and too many dynamic fields {es-pull}108444[#108444] -* Apm-data: improve default pipeline performance {es-pull}108396[#108396] (issue: {es-issue}108290[#108290]) -* Apm-data: improve indexing resilience {es-pull}108227[#108227] -* Apm-data: increase priority above Fleet templates {es-pull}108885[#108885] -* Apm-data: increase version for templates {es-pull}108340[#108340] -* Apm-data: set codec: best_compression for logs-apm.* data streams {es-pull}108862[#108862] -* Remove `default_field: message` from metrics index templates {es-pull}110651[#110651] - -Distributed:: -* Add `wait_for_completion` parameter to delete snapshot request {es-pull}109462[#109462] (issue: {es-issue}101300[#101300]) -* Improve mechanism for extracting the result of a `PlainActionFuture` {es-pull}110019[#110019] (issue: {es-issue}108125[#108125]) - -ES|QL:: -* Add `BlockHash` for 3 `BytesRefs` {es-pull}108165[#108165] -* Allow `LuceneSourceOperator` to early terminate {es-pull}108820[#108820] -* Check if `CsvTests` required capabilities exist {es-pull}108684[#108684] -* ESQL: Add aggregates node level reduction {es-pull}107876[#107876] -* ESQL: Add more time span units {es-pull}108300[#108300] -* ESQL: Implement LOOKUP, an "inline" enrich {es-pull}107987[#107987] (issue: {es-issue}107306[#107306]) -* ESQL: Renamed `TopList` to Top {es-pull}110347[#110347] -* ESQL: Union Types Support {es-pull}107545[#107545] (issue: {es-issue}100603[#100603]) -* ESQL: add REPEAT string function {es-pull}109220[#109220] -* ES|QL Add primitive float support to the Compute Engine {es-pull}109746[#109746] (issue: {es-issue}109178[#109178]) -* ES|QL Add primitive float variants of all aggregators to the compute engine {es-pull}109781[#109781] -* ES|QL: vectorize eval {es-pull}109332[#109332] -* Optimize ST_DISTANCE filtering with Lucene circle intersection query {es-pull}110102[#110102] (issue: {es-issue}109972[#109972]) -* Optimize for single value in ordinals grouping {es-pull}108118[#108118] -* Rewrite away type converting functions that do not convert types {es-pull}108713[#108713] (issue: {es-issue}107716[#107716]) -* ST_DISTANCE Function {es-pull}108764[#108764] (issue: {es-issue}108212[#108212]) -* Support metrics counter types in ESQL {es-pull}107877[#107877] -* [ESQL] CBRT function {es-pull}108574[#108574] -* [ES|QL] Convert string to datetime when the other size of an arithmetic operator is `date_period` or `time_duration` {es-pull}108455[#108455] -* [ES|QL] Support Named and Positional Parameters in `EsqlQueryRequest` {es-pull}108421[#108421] (issue: {es-issue}107029[#107029]) -* [ES|QL] `weighted_avg` {es-pull}109993[#109993] - -Engine:: -* Drop shards close timeout when stopping node. {es-pull}107978[#107978] (issue: {es-issue}107938[#107938]) -* Update translog `writeLocation` for `flushListener` after commit {es-pull}109603[#109603] - -Geo:: -* Optimize `GeoBounds` and `GeoCentroid` aggregations for single value fields {es-pull}107663[#107663] - -Health:: -* Log details of non-green indicators in `HealthPeriodicLogger` {es-pull}108266[#108266] - -Highlighting:: -* Unified Highlighter to support matched_fields {es-pull}107640[#107640] (issue: {es-issue}5172[#5172]) - -Infra/Core:: -* Add allocation explain output for THROTTLING shards {es-pull}109563[#109563] -* Create custom parser for ISO-8601 datetimes {es-pull}106486[#106486] (issue: {es-issue}102063[#102063]) -* Extend ISO8601 datetime parser to specify forbidden fields, allowing it to be used on more formats {es-pull}108606[#108606] -* add Elastic-internal stable bridge api for use by Logstash {es-pull}108171[#108171] - -Infra/Metrics:: -* Add auto-sharding APM metrics {es-pull}107593[#107593] -* Add request metric to `RestController` to track success/failure (by status code) {es-pull}109957[#109957] -* Allow RA metrics to be reported upon parsing completed or accumulated {es-pull}108726[#108726] -* Provide the `DocumentSizeReporter` with index mode {es-pull}108947[#108947] -* Return noop instance `DocSizeObserver` for updates with scripts {es-pull}108856[#108856] - -Ingest Node:: -* Add `continent_code` support to the geoip processor {es-pull}108780[#108780] (issue: {es-issue}85820[#85820]) -* Add support for the 'Connection Type' database to the geoip processor {es-pull}108683[#108683] -* Add support for the 'Domain' database to the geoip processor {es-pull}108639[#108639] -* Add support for the 'ISP' database to the geoip processor {es-pull}108651[#108651] -* Adding `hits_time_in_millis` and `misses_time_in_millis` to enrich cache stats {es-pull}107579[#107579] -* Adding `user_type` support for the enterprise database for the geoip processor {es-pull}108687[#108687] -* Adding human readable times to geoip stats {es-pull}107647[#107647] -* Include doc size info in ingest stats {es-pull}107240[#107240] (issue: {es-issue}106386[#106386]) -* Make ingest byte stat names more descriptive {es-pull}108786[#108786] -* Return ingest byte stats even when 0-valued {es-pull}108796[#108796] -* Test pipeline run after reroute {es-pull}108693[#108693] - -Logs:: -* Introduce a node setting controlling the activation of the `logs` index mode in logs@settings component template {es-pull}109025[#109025] (issue: {es-issue}108762[#108762]) -* Support index sorting with nested fields {es-pull}110251[#110251] (issue: {es-issue}107349[#107349]) - -Machine Learning:: -* Add Anthropic messages integration to Inference API {es-pull}109893[#109893] -* Add `sparse_vector` query {es-pull}108254[#108254] -* Add model download progress to the download task status {es-pull}107676[#107676] -* Add rate limiting support for the Inference API {es-pull}107706[#107706] -* Add the rerank task to the Elasticsearch internal inference service {es-pull}108452[#108452] -* Default the HF service to cosine similarity {es-pull}109967[#109967] -* GA the update trained model action {es-pull}108868[#108868] -* Handle the "JSON memory allocator bytes" field {es-pull}109653[#109653] -* Inference Processor: skip inference when all fields are missing {es-pull}108131[#108131] -* Log 'No statistics at.. ' message as a warning {ml-pull}2684[#2684] -* Optimise frequent item sets aggregation for single value fields {es-pull}108130[#108130] -* Sentence Chunker {es-pull}110334[#110334] -* [Inference API] Add Amazon Bedrock Support to Inference API {es-pull}110248[#110248] -* [Inference API] Add Mistral Embeddings Support to Inference API {es-pull}109194[#109194] -* [Inference API] Check for related pipelines on delete inference endpoint {es-pull}109123[#109123] - -Mapping:: -* Add ignored field values to synthetic source {es-pull}107567[#107567] -* Apply FLS to the contents of `IgnoredSourceFieldMapper` {es-pull}109931[#109931] -* Binary field enables doc values by default for index mode with synthe… {es-pull}107739[#107739] (issue: {es-issue}107554[#107554]) -* Feature/annotated text store defaults {es-pull}107922[#107922] (issue: {es-issue}107734[#107734]) -* Handle `ignore_above` in synthetic source for flattened fields {es-pull}110214[#110214] -* Opt in keyword field into fallback synthetic source if needed {es-pull}110016[#110016] -* Opt in number fields into fallback synthetic source when doc values a… {es-pull}110160[#110160] -* Reflect latest changes in synthetic source documentation {es-pull}109501[#109501] -* Store source for fields in objects with `dynamic` override {es-pull}108911[#108911] -* Store source for nested objects {es-pull}108818[#108818] -* Support synthetic source for `geo_point` when `ignore_malformed` is used {es-pull}109651[#109651] -* Support synthetic source for `scaled_float` and `unsigned_long` when `ignore_malformed` is used {es-pull}109506[#109506] -* Support synthetic source for date fields when `ignore_malformed` is used {es-pull}109410[#109410] -* Support synthetic source together with `ignore_malformed` in histogram fields {es-pull}109882[#109882] -* Track source for arrays of objects {es-pull}108417[#108417] (issue: {es-issue}90708[#90708]) -* Track synthetic source for disabled objects {es-pull}108051[#108051] - -Network:: -* Detect long-running tasks on network threads {es-pull}109204[#109204] - -Ranking:: -* Enabling profiling for `RankBuilders` and adding tests for RRF {es-pull}109470[#109470] - -Relevance:: -* [Query Rules] Add API calls to get or delete individual query rules within a ruleset {es-pull}109554[#109554] -* [Query Rules] Require Enterprise License for Query Rules {es-pull}109634[#109634] - -Search:: -* Add AVX-512 optimised vector distance functions for int7 on x64 {es-pull}109084[#109084] -* Add `SparseVectorStats` {es-pull}108793[#108793] -* Add `_name` support for top level `knn` clauses {es-pull}107645[#107645] (issues: {es-issue}106254[#106254], {es-issue}107448[#107448]) -* Add a SIMD (AVX2) optimised vector distance function for int7 on x64 {es-pull}108088[#108088] -* Add min/max range of the `event.ingested` field to cluster state for searchable snapshots {es-pull}106252[#106252] -* Add per-field KNN vector format to Index Segments API {es-pull}107216[#107216] -* Add support for hiragana_uppercase & katakana_uppercase token filters in kuromoji analysis plugin {es-pull}106553[#106553] -* Adding support for explain in rrf {es-pull}108682[#108682] -* Allow rescorer with field collapsing {es-pull}107779[#107779] (issue: {es-issue}27243[#27243]) -* Cut over stored fields to ZSTD for compression {es-pull}103374[#103374] -* Limit the value in prefix query {es-pull}108537[#108537] (issue: {es-issue}108486[#108486]) -* Make dense vector field type updatable {es-pull}106591[#106591] -* Multivalue Sparse Vector Support {es-pull}109007[#109007] - -Security:: -* Add bulk delete roles API {es-pull}110383[#110383] -* Remote cluster - API key security model - cluster privileges {es-pull}107493[#107493] - -Snapshot/Restore:: -* Denser in-memory representation of `ShardBlobsToDelete` {es-pull}109848[#109848] -* Log repo UUID at generation/registration time {es-pull}109672[#109672] -* Make repository analysis API available to non-operators {es-pull}110179[#110179] (issue: {es-issue}100318[#100318]) -* Track `RequestedRangeNotSatisfiedException` separately in S3 Metrics {es-pull}109657[#109657] - -Stats:: -* DocsStats: Add human readable bytesize {es-pull}109720[#109720] - -TSDB:: -* Optimise `time_series` aggregation for single value fields {es-pull}107990[#107990] -* Support `ignore_above` on keyword dimensions {es-pull}110337[#110337] - -Vector Search:: -* Adding hamming distance function to painless for `dense_vector` fields {es-pull}109359[#109359] -* Support k parameter for knn query {es-pull}110233[#110233] (issue: {es-issue}108473[#108473]) - -[[feature-8.15.0]] -[float] -=== New features - -Aggregations:: -* Opt `scripted_metric` out of parallelization {es-pull}109597[#109597] - -Application:: -* [Connector API] Add claim sync job endpoint {es-pull}109480[#109480] - -ES|QL:: -* ESQL: Add `ip_prefix` function {es-pull}109070[#109070] (issue: {es-issue}99064[#99064]) -* ESQL: Introduce a casting operator, `::` {es-pull}107409[#107409] -* ESQL: `top_list` aggregation {es-pull}109386[#109386] (issue: {es-issue}109213[#109213]) -* ESQL: add Arrow dataframes output format {es-pull}109873[#109873] -* Reapply "ESQL: Expose "_ignored" metadata field" {es-pull}108871[#108871] - -Infra/REST API:: -* Add a capabilities API to check node and cluster capabilities {es-pull}106820[#106820] - -Ingest Node:: -* Directly download commercial ip geolocation databases from providers {es-pull}110844[#110844] -* Mark the Redact processor as Generally Available {es-pull}110395[#110395] - -Logs:: -* Introduce logs index mode as Tech Preview {es-pull}108896[#108896] (issue: {es-issue}108896[#108896]) - -Machine Learning:: -* Add support for Azure AI Studio embeddings and completions to the inference service. {es-pull}108472[#108472] - -Mapping:: -* Add `semantic_text` field type and `semantic` query {es-pull}110338[#110338] -* Add generic fallback implementation for synthetic source {es-pull}108222[#108222] -* Add synthetic source support for `geo_shape` via fallback implementation {es-pull}108881[#108881] -* Add synthetic source support for binary fields {es-pull}107549[#107549] -* Enable fallback synthetic source by default {es-pull}109370[#109370] (issue: {es-issue}106460[#106460]) -* Enable fallback synthetic source for `point` and `shape` {es-pull}109312[#109312] -* Enable fallback synthetic source for `token_count` {es-pull}109044[#109044] -* Implement synthetic source support for annotated text field {es-pull}107735[#107735] -* Implement synthetic source support for range fields {es-pull}107081[#107081] -* Support arrays in fallback synthetic source implementation {es-pull}108878[#108878] -* Support synthetic source for `aggregate_metric_double` when ignore_malf… {es-pull}108746[#108746] - -Ranking:: -* Add text similarity reranker retriever {es-pull}109813[#109813] - -Relevance:: -* Mark Query Rules as GA {es-pull}110004[#110004] - -Search:: -* Add new int4 quantization to dense_vector {es-pull}109317[#109317] -* Adding RankFeature search phase implementation {es-pull}108538[#108538] -* Adding aggregations support for the `_ignored` field {es-pull}101373[#101373] (issue: {es-issue}59946[#59946]) -* Update Lucene version to 9.11 {es-pull}109219[#109219] - -Security:: -* Query Roles API {es-pull}108733[#108733] - -Transform:: -* Introduce _transform/_node_stats API {es-pull}107279[#107279] - -Vector Search:: -* Adds new `bit` `element_type` for `dense_vectors` {es-pull}110059[#110059] - -[[upgrade-8.15.0]] -[float] -=== Upgrades - -Infra/Plugins:: -* Update ASM to 9.7 for plugin scanner {es-pull}108822[#108822] (issue: {es-issue}108776[#108776]) - -Ingest Node:: -* Bump Tika dependency to 2.9.2 {es-pull}108144[#108144] - -Network:: -* Upgrade to Netty 4.1.109 {es-pull}108155[#108155] - -Search:: -* Upgrade to Lucene-9.11.1 {es-pull}110234[#110234] - -Security:: -* Upgrade bouncy castle (non-fips) to 1.78.1 {es-pull}108223[#108223] - -Snapshot/Restore:: -* Bump jackson version in modules:repository-azure {es-pull}109717[#109717] - - diff --git a/docs/reference/release-notes/8.15.1.asciidoc b/docs/reference/release-notes/8.15.1.asciidoc deleted file mode 100644 index 7c48f457e3b4e..0000000000000 --- a/docs/reference/release-notes/8.15.1.asciidoc +++ /dev/null @@ -1,103 +0,0 @@ -[[release-notes-8.15.1]] -== {es} version 8.15.1 - -Also see <>. - -[[known-issues-8.15.1]] -[float] -=== Known issues -* Elasticsearch will not start if custom role mappings are configured using the -`xpack.security.authc.realms.*.files.role_mapping` configuration option. As a workaround, custom role mappings -can be configured using the https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-put-role-mapping.html[REST API] (issue: {es-issue}112503[#112503]) - -* ES|QL queries can lead to node crashes due to Out Of Memory errors when: -** Multiple indices match the query pattern -** These indices have many conflicting field mappings -** Many of those fields are included in the request -These issues deplete heap memory, increasing the likelihood of OOM errors. (issue: {es-issue}111964[#111964], {es-issue}111358[#111358]). -In Kibana, you might indirectly execute these queries when using Discover, or adding a Field Statistics panel to a dashboard. -+ -To work around this issue, you have a number of options: -** Downgrade to an earlier version -** Upgrade to 8.15.2 upon release -** Follow the instructions to -<> -** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts. - -* Index Stats, Node Stats and Cluster Stats API can return a null pointer exception if an index contains a `dense_vector` field -but there is an index segment that does not contain any documents with a dense vector field ({es-pull}112720[#112720]). Workarounds: -** If the affected index already contains documents with a dense vector field, force merge the index to a single segment. -** If the affected index does not already contain documents with a dense vector field, index a document with a dense vector field -and then force merge to a single segment. -** If the affected index's `dense_vector` fields are unused, reindex without the `dense_vector` fields. - -* Synthetic source bug. Synthetic source may fail generating the _source at runtime, causing failures in get APIs or -partial failures in the search APIs. The result is that for the affected documents the _source can't be retrieved. -There is no workaround and the only option to is to upgrade to 8.15.2 when released. -+ -If you use synthetic source then you may be affected by this bug if the following is true: -** If you have more fields then the `index.mapping.total_fields.limit` setting allows. -** If you use dynamic mappings and the `index.mapping.total_fields.ignore_dynamic_beyond_limit` setting is enabled. - -[[bug-8.15.1]] -[float] -=== Bug fixes - -Aggregations:: -* Revert "Avoid bucket copies in Aggs" {es-pull}111758[#111758] (issue: {es-issue}111679[#111679]) - -Authorization:: -* Fix DLS over Runtime Fields {es-pull}112260[#112260] (issue: {es-issue}111637[#111637]) - -ES|QL:: -* Avoid losing error message in failure collector {es-pull}111983[#111983] (issue: {es-issue}111894[#111894]) -* Avoid wrapping rejection exception in exchange {es-pull}112178[#112178] (issue: {es-issue}112106[#112106]) -* ESQL: Fix for overzealous validation in case of invalid mapped fields {es-pull}111475[#111475] (issue: {es-issue}111452[#111452]) - -Geo:: -* Add maximum nested depth check to WKT parser {es-pull}111843[#111843] -* Always check `crsType` when folding spatial functions {es-pull}112090[#112090] (issue: {es-issue}112089[#112089]) -* Fix NPE when executing doc value queries over shape geometries with empty segments {es-pull}112139[#112139] - -Indices APIs:: -* Fix template alias parsing livelock {es-pull}112217[#112217] - -Infra/Core:: -* Fix windows memory locking {es-pull}111866[#111866] (issue: {es-issue}111847[#111847]) - -Ingest Node:: -* Fixing incorrect bulk request took time {es-pull}111863[#111863] (issue: {es-issue}111854[#111854]) -* Improve performance of grok pattern cycle detection {es-pull}111947[#111947] - -Logs:: -* Merge multiple ignored source entires for the same field {es-pull}111994[#111994] (issue: {es-issue}111694[#111694]) - -Machine Learning:: -* [Inference API] Move Delete inference checks to threadpool worker {es-pull}111646[#111646] - -Mapping:: -* Check for valid `parentDoc` before retrieving its previous {es-pull}112005[#112005] (issue: {es-issue}111990[#111990]) -* Fix calculation of parent offset for ignored source in some cases {es-pull}112046[#112046] -* Fix synthetic source for empty nested objects {es-pull}111943[#111943] (issue: {es-issue}111811[#111811]) -* No error when `store_array_source` is used without synthetic source {es-pull}111966[#111966] -* Prevent synthetic field loaders accessing stored fields from using stale data {es-pull}112173[#112173] (issue: {es-issue}112156[#112156]) - -Ranking:: -* Properly handle filters on `TextSimilarityRank` retriever {es-pull}111673[#111673] - -Relevance:: -* Semantic reranking should fail whenever inference ID does not exist {es-pull}112038[#112038] (issue: {es-issue}111934[#111934]) -* [Bugfix] Add `accessDeclaredMembers` permission to allow search application templates to parse floats {es-pull}111285[#111285] - -Search:: -* Explain Function Score Query {es-pull}111807[#111807] - -Security:: -* Fix "unexpected field [remote_cluster]" for CCS (RCS 1.0) when using API key that references `remote_cluster` {es-pull}112226[#112226] -* Fix connection timeout for `OpenIdConnectAuthenticator` get Userinfo {es-pull}112230[#112230] - -Vector Search:: -* Fix `NullPointerException` when doing knn search on empty index without dims {es-pull}111756[#111756] (issue: {es-issue}111733[#111733]) -* Speed up dense/sparse vector stats {es-pull}111729[#111729] (issue: {es-issue}111715[#111715]) - - diff --git a/docs/reference/release-notes/8.15.2.asciidoc b/docs/reference/release-notes/8.15.2.asciidoc deleted file mode 100644 index 7dfd8690109b2..0000000000000 --- a/docs/reference/release-notes/8.15.2.asciidoc +++ /dev/null @@ -1,42 +0,0 @@ -[[release-notes-8.15.2]] -== {es} version 8.15.2 - -Also see <>. - -[[bug-8.15.2]] -[float] -=== Bug fixes - -Authorization:: -* Fix remote cluster credential secure settings reload {es-pull}111535[#111535] - -ES|QL:: -* ESQL: Don't mutate the `BoolQueryBuilder` in plan {es-pull}111519[#111519] -* ES|QL: Fix `ResolvedEnrichPolicy` serialization (bwc) in v 8.15 {es-pull}112985[#112985] (issue: {es-issue}112968[#112968]) -* Fix union-types where one index is missing the field {es-pull}111932[#111932] (issue: {es-issue}111912[#111912]) -* Support widening of numeric types in union-types {es-pull}112610[#112610] (issue: {es-issue}111277[#111277]) - -Infra/Core:: -* JSON parse failures should be 4xx codes {es-pull}112703[#112703] -* Json parsing exceptions should not cause 500 errors {es-pull}111548[#111548] (issue: {es-issue}111542[#111542]) -* Make sure file accesses in `DnRoleMapper` are done in stack frames with permissions {es-pull}112400[#112400] - -Ingest Node:: -* Fix missing header in `put_geoip_database` JSON spec {es-pull}112581[#112581] - -Logs:: -* Fix encoding of dynamic arrays in ignored source {es-pull}112713[#112713] - -Mapping:: -* Full coverage of ECS by ecs@mappings when `date_detection` is disabled {es-pull}112444[#112444] (issue: {es-issue}112398[#112398]) - -Search:: -* Fix parsing error in `_terms_enum` API {es-pull}112872[#112872] (issue: {es-issue}94378[#94378]) - -Security:: -* Allowlist `tracestate` header on remote server port {es-pull}112649[#112649] - -Vector Search:: -* Fix NPE in `dense_vector` stats {es-pull}112720[#112720] - - diff --git a/docs/reference/release-notes/8.16.0.asciidoc b/docs/reference/release-notes/8.16.0.asciidoc deleted file mode 100644 index 7b2e7459be968..0000000000000 --- a/docs/reference/release-notes/8.16.0.asciidoc +++ /dev/null @@ -1,8 +0,0 @@ -[[release-notes-8.16.0]] -== {es} version 8.16.0 - -coming[8.16.0] - -Also see <>. - - diff --git a/docs/reference/release-notes/8.17.0.asciidoc b/docs/reference/release-notes/8.17.0.asciidoc deleted file mode 100644 index 59962fd83e9b7..0000000000000 --- a/docs/reference/release-notes/8.17.0.asciidoc +++ /dev/null @@ -1,8 +0,0 @@ -[[release-notes-8.17.0]] -== {es} version 8.17.0 - -coming[8.17.0] - -Also see <>. - - diff --git a/docs/reference/release-notes/8.2.0.asciidoc b/docs/reference/release-notes/8.2.0.asciidoc deleted file mode 100644 index 1305845ff0e0b..0000000000000 --- a/docs/reference/release-notes/8.2.0.asciidoc +++ /dev/null @@ -1,350 +0,0 @@ -[[release-notes-8.2.0]] -== {es} version 8.2.0 - -// Also see <>. -[[known-issues-8.2.0]] -[float] -=== Known issues - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] -[[bug-8.2.0]] -[float] -=== Bug fixes - -Aggregations:: -* Don't apply the rewrite-as-range optimization if field is multivalued {es-pull}84535[#84535] (issue: {es-issue}82903[#82903]) -* Fix `AdaptingAggregator` `toString` method {es-pull}86042[#86042] -* Fix: nested top metrics sort on keyword field {es-pull}85058[#85058] -* Fix: use the correct field name when reading data from multi fields {es-pull}84752[#84752] - -Analysis:: -* Add tests for `min_hash` configuration and fix settings names {es-pull}84753[#84753] (issue: {es-issue}84578[#84578]) - -Authorization:: -* Add delete privilege to `kibana_system` for APM {es-pull}85085[#85085] -* Ensure API key can only see itself with `QueryApiKey` API {es-pull}84859[#84859] -* Fix ownership of refresh tokens {es-pull}85010[#85010] -* Grant `kibana_system` role read access to APM data streams {es-pull}85744[#85744] -* Handle role descriptor retrieval for internal users {es-pull}85049[#85049] -* Ignore app priv failures when resolving superuser {es-pull}85519[#85519] - -EQL:: -* Clean any used memory by the sequence matcher and circuit breaker used bytes in case of exception {es-pull}84451[#84451] - -Engine:: -* Increase store ref before snapshotting index commit {es-pull}84776[#84776] - -Geo:: -* Fix fields wildcard support to vector tile search API {es-pull}85595[#85595] (issue: {es-issue}85592[#85592]) - -Highlighting:: -* Fix wildcard highlighting on `match_only_text` {es-pull}85500[#85500] (issue: {es-issue}85493[#85493]) - -ILM+SLM:: -* Fix Stacktraces Taking Memory in ILM Error Step Serialization {es-pull}84266[#84266] -* Invoke initial `AsyncActionStep` for newly created indices {es-pull}84541[#84541] (issue: {es-issue}77269[#77269]) -* Retry clean and create snapshot if it already exists #83694 {es-pull}84829[#84829] (issue: {es-issue}83694[#83694]) -* Skip the shrink step if the number of shards of the shrunk index is the same with the original {es-pull}84434[#84434] (issue: {es-issue}80180[#80180]) - -Indices APIs:: -* Remove existing indices/datastreams/aliases before simulating index template {es-pull}84675[#84675] (issue: {es-issue}84256[#84256]) - -Infra/Core:: -* Fix `NullPointerException` in `SystemIndexMetadataUpgradeService` hidden alias handling {es-pull}84780[#84780] (issue: {es-issue}81411[#81411]) -* Prevent `ThreadContext` header leak when sending response {es-pull}68649[#68649] (issue: {es-issue}68278[#68278]) -* Relax data path deprecations from critical to warn {es-pull}85952[#85952] -* Require and preserve content type for filtered rest requests {es-pull}84914[#84914] (issue: {es-issue}84784[#84784]) -* Return empty version instead of blowing up if we cannot find it {es-pull}85244[#85244] -* Validate index format agreement for system index descriptors {es-pull}85173[#85173] -* Wrap thread creation in `doPrivileged` call {es-pull}85180[#85180] - -Infra/Plugins:: -* Strengthen elasticsearch plugin version check {es-pull}85340[#85340] (issue: {es-issue}85336[#85336]) - -Infra/REST API:: -* Correctly return `_type` field for documents in V7 compatiblity mode {es-pull}84873[#84873] (issue: {es-issue}84173[#84173]) - -Ingest:: -* Mark `GeoIpDownloaderTask` as completed after cancellation {es-pull}84028[#84028] -* `CompoundProcessor` should also catch exceptions when executing a processor {es-pull}84838[#84838] (issue: {es-issue}84781[#84781]) - -License:: -* Fix license downgrade warnings for API Keys and Tokens {es-pull}85276[#85276] (issue: {es-issue}75271[#75271]) - -Machine Learning:: -* Allow retrieving `boolean` fields from `_source` in DFA jobs {es-pull}85672[#85672] -* Avoid multiple queued quantiles documents in renormalizer {es-pull}85555[#85555] (issue: {es-issue}85539[#85539]) -* Disallow new trained model deployments when nodes are different versions {es-pull}85465[#85465] -* Do not fetch source when finding index of last state docs {es-pull}85334[#85334] -* Ensure that inference index primary shards are available before attempting to load model {es-pull}85569[#85569] -* Fix Kibana date format and similar overrides in text structure endpoint {es-pull}84967[#84967] -* Fix race condition when stopping a recently relocated datafeed {es-pull}84636[#84636] -* Fix serialisation of text embedding updates {es-pull}85863[#85863] -* Fixes for multi-line start patterns in text structure endpoint {es-pull}85066[#85066] -* Fixes to making old ML indices hidden {es-pull}85383[#85383] -* Reallocate model deployments on node shutdown events {es-pull}85310[#85310] -* Retry datafeed searches on skipped CCS clusters {es-pull}84052[#84052] (issue: {es-issue}83838[#83838]) -* Return all Datafeeds with GET Anomaly Detector {es-pull}84759[#84759] - -Mapping:: -* Do not fail on duplicated content field filters {es-pull}85382[#85382] -* Runtime fields core-with-mapped tests support tsdb {es-pull}83577[#83577] - -Packaging:: -* Remove use of Cloudflare zlib {es-pull}84680[#84680] - -Rollup:: -* Add support for comma delimited index patterns to rollup job configuration {es-pull}47041[#47041] (issue: {es-issue}45591[#45591]) - -SQL:: -* Add range checks to interval multiplication operation {es-pull}83478[#83478] (issue: {es-issue}83336[#83336]) -* Avoid empty last pages for GROUP BY queries when possible {es-pull}84356[#84356] (issue: {es-issue}75528[#75528]) -* Fix SQLCompatIT.testCursorFromOldNodeFailsOnNewNode {es-pull}85531[#85531] (issue: {es-issue}85520[#85520]) -* Fix issues with format=txt when paging through result sets and in mixed node environments {es-pull}83833[#83833] (issues: {es-issue}83581[#83581], {es-issue}83788[#83788]) -* Improve ROUND and TRUNCATE to better manage Long values and big Doubles {es-pull}85106[#85106] (issues: {es-issue}85105[#85105], {es-issue}49391[#49391]) -* Use exact attributes for script templates from scalar functions {es-pull}84813[#84813] (issue: {es-issue}80551[#80551]) -* `RANDOM()` always evaluates to `NULL` if `` is `NULL` {es-pull}84632[#84632] (issue: {es-issue}84627[#84627]) - -Search:: -* Fix point visitor in `DiskUsage` API {es-pull}84909[#84909] -* Fix skip caching factor with `indices.queries.cache.all_segments` {es-pull}85510[#85510] -* Increase store ref before analyzing disk usage {es-pull}84774[#84774] -* Limit concurrent shard requests in disk usage API {es-pull}84900[#84900] (issue: {es-issue}84779[#84779]) -* Rewrite `match_all` inside `must_not` {es-pull}85999[#85999] -* `DotExpandingXContentParser` to expose the original token location {es-pull}84970[#84970] -* `TransportBroadcastAction` should always set response for each shard {es-pull}84926[#84926] - -Security:: -* `Authentication.token` now uses version from the existing authentication {es-pull}85978[#85978] -* Ensure tokens represent effective user's identity in all cases {es-pull}84263[#84263] - -Snapshot/Restore:: -* Don't fail if there is no symlink for AWS Web Identity Token {es-pull}84697[#84697] -* Expose proxy settings for GCS repositories {es-pull}85785[#85785] (issue: {es-issue}84569[#84569]) -* Fix atomic writes in HDFS {es-pull}85210[#85210] -* Fix leaking listeners bug on frozen tier {es-pull}85239[#85239] -* Fix snapshot status messages on node-left {es-pull}85021[#85021] -* Ignore frozen shared cache file during data folder upgrades {es-pull}85638[#85638] (issue: {es-issue}85603[#85603]) -* [s3-repository] Lookup AWS Region for STS Client from STS endpoint {es-pull}84585[#84585] (issue: {es-issue}83826[#83826]) - -Stats:: -* Discard intermediate results upon cancellation for stats endpoints {es-pull}82685[#82685] (issue: {es-issue}82337[#82337]) - -Transform:: -* Correctly validate permissions when retention policy is configured {es-pull}85413[#85413] (issue: {es-issue}85409[#85409]) - -Watcher:: -* Avoiding watcher validation errors when a data stream points to more than one index {es-pull}85507[#85507] (issue: {es-issue}85508[#85508]) -* Log at WARN level for Watcher cluster state validation errors {es-pull}85632[#85632] -* No longer require master node to install Watcher templates {es-pull}85287[#85287] (issue: {es-issue}85043[#85043]) - -[[enhancement-8.2.0]] -[float] -=== Enhancements - -Aggregations:: -* Aggs: no filter-by-filter if `_doc_count` field {es-pull}84427[#84427] (issue: {es-issue}84048[#84048]) -* Extract agg bounds from queries in FILTER {es-pull}83902[#83902] -* Give Lucene more opportunities to enable the filter-by-filter optimization {es-pull}85322[#85322] -* Improve performance of `date_histogram` when date histogram is in a BoostingQuery {es-pull}83751[#83751] (issues: {es-issue}82384[#82384], {es-issue}75542[#75542]) - -Allocation:: -* Make allocation explanations more actionable {es-pull}83983[#83983] -* Use static empty store files metadata {es-pull}84034[#84034] - -Audit:: -* User Profile - Audit security config change for profile APIs {es-pull}84785[#84785] - -Authentication:: -* Adds domain information to authentication object {es-pull}82639[#82639] -* Improve BWC for persisted authentication headers {es-pull}83913[#83913] (issue: {es-issue}83567[#83567]) -* Warn on SAML attributes with special attribute names {es-pull}85248[#85248] (issue: {es-issue}48613[#48613]) - -Authorization:: -* Add elastic/enterprise-search-server service account {es-pull}83325[#83325] -* Add index privileges for logs-enterprise_search.api-default to the enterprise-search-server service account {es-pull}84965[#84965] -* Add indices permissions to Enterprise Search service account {es-pull}85726[#85726] -* Note restricted indices in access denied message {es-pull}85013[#85013] -* Security global privilege for updating profile data of applications {es-pull}83728[#83728] -* [Osquery] Extend `kibana_system` role with an access to `osquery_manager` indices {es-pull}84279[#84279] - -CRUD:: -* Speed up Reading `RetentionLeases` from the Wire {es-pull}85159[#85159] - -Cluster Coordination:: -* Avoid deserializing cluster states on master {es-pull}58416[#58416] -* Improve logging for connect-back failures {es-pull}84915[#84915] -* Remove intermediate map from master task execution {es-pull}84406[#84406] -* Reuse `JoinTaskExecutor` {es-pull}85325[#85325] -* Speed up `MetadataStateFormat` Writes {es-pull}85138[#85138] - -Data streams:: -* Speed up `DatastreamTimestampFieldMapper#postParse` {es-pull}85270[#85270] - -Discovery-Plugins:: -* Support IMDSv2 for EC2 Discovery {es-pull}84410[#84410] (issue: {es-issue}80398[#80398]) - -Distributed:: -* Add elasticsearch health API {es-pull}83119[#83119] - -Geo:: -* Add `geohex_grid` aggregation to vector tiles API {es-pull}84553[#84553] -* Added buffer pixels to vector tile spec parsing {es-pull}84710[#84710] (issue: {es-issue}84492[#84492]) -* Normalise polygons only when necessary {es-pull}84229[#84229] (issue: {es-issue}35349[#35349]) -* Support GeoJSON for `geo_point` {es-pull}85120[#85120] - -Health:: -* Fix naming in health indicators {es-pull}83587[#83587] -* ILM/SLM health indicator services {es-pull}83440[#83440] -* Introduce dedicated interface for health indicator details {es-pull}83417[#83417] -* Repository integrity health indicator services {es-pull}83445[#83445] -* Shards allocation health indicator services {es-pull}83513[#83513] - -ILM+SLM:: -* Cache ILM policy name on `IndexMetadata` {es-pull}83603[#83603] (issue: {es-issue}83582[#83582]) -* GET _index_template and GET _component_template request support query parameter flat_settings {es-pull}83297[#83297] -* Make rollover cancellable #81763 {es-pull}84584[#84584] (issue: {es-issue}81763[#81763]) -* Rollover add max_primary_shard_docs condition {es-pull}80981[#80981] -* Speed up ILM cluster task execution {es-pull}85405[#85405] (issue: {es-issue}82708[#82708]) - -Indices APIs:: -* Batch add index block cluster state updates {es-pull}84374[#84374] -* Batch close-indices cluster state updates {es-pull}84259[#84259] -* Batch open-indices cluster state updates {es-pull}83760[#83760] -* Remove LegacyCTRAL from `TransportRolloverAction` {es-pull}84166[#84166] - -Infra/Core:: -* Add support for negtive epoch timestamps {es-pull}80208[#80208] (issues: {es-issue}79135[#79135], {es-issue}72123[#72123], {es-issue}40983[#40983]) -* Allow yaml values for dynamic node settings {es-pull}85186[#85186] (issue: {es-issue}65577[#65577]) -* Improve XContent Array Parser {es-pull}84477[#84477] -* Optimize `ImmutableOpenMap.Builder` {es-pull}85184[#85184] -* Provide 'system' attribute when resolving system indices {es-pull}85042[#85042] (issue: {es-issue}82671[#82671]) -* Remove Lucene split packages {es-pull}82132[#82132] (issue: {es-issue}81981[#81981]) -* Simplify reading a list and converting it to a map from stream {es-pull}84183[#84183] -* Speed up CompressedXContent Serialization {es-pull}84802[#84802] -* Update `readMap` to avoid resizing map during reading {es-pull}84045[#84045] - -Infra/Plugins:: -* Warn on slow signature verification {es-pull}84766[#84766] (issue: {es-issue}80480[#80480]) - -Infra/Scripting:: -* Script: Fields API for Dense Vector {es-pull}83550[#83550] - -Ingest:: -* Do not throw exceptions when resolving paths in ingest documents {es-pull}84659[#84659] -* RemoveProcessor updated to support fieldsToKeep {es-pull}83665[#83665] - -Machine Learning:: -* Add ML memory stats API {es-pull}83802[#83802] -* Add support for RoBERTa and BART NLP models {es-pull}84777[#84777] -* Add throughput stats for Trained Model Deployments {es-pull}84628[#84628] -* Improve `zero_shot_classification` tokenization performance {es-pull}84988[#84988] (issue: {es-issue}84820[#84820]) - -Mapping:: -* Check the utf8 length of keyword field is not bigger than 32766 in ES, rather than in Lucene. {es-pull}83738[#83738] (issue: {es-issue}80865[#80865]) -* Make `FieldMapper.Param` Cheaper to Construct {es-pull}85191[#85191] -* Terms enum support for doc value only keyword fields {es-pull}83482[#83482] (issue: {es-issue}83451[#83451]) - -Network:: -* Use Throttling Netty Write Handler on HTTP Path {es-pull}84751[#84751] - -Query Languages:: -* Add `unsigned_long` type support {es-pull}65145[#65145] (issue: {es-issue}63312[#63312]) - -Recovery:: -* Improve failure logging in recovery-from-snapshot {es-pull}84910[#84910] - -Reindex:: -* Use `SecureString` for reindex from remote password {es-pull}85091[#85091] - -SQL:: -* Add leniency option to SQL CLI {es-pull}83795[#83795] (issue: {es-issue}67436[#67436]) -* Forward warning headers to JDBC driver {es-pull}84499[#84499] -* List data streams as VIEWs {es-pull}85168[#85168] (issue: {es-issue}83449[#83449]) -* PIT for `GROUP BY` and `PIVOT` queries {es-pull}84605[#84605] (issue: {es-issue}84349[#84349]) -* Replace scroll cursors with point-in-time and `search_after` {es-pull}83381[#83381] (issues: {es-issue}61873[#61873], {es-issue}80523[#80523]) - -Search:: -* Add filtering to fieldcaps endpoint {es-pull}83636[#83636] (issue: {es-issue}82966[#82966]) -* Group field caps response by index mapping hash {es-pull}83494[#83494] (issues: {es-issue}78665[#78665], {es-issue}82879[#82879]) -* Integrate filtering support for ANN {es-pull}84734[#84734] (issue: {es-issue}81788[#81788]) -* Speed up merging field-caps response {es-pull}83704[#83704] - -Security:: -* Bind host all instead of just _site_ when needed {es-pull}83145[#83145] -* Fleet: Add a new mapping for .fleet-actions-results `action_input_type` field {es-pull}84316[#84316] -* Update X509Certificate principal methods {es-pull}85163[#85163] (issue: {es-issue}81008[#81008]) -* User Profile - Add APIs for enable/disable profile {es-pull}84548[#84548] -* User Profile - Add rest spec files and tests {es-pull}83307[#83307] -* User Profile - More REST spec, tests, API docs {es-pull}84597[#84597] -* User Profile - Update APIs to work with domain {es-pull}83570[#83570] -* User Profile - Update xpack usage output for domains {es-pull}84747[#84747] -* User Profile - capture domain when creating API keys and tokens {es-pull}84547[#84547] -* User Profile: Add feature flag {es-pull}83347[#83347] -* User Profile: Add initial search profile API {es-pull}83191[#83191] -* User Profile: handle racing on creating new profile {es-pull}84208[#84208] - -TSDB:: -* TSDB: Expand `_id` on version conflict {es-pull}84957[#84957] -* TSDB: Reject the nested object fields that are configured time_series_dimension {es-pull}83920[#83920] -* TSDB: routingPath object type check improvement {es-pull}83310[#83310] -* TSDB: shrink `_id` inverted index {es-pull}85008[#85008] - -Watcher:: -* Add list of allowed domains for Watcher email action {es-pull}84894[#84894] (issue: {es-issue}84739[#84739]) - -[[feature-8.2.0]] -[float] -=== New features - -Aggregations:: -* New `random_sampler` aggregation for sampling documents in aggregations {es-pull}84363[#84363] - -Authentication:: -* Add JWT realm support for JWT validation {es-pull}83155[#83155] -* Add smoke test for JWT realm wiring {es-pull}84249[#84249] -* Support mail, name, and dn claims in JWT realms {es-pull}84907[#84907] - -Authorization:: -* API Key APIs with Security Domain {es-pull}84704[#84704] - -Health:: -* Add Health Indicator Plugin {es-pull}83205[#83205] -* Adding impacts block to the health info API response {es-pull}84899[#84899] (issue: {es-issue}84773[#84773]) - -Indices APIs:: -* Adding cat api for component template {es-pull}71274[#71274] (issue: {es-issue}68941[#68941]) - -Infra/Core:: -* Introduce an unauthenticated endpoint for readiness checks {es-pull}84375[#84375] (issue: {es-issue}81168[#81168]) - -Machine Learning:: -* Adds new `change_point` pipeline aggregation {es-pull}83428[#83428] - -Search:: -* Introduce lookup runtime fields {es-pull}82385[#82385] -* Resolve wildcards in disk usage API {es-pull}84832[#84832] - -TSDB:: -* TSDB: Support GET and DELETE and doc versioning {es-pull}82633[#82633] - -[[upgrade-8.2.0]] -[float] -=== Upgrades - -Infra/Core:: -* Upgrade jackson for x-content to 2.13.2 {es-pull}84905[#84905] - -Ingest:: -* Bump commons-compress to 1.21 {es-pull}85581[#85581] - -Network:: -* Upgrade Netty to 4.1.74 {es-pull}84562[#84562] - -Packaging:: -* Upgrade to JDK 18.0.0+36 {es-pull}85376[#85376] (issue: {es-issue}85357[#85357]) - -Search:: -* Upgrade to lucene 9.1.0-snapshot-5b522487ba8 {es-pull}85025[#85025] - - diff --git a/docs/reference/release-notes/8.2.1.asciidoc b/docs/reference/release-notes/8.2.1.asciidoc deleted file mode 100644 index 309488d48d3e7..0000000000000 --- a/docs/reference/release-notes/8.2.1.asciidoc +++ /dev/null @@ -1,73 +0,0 @@ -[[release-notes-8.2.1]] -== {es} version 8.2.1 - -Also see <>. -[[known-issues-8.2.1]] -[float] -=== Known issues - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] -[[bug-8.2.1]] -[float] -=== Bug fixes - -Aggregations:: -* Fix `AdaptingAggregator` `toString` method {es-pull}86042[#86042] -* Less complexity in after-key parsing for unmapped fields {es-pull}86359[#86359] (issue: {es-issue}85928[#85928]) - -Authentication:: -* Ensure authentication is wire compatible when setting user {es-pull}86741[#86741] (issue: {es-issue}86716[#86716]) - -Cluster Coordination:: -* Avoid breaking add/clear voting exclusions {es-pull}86657[#86657] - -Geo:: -* Fix bounded hexagonal grids when they contain the bin on one of the poles {es-pull}86460[#86460] -* Fix mvt polygon orientation {es-pull}86555[#86555] (issue: {es-issue}86560[#86560]) - -ILM+SLM:: -* Fix `max_primary_shard_size` resize factor math {es-pull}86897[#86897] -* Reroute after migrating to data tiers routing {es-pull}86574[#86574] (issue: {es-issue}86572[#86572]) - -Infra/Core:: -* Fix `assertDefaultThreadContext` enumerating allowed headers {es-pull}86262[#86262] -* Forward port MDP deprecation info API {es-pull}86103[#86103] -* Make data directories work with symlinks again {es-pull}85878[#85878] (issue: {es-issue}85701[#85701]) -* Set autoexpand replicas on Fleet actions data stream {es-pull}85511[#85511] -* Do not autocreate alias for non-primary system indices {es-pull}85977[#85977] (issue: {es-issue}85072[#85072]) - -Ingest:: -* Don't download geoip databases if geoip system index is blocked {es-pull}86842[#86842] -* Fix NPE when using object field as match field for enrich policy {es-pull}86089[#86089] (issue: {es-issue}86058[#86058]) -* Handle `.geoip_databases` being an alias or a concrete index {es-pull}85792[#85792] (issue: {es-issue}85756[#85756]) - -Machine Learning:: -* Adjust memory overhead for `PyTorch` models {es-pull}86416[#86416] -* Fix `max_model_memory_limit` reported by `_ml/info` when autoscaling is enabled {es-pull}86660[#86660] -* Improve reliability of job stats in larger clusters {es-pull}86305[#86305] -* Make autoscaling and task assignment use same memory staleness definition {es-pull}86632[#86632] (issue: {es-issue}86616[#86616]) -* Fix edge case which could cause the model bounds to inflate after detecting seasonality {ml-pull}2261[#2261] - -Packaging:: -* Fix edge case where user-defined heap settings are ignored {es-pull}86438[#86438] (issue: {es-issue}86431[#86431]) - -Security:: -* Authentication.token now uses version from the existing authentication {es-pull}85978[#85978] - -Snapshot/Restore:: -* Better failure for source-only snapshots of partially/fully mounted indices {es-pull}86207[#86207] -* Check if searchable snapshots cache pre-allocation is successful in Windows {es-pull}86192[#86192] (issue: {es-issue}85725[#85725]) -* Delay searchable snapshot allocation during shutdown {es-pull}86153[#86153] (issue: {es-issue}85052[#85052]) -* Support generating AWS role session name in case it's not provided {es-pull}86255[#86255] - -Stats:: -* Correctly calculate disk usage for frozen data tier telemetry {es-pull}86580[#86580] (issue: {es-issue}86055[#86055]) - -[[upgrade-8.2.1]] -[float] -=== Upgrades - -Packaging:: -* Switch to OpenJDK and upgrade to 18.0.1 {es-pull}86554[#86554] - - diff --git a/docs/reference/release-notes/8.2.2.asciidoc b/docs/reference/release-notes/8.2.2.asciidoc deleted file mode 100644 index 28647c00cbb0c..0000000000000 --- a/docs/reference/release-notes/8.2.2.asciidoc +++ /dev/null @@ -1,35 +0,0 @@ -[[release-notes-8.2.2]] -== {es} version 8.2.2 - -Also see <>. -[[known-issues-8.2.2]] -[float] -=== Known issues - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] -[[bug-8.2.2]] -[float] -=== Bug fixes - -Audit:: -* Fix audit logging to consistently include port number in `origin.address` {es-pull}86732[#86732] - -CCR:: -* Fix CCR following a datastream with closed indices on the follower corrupting the datastream {es-pull}87076[#87076] (issue: {es-issue}87048[#87048]) - -Geo:: -* Guard for adding null value tags to vector tiles {es-pull}87051[#87051] - -Infra/Core:: -* Adjust osprobe assertion for burst cpu {es-pull}86990[#86990] - -Machine Learning:: -* Fix ML task auditor exception early in cluster lifecycle {es-pull}87023[#87023] (issue: {es-issue}87002[#87002]) -* Adjacency weighting fixes in categorization {ml-pull}2277[#2277] - -[[enhancement-8.2.2]] -[float] -=== Enhancements - -Machine Learning:: -* Make ML native processes work with glibc 2.35 (required for Ubuntu 22.04) {ml-pull}2272[#2272] diff --git a/docs/reference/release-notes/8.2.3.asciidoc b/docs/reference/release-notes/8.2.3.asciidoc deleted file mode 100644 index f4742e38b1e20..0000000000000 --- a/docs/reference/release-notes/8.2.3.asciidoc +++ /dev/null @@ -1,39 +0,0 @@ -[[release-notes-8.2.3]] -== {es} version 8.2.3 - -Also see <>. -[[known-issues-8.2.3]] -[float] -=== Known issues - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] -[[bug-8.2.3]] -[float] -=== Bug fixes - -Authorization:: -* Fix resolution of wildcard application privileges {es-pull}87293[#87293] - -CCR:: -* Remove some blocking in CcrRepository {es-pull}87235[#87235] - -Indices APIs:: -* Add Resolve Index API to the "read" permission for an index {es-pull}87052[#87052] (issue: {es-issue}86977[#86977]) - -Infra/Core:: -* Clean up `DeflateCompressor` after exception {es-pull}87163[#87163] (issue: {es-issue}87160[#87160]) - -Security:: -* Security plugin close releasable realms {es-pull}87429[#87429] (issue: {es-issue}86286[#86286]) - -Snapshot/Restore:: -* Fork after calling `getRepositoryData` from `StoreRecovery` {es-pull}87254[#87254] (issue: {es-issue}87237[#87237]) - -[[enhancement-8.2.3]] -[float] -=== Enhancements - -Infra/Core:: -* Force property expansion for security policy {es-pull}87396[#87396] - - diff --git a/docs/reference/release-notes/8.3.0.asciidoc b/docs/reference/release-notes/8.3.0.asciidoc deleted file mode 100644 index 17b5edcbed392..0000000000000 --- a/docs/reference/release-notes/8.3.0.asciidoc +++ /dev/null @@ -1,367 +0,0 @@ -[[release-notes-8.3.0]] -== {es} version 8.3.0 - -Also see <>. -[[known-issues-8.3.0]] -[float] -=== Known issues - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] -[[bug-8.3.0]] -[float] -=== Bug fixes - -Aggregations:: -* Allow `serial_diff` under `min_doc_count` aggs {es-pull}86401[#86401] -* Allow bucket paths to specify `_count` within a bucket {es-pull}85720[#85720] -* Fix a bug with flattened fields in terms aggregations {es-pull}87392[#87392] -* Fix flaky `top_metrics` test {es-pull}86582[#86582] (issue: {es-issue}86377[#86377]) -* Fix: check field existence before trying to merge running stats {es-pull}86926[#86926] -* Fix: ordering terms aggregation on top metrics null values {es-pull}85774[#85774] -* Serialize interval in auto date histogram aggregation {es-pull}85473[#85473] - -Audit:: -* Fix audit logging to consistently include port number in `origin.address` {es-pull}86732[#86732] -* Support removing ignore filters for audit logging {es-pull}87675[#87675] (issue: {es-issue}68588[#68588]) - -Authentication:: -* An authorized user can disable a user with same name but different realm {es-pull}86473[#86473] -* Fix clearing of `lastSuccessfulAuthCache` when clear all realm cache API is called {es-pull}86909[#86909] (issue: {es-issue}86650[#86650]) - -Authorization:: -* Fix resolution of wildcard application privileges {es-pull}87293[#87293] - -CAT APIs:: -* Get hidden indices stats in `GET _cat/shards` {es-pull}86601[#86601] (issue: {es-issue}84656[#84656]) - -CCR:: -* Prevent invalid datastream metadata when CCR follows a datastream with closed indices on the follower {es-pull}87076[#87076] (issue: {es-issue}87048[#87048]) -* Remove some blocking in CcrRepository {es-pull}87235[#87235] - -Cluster Coordination:: -* Add `master_timeout` support to voting config exclusions APIs {es-pull}86670[#86670] -* Small fixes to clear voting config excls API {es-pull}87828[#87828] - -Discovery-Plugins:: -* [discovery-gce] Fix initialisation of transport in FIPS mode {es-pull}85817[#85817] (issue: {es-issue}85803[#85803]) - -Distributed:: -* Enforce external id uniqueness during `DesiredNode` construction {es-pull}84227[#84227] - -Engine:: -* Fork to WRITE thread before failing shard in `updateCheckPoints` {es-pull}87458[#87458] (issue: {es-issue}87094[#87094]) -* Removing Blocking Wait for Close in `RecoverySourceHandler` {es-pull}86127[#86127] (issue: {es-issue}85839[#85839]) - -Features:: -* Fix 'autoGeneratedTimestamp should not be set externally' error when retrying IndexRequest {es-pull}86184[#86184] (issue: {es-issue}83927[#83927]) - -Geo:: -* Fix Geotile aggregations on `geo_shapes` for precision 0 {es-pull}87202[#87202] (issue: {es-issue}87201[#87201]) -* Fix `null_value` for array-valued `geo_point` fields {es-pull}85959[#85959] -* Guard for adding null value tags to vector tiles {es-pull}87051[#87051] -* Quantize geo queries to remove true negatives from search results {es-pull}85441[#85441] (issue: {es-issue}40891[#40891]) - -Highlighting:: -* `FastVectorHighlighter` should use `ValueFetchers` to load source data {es-pull}85815[#85815] (issues: {es-issue}75011[#75011], {es-issue}84690[#84690], {es-issue}82458[#82458], {es-issue}80895[#80895]) - -ILM+SLM:: -* Make the ILM Move to Error Step Batched {es-pull}85565[#85565] (issue: {es-issue}81880[#81880]) - -Indices APIs:: -* Make `GetIndexAction` cancellable {es-pull}87681[#87681] - -Infra/Circuit Breakers:: -* Make CBE message creation more robust {es-pull}87881[#87881] - -Infra/Core:: -* Adjust osprobe assertion for burst cpu {es-pull}86990[#86990] -* Clean up `DeflateCompressor` after exception {es-pull}87163[#87163] (issue: {es-issue}87160[#87160]) -* Error on direct creation of non-primary system index {es-pull}86707[#86707] -* Fix null message in output {es-pull}86981[#86981] -* Fix using `FilterOutputStream` without overriding bulk write {es-pull}86304[#86304] -* Hide system indices and their aliases in upgraded clusters {es-pull}87125[#87125] -* Refactor code to avoid JDK bug: JDK-8285835 {es-pull}86614[#86614] - -Infra/Logging:: -* Temporarily provide `SystemPropertiesPropertySource` {es-pull}87149[#87149] - -Infra/Node Lifecycle:: -* Upgrade folders after settings validation {es-pull}87319[#87319] - -Infra/Plugins:: -* Use Windows newlines when listing plugin information on Windows {es-pull}86408[#86408] (issue: {es-issue}86352[#86352]) - -Infra/REST API:: -* Fix min node version before state recovery {es-pull}86482[#86482] - -Infra/Scripting:: -* Allow to sort by script value using `SemVer` semantics {es-pull}85990[#85990] (issues: {es-issue}85989[#85989], {es-issue}82287[#82287]) -* Script: Fix setter shortcut for unbridged setters {es-pull}86868[#86868] -* Script: Load Whitelists as Resource {es-pull}87539[#87539] - -Infra/Settings:: -* Permit removal of archived index settings {es-pull}86107[#86107] - -Ingest:: -* Execute self-reference checks once per pipeline {es-pull}85926[#85926] (issue: {es-issue}85790[#85790]) - -Java Low Level REST Client:: -* Do not retry client requests when failing with `ContentTooLargeException` {es-pull}87248[#87248] (issue: {es-issue}86041[#86041]) - -License:: -* Consistent response for starting basic license {es-pull}86272[#86272] (issue: {es-issue}86244[#86244]) - -Machine Learning:: -* Fix ML task auditor exception early in cluster lifecycle {es-pull}87023[#87023] (issue: {es-issue}87002[#87002]) -* Fix `WordPiece` tokenization of unknown words with known subwords {es-pull}87510[#87510] -* Fix distribution change check for `change_point` aggregation {es-pull}86423[#86423] -* Fixes inference timeout handling bug that throws unexpected `NullPointerException` {es-pull}87533[#87533] -* Correct logic for restart from failover fine tuning hyperparameters for training classification and regression models {ml-pull}2251[#2251] -* Fix possible source of "x = NaN, distribution = class boost::math::normal_distribution<..." log errors training classification and regression models {ml-pull}2249[#2249] -* Fix some bugs affecting decision to stop optimizing hyperparameters for training classification and regression models {ml-pull}2259[#2259] -* Fix cause of "Must provide points at which to evaluate function" log error training classification and regression models {ml-pull}2268[#2268] -* Fix a source of "Discarding sample = nan, weights = ..." log errors for time series anomaly detection {ml-pull}2286[#2286] - -Mapping:: -* Don't run `include_in_parent` when in `copy_to` context {es-pull}87123[#87123] (issue: {es-issue}87036[#87036]) - -Network:: -* Reject `openConnection` attempt while closing {es-pull}86315[#86315] (issue: {es-issue}86249[#86249]) - -Recovery:: -* Fail shard if STARTED after master failover {es-pull}87451[#87451] (issue: {es-issue}87367[#87367]) - -SQL:: -* Fix FORMAT function to comply with Microsoft SQL Server specification {es-pull}86225[#86225] (issue: {es-issue}66560[#66560]) -* Implement binary format support for SQL clear cursor {es-pull}84230[#84230] (issue: {es-issue}53359[#53359]) - -Search:: -* Add status field to Multi Search Template Responses {es-pull}85496[#85496] (issue: {es-issue}83029[#83029]) -* Fields API to allow fetching values when `_source` is disabled {es-pull}87267[#87267] (issue: {es-issue}87072[#87072]) -* Fix `_terms_enum` on unconfigured `constant_keyword` {es-pull}86191[#86191] (issues: {es-issue}86187[#86187], {es-issue}86267[#86267]) -* Fix status code when open point in time without `keep_alive` {es-pull}87011[#87011] (issue: {es-issue}87003[#87003]) -* Handle empty point values in `DiskUsage` API {es-pull}87826[#87826] (issue: {es-issue}87761[#87761]) -* Make sure to rewrite explain query on coordinator {es-pull}87013[#87013] (issue: {es-issue}64281[#64281]) - -Security:: -* Make user and role name constraint consistent with max document ID {es-pull}86728[#86728] (issue: {es-issue}66020[#66020]) -* Security plugin close releasable realms {es-pull}87429[#87429] (issue: {es-issue}86286[#86286]) - -Snapshot/Restore:: -* DONE should mean fully processed in snapshot status {es-pull}86414[#86414] -* Distinguish missing and invalid repositories {es-pull}85551[#85551] (issue: {es-issue}85550[#85550]) -* Fork after calling `getRepositoryData` from `StoreRecovery` {es-pull}87264[#87264] (issue: {es-issue}87237[#87237]) -* Fork after calling `getRepositoryData` from `StoreRecovery` {es-pull}87254[#87254] (issue: {es-issue}87237[#87237]) -* Throw exception on illegal `RepositoryData` updates {es-pull}87654[#87654] -* Upgrade Azure SDK to 12.16.0 {es-pull}86135[#86135] - -Stats:: -* Run `TransportClusterInfoActions` on MANAGEMENT pool {es-pull}87679[#87679] - -TSDB:: -* TSDB: fix the time_series in order collect priority {es-pull}85526[#85526] -* TSDB: fix wrong initial value of tsidOrd in TimeSeriesIndexSearcher {es-pull}85713[#85713] (issue: {es-issue}85711[#85711]) - -Transform:: -* Fix transform `_start` permissions to use stored headers in the config {es-pull}86802[#86802] -* [Transforms] fix bug when unsetting retention policy {es-pull}87711[#87711] - -[[deprecation-8.3.0]] -[float] -=== Deprecations - -Authentication:: -* Configuring a bind DN in an LDAP or Active Directory (AD) realm without a corresponding bind password is deprecated {es-pull}85326[#85326] (issue: {es-issue}47191[#47191]) - -[[enhancement-8.3.0]] -[float] -=== Enhancements - -Aggregations:: -* Improve min and max performance while in a `random_sampler` aggregation {es-pull}85118[#85118] - -Authentication:: -* Support configurable claims in JWT Realm Tokens {es-pull}86533[#86533] -* Warn on user roles disabled due to licensing requirements for document or field level security {es-pull}85393[#85393] (issue: {es-issue}79207[#79207]) -* `TokenService` decode JWTs, change warn to debug {es-pull}86498[#86498] - -Authorization:: -* Add delete privilege to `kibana_system` for Synthetics {es-pull}85844[#85844] -* Authorize painless execute as index action when an index is specified {es-pull}85512[#85512] (issue: {es-issue}86428[#86428]) -* Better error message for run-as denials {es-pull}85501[#85501] (issue: {es-issue}72904[#72904]) -* Improve "Has Privilege" performance for boolean-only response {es-pull}86685[#86685] -* Relax restrictions for role names in roles API {es-pull}86604[#86604] (issue: {es-issue}86480[#86480]) -* [Osquery] Extend `kibana_system` role with an access to osquery_manager… {es-pull}86609[#86609] - -Autoscaling:: -* Add support for CPU ranges in desired nodes {es-pull}86434[#86434] - -Cluster Coordination:: -* Block joins while applier is busy {es-pull}84919[#84919] -* Compute master task batch summary lazily {es-pull}86210[#86210] -* Log `cluster.initial_master_nodes` at startup {es-pull}86101[#86101] -* Reduce resource needs of join validation {es-pull}85380[#85380] (issue: {es-issue}83204[#83204]) -* Report pending joins in `ClusterFormationFailureHelper` {es-pull}85635[#85635] -* Speed up map diffing (2) {es-pull}86375[#86375] - -Discovery-Plugins:: -* Remove redundant jackson dependencies from discovery-azure {es-pull}87898[#87898] - -Distributed:: -* Keep track of desired nodes cluster membership {es-pull}84165[#84165] - -Engine:: -* Cache immutable translog lastModifiedTime {es-pull}82721[#82721] (issue: {es-issue}82720[#82720]) -* Increase `force_merge` threadpool size based on the allocated processors {es-pull}87082[#87082] (issue: {es-issue}84943[#84943]) -* More optimal forced merges when max_num_segments is greater than 1 {es-pull}85065[#85065] - -Geo:: -* Support 'GeoJSON' in CartesianPoint for 'point' {es-pull}85442[#85442] -* Support geo label position as runtime field {es-pull}86154[#86154] -* Support geo label position through REST vector tiles API {es-pull}86458[#86458] (issue: {es-issue}86044[#86044]) - -Health:: -* Add a basic check for tier preference and allocation filter clashing {es-pull}85071[#85071] -* Add preflight checks to Health API to ensure health is obtainable {es-pull}86404[#86404] -* Add tier information on health api migrate tiers user actions {es-pull}87486[#87486] -* Health api add indicator doc links {es-pull}86904[#86904] (issue: {es-issue}86892[#86892]) -* Health api copy editing {es-pull}87010[#87010] -* Return a default user action if no actions could be determined {es-pull}87079[#87079] - -ILM+SLM:: -* Make the ILM and SLM `history_index_enabled` settings dynamic {es-pull}86493[#86493] - -Indices APIs:: -* Batch execute template and pipeline cluster state operations {es-pull}86017[#86017] - -Infra/Core:: -* Add mapping for tags for the elastic agent {es-pull}86298[#86298] -* Expand jar hell to include modules {es-pull}86622[#86622] -* Faster GET _cluster/settings API {es-pull}86405[#86405] (issue: {es-issue}82342[#82342]) -* Faster string writes by saving stream flushes {es-pull}86114[#86114] -* Fleet: Add `start_time` and `minimum_execution_duration` attributes to actions {es-pull}86167[#86167] -* Force property expansion for security policy {es-pull}87396[#87396] -* Refactor array part into a `BytesRefArray` which can be serialized and … {es-pull}85826[#85826] -* Speed up ip v4 parser {es-pull}86253[#86253] -* Use varhandles for primitive type conversion in more places {es-pull}85577[#85577] (issue: {es-issue}78823[#78823]) - -Infra/Scripting:: -* Script: add ability to alias classes in whitelist {es-pull}86899[#86899] - -Ingest:: -* Iteratively execute synchronous ingest processors {es-pull}84250[#84250] (issue: {es-issue}84274[#84274]) -* Skip `ensureNoSelfReferences` check in `IngestService` {es-pull}87337[#87337] - -License:: -* Initialize active realms without logging a message {es-pull}86134[#86134] (issue: {es-issue}81380[#81380]) - -Machine Learning:: -* A text categorization aggregation that works like ML categorization {es-pull}80867[#80867] -* Add new _infer endpoint for all supervised models and deprecate deployment infer api {es-pull}86361[#86361] -* Adds new `question_answering` NLP task for extracting answers to questions from a document {es-pull}85958[#85958] -* Adds start and end params to `_preview` and excludes cold/frozen tiers from unbounded previews {es-pull}86989[#86989] -* Adjust automatic JVM heap sizing for dedicated ML nodes {es-pull}86399[#86399] -* Replace the implementation of the `categorize_text` aggregation {es-pull}85872[#85872] -* Upgrade PyTorch to version 1.11 {ml-pull}2233[#2233], {ml-pull}2235[#2235],{ml-pull}2238[#2238] -* Upgrade zlib to version 1.2.12 on Windows {ml-pull}2253[#2253] -* Upgrade libxml2 to version 2.9.14 on Linux and Windows {ml-pull}2287[#2287] -* Improve time series model stability and anomaly scoring consistency for data - for which many buckets are empty {ml-pull}2267[#2267] -* Address root cause for actual equals typical equals zero anomalies {ml-pull}2270[#2270] -* Better handling of outliers in update immediately after detecting changes in time series {ml-pull}2280[#2280] - -Mapping:: -* Intern field names in Mappers {es-pull}86301[#86301] -* Replace BYTE_BLOCK_SIZE - 2 with indexWriter#MAX_TERM_LENGTH {es-pull}85518[#85518] - -Network:: -* Log node identity at startup {es-pull}85773[#85773] - -Search:: -* GeoBoundingBox query should work on bounding box with equal latitude or longitude {es-pull}85788[#85788] (issue: {es-issue}77717[#77717]) -* Improve error message for search API url parameters {es-pull}86984[#86984] (issue: {es-issue}79719[#79719]) - -Security:: -* Add run-as support for OAuth2 tokens {es-pull}86680[#86680] -* Relax username restrictions for User APIs {es-pull}86398[#86398] (issue: {es-issue}86326[#86326]) -* User Profile - Add hint support to SuggestProfiles API {es-pull}85890[#85890] -* User Profile - Add new action origin and internal user {es-pull}86026[#86026] -* User Profile - Support request cancellation on HTTP disconnect {es-pull}86332[#86332] -* User Profile - add caching for `hasPrivileges` check {es-pull}86543[#86543] - -Snapshot/Restore:: -* Add parameter to exclude indices in a snapshot from response {es-pull}86269[#86269] (issue: {es-issue}82937[#82937]) - -Stats:: -* Add documentation for "io_time_in_millis" {es-pull}84911[#84911] - -TLS:: -* Set `serverAuth` extended key usage for generated certificates and CSRs {es-pull}86311[#86311] (issue: {es-issue}81067[#81067]) - -TSDB:: -* Aggregation Execution Context add timestamp provider {es-pull}85850[#85850] - -Transform:: -* Prefer secondary auth headers for transforms {es-pull}86757[#86757] -* Support `range` aggregation in transform {es-pull}86501[#86501] - -[[feature-8.3.0]] -[float] -=== New features - -Authorization:: -* Has privileges API for profiles {es-pull}85898[#85898] - -Geo:: -* New geo_grid query to be used with geogrid aggregations {es-pull}86596[#86596] (issue: {es-issue}85727[#85727]) - -Health:: -* Add support for `impact_areas` to health impacts {es-pull}85830[#85830] (issue: {es-issue}85829[#85829]) -* Add troubleshooting guides to shards allocation actions {es-pull}87078[#87078] -* Adding potential impacts to remaining health indicators {es-pull}86197[#86197] -* Health api drill down {es-pull}85234[#85234] (issue: {es-issue}84793[#84793]) -* New service to keep track of the master history as seen from each node {es-pull}85941[#85941] -* Sorting impact index names by index priority {es-pull}85347[#85347] - -Mapping:: -* Add support for dots in field names for metrics usecases {es-pull}86166[#86166] (issue: {es-issue}63530[#63530]) -* Synthetic source {es-pull}85649[#85649] - -SQL:: -* SQ: Allow partial results in SQL queries {es-pull}85897[#85897] (issue: {es-issue}33148[#33148]) - -Search:: -* Snapshots as simple archives {es-pull}86261[#86261] (issue: {es-issue}81210[#81210]) - -TSDB:: -* TSDB: Implement downsampling on time-series indices {es-pull}85708[#85708] (issues: {es-issue}69799[#69799], {es-issue}65769[#65769]) - -[[upgrade-8.3.0]] -[float] -=== Upgrades - -Infra/CLI:: -* Upgrade procrun executables to 1.3.1 {es-pull}86710[#86710] - -Infra/Core:: -* Upgrade jackson to 2.13.2 {es-pull}86051[#86051] - -Ingest:: -* Upgrading to tika 2.4 {es-pull}86015[#86015] - -Network:: -* Upgrade to Netty 4.1.76 {es-pull}86252[#86252] - -Packaging:: -* Update Iron Bank base image to 8.6 {es-pull}86796[#86796] - -SQL:: -* Update dependency - JLine - to v 3.21.0 {es-pull}83767[#83767] (issue: {es-issue}83575[#83575]) - -Search:: -* Update to public lucene 9.2.0 release {es-pull}87162[#87162] - -Snapshot/Restore:: -* Upgrade GCS Plugin to 1.118.1 {es-pull}87800[#87800] - - diff --git a/docs/reference/release-notes/8.3.1.asciidoc b/docs/reference/release-notes/8.3.1.asciidoc deleted file mode 100644 index 6ea9d008f7989..0000000000000 --- a/docs/reference/release-notes/8.3.1.asciidoc +++ /dev/null @@ -1,43 +0,0 @@ -[[release-notes-8.3.1]] -== {es} version 8.3.1 - -Also see <>. -[[known-issues-8.3.1]] -[float] -=== Known issues - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] -[[bug-8.3.1]] -[float] -=== Bug fixes - -Audit:: -* Support removing ignore filters for audit logging {es-pull}87675[#87675] (issue: {es-issue}68588[#68588]) - -Ingest:: -* Don't ignore pipeline for upserts in bulk api {es-pull}87719[#87719] (issue: {es-issue}87131[#87131]) -* Geoip processor should respect the `ignore_missing` in case of missing database {es-pull}87793[#87793] (issue: {es-issue}87345[#87345]) - -Machine Learning:: -* Improve trained model stats API performance {es-pull}87978[#87978] - -Snapshot/Restore:: -* Use the provided SAS token without SDK sanitation that can produce invalid signatures {es-pull}88155[#88155] (issue: {es-issue}88140[#88140]) - -Stats:: -* Run `TransportClusterInfoActions` on MANAGEMENT pool {es-pull}87679[#87679] - -Transform:: -* Execute `_refresh` (with system permissions) separately from `delete by query` (with user permissions) {es-pull}88005[#88005] (issue: {es-issue}88001[#88001]) - -[[enhancement-8.3.1]] -[float] -=== Enhancements - -Discovery-Plugins:: -* Remove redundant jackson dependencies from discovery-azure {es-pull}87898[#87898] - -Performance:: -* Warn about impact of large readahead on search {es-pull}88007[#88007] - - diff --git a/docs/reference/release-notes/8.3.2.asciidoc b/docs/reference/release-notes/8.3.2.asciidoc deleted file mode 100644 index 9e70db95fbc0e..0000000000000 --- a/docs/reference/release-notes/8.3.2.asciidoc +++ /dev/null @@ -1,26 +0,0 @@ -[[release-notes-8.3.2]] -== {es} version 8.3.2 - -Also see <>. - -{es} 8.3.2 is a version compatibility release for the {stack}. - -[[known-issues-8.3.2]] -[float] -=== Known issues - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] - -[[bug-8.3.2]] -[float] -=== Bug fixes - -Geo:: -* Fix potential circuit breaker leak on `InternalGeoGrid` {es-pull}88273[#88273] (issue: {es-issue}88261[#88261]) - -[[feature-8.3.2]] -[float] -=== New features - -Heath:: -* Add user action for the `instance_has_master` indicator {es-pull}87963[#87963] diff --git a/docs/reference/release-notes/8.3.3.asciidoc b/docs/reference/release-notes/8.3.3.asciidoc deleted file mode 100644 index 8dc807ca293cb..0000000000000 --- a/docs/reference/release-notes/8.3.3.asciidoc +++ /dev/null @@ -1,46 +0,0 @@ -[[release-notes-8.3.3]] -== {es} version 8.3.3 - -Also see <>. - -[[known-issues-8.3.3]] -[float] -=== Known issues - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] - - -[[bug-8.3.3]] -[float] -=== Bug fixes - -Infra/Core:: -* Add `build_flavor` back to info API REST response {es-pull}88336[#88336] (issue: {es-issue}88318[#88318]) - -Mapping:: -* Enforce max values limit only when running a script {es-pull}88295[#88295] - -Monitoring:: -* Switch cgroup memory fields to keyword {es-pull}88260[#88260] - -Packaging:: -* Fix Docker positional parameter passing {es-pull}88584[#88584] - -Security:: -* Ensure `CreateApiKey` always creates a new document {es-pull}88413[#88413] - -[[enhancement-8.3.3]] -[float] -=== Enhancements - -Security:: -* New setting to close idle connections in OIDC back-channel {es-pull}87773[#87773] - -[[upgrade-8.3.3]] -[float] -=== Upgrades - -Packaging:: -* Upgrade to OpenJDK 18.0.2+9 {es-pull}88675[#88675] (issue: {es-issue}88673[#88673]) - - diff --git a/docs/reference/release-notes/8.4.0.asciidoc b/docs/reference/release-notes/8.4.0.asciidoc deleted file mode 100644 index c3820ac12481a..0000000000000 --- a/docs/reference/release-notes/8.4.0.asciidoc +++ /dev/null @@ -1,356 +0,0 @@ -[[release-notes-8.4.0]] -== {es} version 8.4.0 - -Also see <>. - -[[known-issues-8.4.0]] -[float] -=== Known issues - -// tag::ml-pre-7-datafeeds-known-issue[] -* {ml-cap} {dfeeds} cannot be listed if any are not modified since version 6.x -+ -If you have a {dfeed} that was created in version 5.x or 6.x and has not -been updated since 7.0, it is not possible to list {dfeeds} in -8.4 and 8.5. This means that {anomaly-jobs} cannot be managed using -{kib}. This issue is fixed in 8.6.0. -+ -If you upgrade to 8.4 or 8.5 with such a {dfeed}, you need to -work around the problem by updating each {dfeed}'s authorization information -using https://support.elastic.dev/knowledge/view/b5a879db[these steps]. -// end::ml-pre-7-datafeeds-known-issue[] - -// tag::file-based-settings-deadlock-known-issue[] -* Orchestrators which use a `settings.json` file to configure Elasticsearch may -encounter deadlocks during master elections (issue: {es-issue}92812[#92812]) -+ -To resolve the deadlock, remove the `settings.json` file and restart the -affected node. -// end::file-based-settings-deadlock-known-issue[] - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] - -// tag::ingest-processor-log4j-cluster-instability-known-issue[] -* When the {ref}/attachment.html[ingest attachment processor] is used, the -interaction of https://tika.apache.org/[Tika] with log4j 2.18.0 and higher -(introduced in {es} 8.4.0) results in excessive logging. This logging is so -excessive that it can lead to cluster instability, to the point where the -cluster is unusable and nodes must be restarted. (issue: {es-issue}91964[#91964]). -This issue is fixed in {es} 8.7.0 ({es-pull}93878[#93878]) -+ -To resolve the issue, upgrade to 8.7.0 or higher. -// end::ingest-processor-log4j-cluster-instability-known-issue[] - -[[bug-8.4.0]] -[float] -=== Bug fixes - -Aggregations:: -* Fix multi-value handling in composite agg {es-pull}88638[#88638] -* Fix: extract matrix stats using `bucket_selector` `buckets_path` {es-pull}88271[#88271] (issue: {es-issue}87454[#87454]) -* Make the metric in the `buckets_path` parameter optional {es-pull}87220[#87220] (issue: {es-issue}72983[#72983]) -* Propagate alias filters to significance aggs filters {es-pull}88221[#88221] (issue: {es-issue}81585[#81585]) - -Allocation:: -* Clamp auto-expand replicas to the closest value {es-pull}87505[#87505] (issue: {es-issue}84788[#84788]) -* Prevent re-balancing using outdated node weights in some cases {es-pull}88385[#88385] (issue: {es-issue}88384[#88384]) -* Remove any existing `read_only_allow_delete` index blocks when `cluster.routing.allocation.disk.threshold_enabled` is set to `false` {es-pull}87841[#87841] (issue: {es-issue}86383[#86383]) -* Replace health request with a state observer {es-pull}88641[#88641] - -Authentication:: -* Fix unique realm name check to cover default realms {es-pull}87999[#87999] - -Authorization:: -* Add rollover permissions for `remote_monitoring_agent` {es-pull}87717[#87717] (issue: {es-issue}84161[#84161]) - -Autoscaling:: -* Autoscaling during shrink {es-pull}88292[#88292] (issue: {es-issue}85480[#85480]) -* Do not include desired nodes in snapshots {es-pull}87695[#87695] - -Cluster Coordination:: -* Improve rejection of ambiguous voting config name {es-pull}89239[#89239] - -Data streams:: -* Fix renaming data streams with CCR replication {es-pull}88875[#88875] (issue: {es-issue}81751[#81751]) - -Distributed:: -* Fixed NullPointerException on bulk request {es-pull}88385[#88385] - -EQL:: -* Avoid attempting PIT close on PIT open failure {es-pull}87498[#87498] -* Improve EQL Sequence circuit breaker precision {es-pull}88538[#88538] (issue: {es-issue}88300[#88300]) - -Geo:: -* Geo_line aggregation returns a geojson point when the resulting line has only one point {es-pull}89199[#89199] (issue: {es-issue}85748[#85748]) -* Sort ranges in `geo_distance` aggregation {es-pull}89154[#89154] (issue: {es-issue}89147[#89147]) - -Health:: -* Fix NPE when checking if the last snapshot was success {es-pull}88811[#88811] -* Fixing a version check for master stability functionality {es-pull}89322[#89322] -* Fixing internal action names {es-pull}89182[#89182] -* Using the correct connection to fetch remote master history {es-pull}87299[#87299] - -Highlighting:: -* Handle ordering in plain highlighter for multiple inputs {es-pull}87414[#87414] (issue: {es-issue}87210[#87210]) - -ILM+SLM:: -* Batch ILM move to retry step task update {es-pull}86759[#86759] - -Infra/CLI:: -* Quote paths with whitespace in Windows service CLIs {es-pull}89072[#89072] (issue: {es-issue}89043[#89043]) - -Infra/Core:: -* Always close directory streams {es-pull}88560[#88560] -* Delete invalid settings for system indices {es-pull}88903[#88903] (issue: {es-issue}88324[#88324]) -* Disallow three-digit minor and revision versions {es-pull}87338[#87338] -* Handle snapshot restore in file settings {es-pull}89321[#89321] (issue: {es-issue}89183[#89183]) -* System indices ignore all user templates {es-pull}87260[#87260] (issues: {es-issue}42508[#42508], {es-issue}74271[#74271]) - -Infra/Node Lifecycle:: -* Fix message for stalled shutdown {es-pull}89254[#89254] - -Infra/Plugins:: -* Disable URL connection caching in SPIClassIterator {es-pull}88586[#88586] (issue: {es-issue}88275[#88275]) - -Infra/Scripting:: -* Script: `UpdateByQuery` can read doc version if requested {es-pull}88740[#88740] - -Machine Learning:: -* Address potential bug where trained models get stuck in starting after being allocated to node {es-pull}88945[#88945] -* Fix BERT and MPNet tokenization bug when handling unicode accents {es-pull}88907[#88907] (issue: {es-issue}88900[#88900]) -* Fix NLP `question_answering` task when best answer is only one token {es-pull}88347[#88347] -* Include start params in `_stats` for non-started model deployments {es-pull}89091[#89091] -* Fix minor tokenization bug when using fill_mask task with roberta tokenizer {es-pull}88825[#88825] -* Fix potential cause of classification and regression job failures {ml-pull}2385[#2385] - -Mapping:: -* Assign the right path to objects merged when parsing mappings {es-pull}89389[#89389] (issue: {es-issue}88573[#88573]) -* Don't modify source map when parsing composite runtime field {es-pull}89114[#89114] - -Network:: -* Ensure that the extended socket options TCP_KEEPXXX are available {es-pull}88935[#88935] (issue: {es-issue}88897[#88897]) - -SQL:: -* Fix `SqlSearchIT` `testAllTypesWithRequestToOldNodes` {es-pull}88883[#88883] (issue: {es-issue}88866[#88866]) -* Fix date range checks {es-pull}87151[#87151] (issue: {es-issue}77179[#77179]) -* fix object equals {es-pull}87887[#87887] - -Search:: -* Fix: use status code 500 for aggregation reduce phase errors if no shard failed {es-pull}88551[#88551] (issue: {es-issue}20004[#20004]) -* Override bulk visit methods of exitable point visitor {es-pull}82120[#82120] - -Security:: -* Ensure `secureString` remain open when reloading secure settings {es-pull}88922[#88922] - -Snapshot/Restore:: -* Fix queued snapshot assignments after partial snapshot fails due to delete {es-pull}88470[#88470] (issue: {es-issue}86724[#86724]) - -Transform:: -* Handle update error correctly {es-pull}88619[#88619] - -[[deprecation-8.4.0]] -[float] -=== Deprecations - -Vector Search:: -* Deprecate the `_knn_search` endpoint {es-pull}88828[#88828] - -[[enhancement-8.4.0]] -[float] -=== Enhancements - -Aggregations:: -* Adding cardinality support for `random_sampler` agg {es-pull}86838[#86838] -* Minor `RangeAgg` optimization {es-pull}86935[#86935] (issue: {es-issue}84262[#84262]) -* Speed counting filters/range/date_histogram aggs {es-pull}81322[#81322] -* Update bucket metric pipeline agg paths to allow intermediate single bucket and bucket qualified multi-bucket aggs {es-pull}85729[#85729] - -Allocation:: -* Add debug information to `ReactiveReason` about assigned and unassigned shards {es-pull}86132[#86132] (issue: {es-issue}85243[#85243]) -* Optimize log cluster health performance. {es-pull}87723[#87723] -* Use desired nodes during data tier allocation decisions {es-pull}87735[#87735] - -Audit:: -* Audit API key ID when create or grant API keys {es-pull}88456[#88456] -* Include API key metadata in audit log when an API key is created, granted, or updated {es-pull}88642[#88642] -* Updatable API keys - logging audit trail event {es-pull}88276[#88276] -* User Profile - audit support for security domain {es-pull}87097[#87097] - -Authentication:: -* If signature validation fails, reload JWKs and retry if new JWKs are found {es-pull}88023[#88023] - -Authorization:: -* App permissions with action patterns do not retrieve privileges {es-pull}85455[#85455] -* Cancellable Profile Has Privilege check {es-pull}87224[#87224] -* Return action denied error when user with insufficient privileges (`manage_own_api_key`) attempts a grant API key request {es-pull}87461[#87461] (issue: {es-issue}87438[#87438]) -* Update indices permissions to Enterprise Search service account {es-pull}88703[#88703] - -Autoscaling:: -* Add processors to autoscaling capacity response {es-pull}87895[#87895] -* Keep track of desired nodes status in cluster state {es-pull}87474[#87474] - -Cluster Coordination:: -* Deduplicate mappings in persisted cluster state {es-pull}88479[#88479] -* Expose segment details in PCSS debug log {es-pull}87412[#87412] -* Periodic warning for 1-node cluster w/ seed hosts {es-pull}88013[#88013] (issue: {es-issue}85222[#85222]) -* Report overall mapping size in cluster stats {es-pull}87556[#87556] - -Data streams:: -* Give doc-value-only mappings to numeric fields on metrics templates {es-pull}87100[#87100] - -Distributed:: -* Adding the ability to register a `PeerFinderListener` to Coordinator {es-pull}88626[#88626] -* Make Desired Nodes API operator-only {es-pull}87778[#87778] (issue: {es-issue}87777[#87777]) -* Support "dry run" mode for updating Desired Nodes {es-pull}88305[#88305] - -FIPS:: -* Log warning when hash function used by cache is not recommended in FIPS mode {es-pull}86740[#86740] -* Log warning when hashers for stored API keys or service tokens are not compliant with FIPS {es-pull}87363[#87363] - -Geo:: -* Optimize geogrid aggregations for singleton points {es-pull}87439[#87439] -* Support cartesian shape with doc values {es-pull}88487[#88487] -* Use a faster but less accurate log algorithm for computing Geotile Y coordinate {es-pull}87515[#87515] -* Use faster maths to project WGS84 to mercator {es-pull}88231[#88231] - -Health:: -* Add health user action for unhealthy SLM policy failure counts {es-pull}88523[#88523] -* Adding a transport action to get cluster formation info {es-pull}87306[#87306] -* Adding additional capability to the `master_is_stable` health indicator service {es-pull}87482[#87482] -* Creating a transport action for the `CoordinationDiagnosticsService` {es-pull}87984[#87984] -* Move the master stability logic into its own service separate from the `HealthIndicatorService` {es-pull}87672[#87672] -* Polling cluster formation state for master-is-stable health indicator {es-pull}88397[#88397] -* Remove cluster block preflight check from health api {es-pull}87520[#87520] (issue: {es-issue}87464[#87464]) - -ILM+SLM:: -* Add min_* conditions to rollover {es-pull}83345[#83345] -* Track the count of failed invocations since last successful policy snapshot {es-pull}88398[#88398] - -Infra/Core:: -* Improve console exception messages {es-pull}87942[#87942] -* Print full exception when console is non-interactive {es-pull}88297[#88297] -* Stop making index read-only when executing force merge index lifecycle management action {es-pull}81162[#81162] (issue: {es-issue}81162[#81162]) -* Stream input and output support for optional collections {es-pull}88127[#88127] -* Update version of internal http client {es-pull}87491[#87491] - -Infra/Logging:: -* Catch an exception when formatting a string fails {es-pull}87132[#87132] - -Infra/Scripting:: -* Script: Add Metadata to ingest context {es-pull}87309[#87309] -* Script: Metadata for update context {es-pull}88333[#88333] - -Infra/Settings:: -* Convert disk watermarks to RelativeByteSizeValues {es-pull}88719[#88719] - -Ingest:: -* Allow pipeline processor to ignore missing pipelines {es-pull}87354[#87354] -* Move the ingest attachment processor to the default distribution {es-pull}87989[#87989] -* Only perform `ensureNoSelfReferences` check during ingest when needed {es-pull}87352[#87352] (issue: {es-issue}87335[#87335]) -* Removing `BouncyCastle` dependencies from ingest-attachment plugin {es-pull}88031[#88031] - -Machine Learning:: -* Add authorization info to ML config listings {es-pull}87884[#87884] -* Add deployed native models to `inference_stats` in trained model stats response {es-pull}88187[#88187] -* Add inference cache hit count to inference node stats {es-pull}88807[#88807] -* Add new `cache_size` parameter to `trained_model` deployments API {es-pull}88450[#88450] -* Expand allowed NER labels to be any I-O-B tagged labels {es-pull}87091[#87091] -* Improve scalability of NLP models {es-pull}87366[#87366] -* Indicate overall deployment failure if all node routes are failed {es-pull}88378[#88378] -* New `frequent_items` aggregation {es-pull}83055[#83055] -* Fairer application of size penalty for model selection for training classification and regression models {ml-pull}2291[#2291] -* Accelerate training for data frame analytics by skipping fine parameter tuning if it is unnecessary {ml-pull}2298[#2298] -* Address some causes of high runtimes training regression and classification models on large data sets with many features {ml-pull}2332[#2332] -* Add caching for PyTorch inference {ml-pull}2305[#2305] -* Improve accuracy of anomaly detection median estimation {ml-pull}2367[#2367] (issue: {ml-issue}2364[#2364]) - -Mapping:: -* Enable synthetic source support on constant keyword fields {es-pull}88603[#88603] -* Speed up `NumberFieldMapper` {es-pull}85688[#85688] - -Monitoring:: -* JvmService use SingleObjectCache {es-pull}87236[#87236] - -Network:: -* Allow start cluster with unreachable remote clusters {es-pull}87298[#87298] -* Increase `http.max_header_size` default to 16kb {es-pull}88725[#88725] (issue: {es-issue}88501[#88501]) - -Query Languages:: -* Add support for VERSION field type in SQL and EQL {es-pull}87590[#87590] (issue: {es-issue}83375[#83375]) - -Rollup:: -* [TSDB] Add Kahan support to downsampling summation {es-pull}87554[#87554] - -SQL:: -* Implement support for partial search results in SQL CLI {es-pull}86982[#86982] (issue: {es-issue}86082[#86082]) -* Update Tableau connector to use connection dialog v2 {es-pull}88462[#88462] - -Search:: -* Add mapping stats for indexed `dense_vectors` {es-pull}86859[#86859] -* Improve error when sorting on incompatible types {es-pull}88399[#88399] (issue: {es-issue}73146[#73146]) -* Support kNN vectors in disk usage action {es-pull}88785[#88785] (issue: {es-issue}84801[#84801]) - -Security:: -* Add setting for `tcp_keepalive` for oidc back-channel {es-pull}87868[#87868] -* Support `run_as` another user when granting API keys {es-pull}88335[#88335] -* Support exists query for API key query {es-pull}87229[#87229] -* Updatable API keys - REST API spec and tests {es-pull}88270[#88270] -* Updatable API keys - noop check {es-pull}88346[#88346] - -Snapshot/Restore:: -* INFO logging of snapshot restore and completion {es-pull}88257[#88257] (issue: {es-issue}86610[#86610]) -* Make snapshot deletes not block the repository during data blob deletes {es-pull}86514[#86514] -* Retry after all S3 get failures that made progress {es-pull}88015[#88015] (issue: {es-issue}87243[#87243]) -* Speed up creating new `IndexMetaDataGenerations` without removed snapshots {es-pull}88344[#88344] -* Update HDFS Repository to HDFS 3.3.3 {es-pull}88039[#88039] - -Stats:: -* Sort ingest pipeline stats by use {es-pull}88035[#88035] - -TLS:: -* Add issuer to GET _ssl/certificates {es-pull}88445[#88445] - -Transform:: -* Add authorization info to transform config listings {es-pull}87570[#87570] -* Implement per-transform num_failure_retries setting {es-pull}87361[#87361] - -[[feature-8.4.0]] -[float] -=== New features - -Authentication:: -* Support updates of API key attributes (single operation route) {es-pull}88186[#88186] - -Health:: -* Master stability health indicator part 1 (when a master has been seen recently) {es-pull}86524[#86524] -* Remove help_url,rename summary to symptom, and `user_actions` to diagnosis {es-pull}88553[#88553] (issue: {es-issue}88474[#88474]) - -Infra/Core:: -* File Settings Service {es-pull}88329[#88329] - -Infra/Logging:: -* Stable logging API - the basic use case {es-pull}86612[#86612] - -Machine Learning:: -* Make composite aggs in datafeeds Generally Available {es-pull}88589[#88589] - -Search:: -* Add 'mode' option to `_source` field mapper {es-pull}88211[#88211] - -TSDB:: -* TSDB: Implement downsampling ILM Action for time-series indices {es-pull}87269[#87269] (issue: {es-issue}68609[#68609]) - -Vector Search:: -* Integrate ANN into `_search` endpoint {es-pull}88694[#88694] (issue: {es-issue}87625[#87625]) - -[[upgrade-8.4.0]] -[float] -=== Upgrades - -Infra/Core:: -* Upgrade to Log4J 2.18.0 {es-pull}88237[#88237] - -Network:: -* Upgrade to Netty 4.1.77 {es-pull}86630[#86630] - - diff --git a/docs/reference/release-notes/8.4.1.asciidoc b/docs/reference/release-notes/8.4.1.asciidoc deleted file mode 100644 index b0339e14a689f..0000000000000 --- a/docs/reference/release-notes/8.4.1.asciidoc +++ /dev/null @@ -1,27 +0,0 @@ -[[release-notes-8.4.1]] -== {es} version 8.4.1 - -Also see <>. - -[[known-issues-8.4.1]] -[float] -=== Known issues - -* When using date range search with format that does not have all date fields (missing month or day) -an incorrectly parsed date could be used. The workaround is to use date pattern with all date fields (year, month, day) -(issue: {es-issue}90187[#90187]) - -include::8.4.0.asciidoc[tag=ml-pre-7-datafeeds-known-issue] - -include::8.4.0.asciidoc[tag=file-based-settings-deadlock-known-issue] - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] - -include::8.4.0.asciidoc[tag=ingest-processor-log4j-cluster-instability-known-issue] - -[[bug-8.4.1]] -[float] -=== Bug fixes - -Machine Learning:: -* [ML] Validate trained model deployment `queue_capacity` limit {es-pull}89611[#89611] (issue: {es-issue}89555[#89555]) diff --git a/docs/reference/release-notes/8.4.2.asciidoc b/docs/reference/release-notes/8.4.2.asciidoc deleted file mode 100644 index 7fad1e6c467ae..0000000000000 --- a/docs/reference/release-notes/8.4.2.asciidoc +++ /dev/null @@ -1,92 +0,0 @@ -[[release-notes-8.4.2]] -== {es} version 8.4.2 - -Also see <>. - -[[known-issues-8.4.2]] -[float] -=== Known issues - -* **This version contains a regression in `multi_match` queries that use the -`cross_fields` scoring type.** {es} -+ -When running a <> query with the -`cross_fields` type, {es} can sometimes throw an IllegalArgument exception -with the message "totalTermFreq must be at least docFreq". If you use the -`cross_fields` scoring type, it is recommended that you skip version 8.4.2. -This regression was fixed in version 8.4.3. - -* When using date range search with format that does not have all date fields (missing month or day) -an incorrectly parsed date could be used. The workaround is to use date pattern with all date fields (year, month, day) -(issue: {es-issue}90187[#90187]) - -include::8.4.0.asciidoc[tag=ml-pre-7-datafeeds-known-issue] - -include::8.4.0.asciidoc[tag=file-based-settings-deadlock-known-issue] - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] - -include::8.4.0.asciidoc[tag=ingest-processor-log4j-cluster-instability-known-issue] - -[[bug-8.4.2]] -[float] -=== Bug fixes - -Allocation:: -* Fix debug mode in `MaxRetryAllocationDecider` {es-pull}89973[#89973] - -Authentication:: -* Fix double sending of response in `TransportOpenIdConnectPrepareAuthenticationAction` {es-pull}89930[#89930] - -Autoscaling:: -* Fix issue with autoscaling after a clone or split {es-pull}89768[#89768] (issue: {es-issue}89758[#89758]) - -Health:: -* Fix the conditions for fetching remote master history {es-pull}89472[#89472] (issue: {es-issue}89431[#89431]) - -ILM+SLM:: -* Copy `isHidden` during ILM alias swap {es-pull}89650[#89650] (issue: {es-issue}89604[#89604]) - -Infra/Core:: -* Extend the date rounding logic to be conditional {es-pull}89693[#89693] (issues: {es-issue}89096[#89096], {es-issue}58986[#58986]) -* Fix `FileSettingsService` hang on error update {es-pull}89630[#89630] -* Implement fix to terminate file Watcher thread to avoid deadlock {es-pull}89934[#89934] - -Ingest Node:: -* Fix pipeline `id` not present in ingest metadata inside `on_failure` block {es-pull}89632[#89632] - -Machine Learning:: -* Fix memory leak in `TransportDeleteExpiredDataAction` {es-pull}89935[#89935] -* Do not retain categorization tokens when existing category matches {ml-pull}2398[#2398] - -Network:: -* Fix memory leak when double invoking `RestChannel.sendResponse` {es-pull}89873[#89873] - -Ranking:: -* Avoid negative scores with `cross_fields` type {es-pull}89016[#89016] (issue: {es-issue}44700[#44700]) - -Rollup:: -* Fork `TransportRollupCapsAction` to MANAGEMENT POOL {es-pull}89803[#89803] - -Search:: -* Empty intervals needs to start in position -1 {es-pull}89962[#89962] (issue: {es-issue}89789[#89789]) - -Transform:: -* Scheduler concurrency fix {es-pull}89716[#89716] (issue: {es-issue}88991[#88991]) - -[[enhancement-8.4.2]] -[float] -=== Enhancements - -Allocation:: -* Log unsuccessful attempts to get credentials from web identity tokens {es-pull}88241[#88241] - -Health:: -* Add delayed allocation diagnosis case to shards availability indicator {es-pull}89056[#89056] - -[[upgrade-8.4.2]] -[float] -=== Upgrades - -Packaging:: -* Update OpenJDK to 18.0.2.1 {es-pull}89535[#89535] (issue: {es-issue}89531[#89531]) diff --git a/docs/reference/release-notes/8.4.3.asciidoc b/docs/reference/release-notes/8.4.3.asciidoc deleted file mode 100644 index 2efa07f51f44b..0000000000000 --- a/docs/reference/release-notes/8.4.3.asciidoc +++ /dev/null @@ -1,37 +0,0 @@ -[[release-notes-8.4.3]] -== {es} version 8.4.3 - -Also see <>. - -[[known-issues-8.4.3]] -[float] -=== Known issues - -* When using date range search with format that does not have all date fields (missing month or day) -an incorrectly parsed date could be used. The workaround is to use date pattern with all date fields (year, month, day) -(issue: {es-issue}90187[#90187]) - -include::8.4.0.asciidoc[tag=ml-pre-7-datafeeds-known-issue] - -include::8.4.0.asciidoc[tag=file-based-settings-deadlock-known-issue] - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] - -include::8.4.0.asciidoc[tag=ingest-processor-log4j-cluster-instability-known-issue] - -[[bug-8.4.3]] -[float] -=== Bug fixes - -Infra/Core:: -* Fix file permission errors to avoid repeated error save loops on Windows {es-pull}90271[#90271] (issue: {es-issue}90222[#90222]) - -Ingest Node:: -* Prevent serialization errors in the nodes stats API {es-pull}90319[#90319] (issue: {es-issue}77973[#77973]) - -[[regression-8.4.3]] -[float] -=== Regressions - -Ranking:: -* Ensure `cross_fields` always uses valid term statistics {es-pull}90314[#90314] diff --git a/docs/reference/release-notes/8.5.0.asciidoc b/docs/reference/release-notes/8.5.0.asciidoc deleted file mode 100644 index 682353f930189..0000000000000 --- a/docs/reference/release-notes/8.5.0.asciidoc +++ /dev/null @@ -1,340 +0,0 @@ -[[release-notes-8.5.0]] -== {es} version 8.5.0 - -Also see <>. - -[[known-issues-8.5.0]] -[float] -=== Known issues - -* It is possible to inadvertently create an alias with the same name as an -index in version 8.5.0. This action leaves the cluster in an invalid state in -which several features will not work correctly, and it may not even be possible -to restart nodes while in this state. Upgrade to 8.5.1 as soon as possible to -avoid the risk of this occurring ({es-pull}91456[#91456]). If your cluster is -affected by this issue, upgrade to 8.5.3 to repair it ({es-pull}91887[#91887]). - -include::8.4.0.asciidoc[tag=ml-pre-7-datafeeds-known-issue] - -include::8.4.0.asciidoc[tag=file-based-settings-deadlock-known-issue] - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] - -include::8.4.0.asciidoc[tag=ingest-processor-log4j-cluster-instability-known-issue] - -[[breaking-8.5.0]] -[float] -=== Breaking changes - -CRUD:: -* Reject unknown bulk actions {es-pull}89450[#89450] - -[[bug-8.5.0]] -[float] -=== Bug fixes - -Aggregations:: -* Fix `auto_date_histogram` > `ip_range` {es-pull}90317[#90317] (issue: {es-issue}90121[#90121]) -* Fail when rebuilding scorer in `breadth_first` mode and query context has changed {es-pull}89993[#89993] (issue: {es-issue}37650[#37650]) -* Fix merging with empty results {es-pull}86939[#86939] (issue: {es-issue}84622[#84622]) -* Fix partial reduce bug in `ip_prefix` {es-pull}89734[#89734] (issue: {es-issue}89686[#89686]) -* Remove unexpected meta parameter in aggregation response {es-pull}89467[#89467] (issue: {es-issue}89455[#89455]) -* Consistently allow sorting `top_metrics` aggregations whose size is greater than 1 {es-pull}89974[#89974] (issue: {es-issue}86663[#86663]) -* Check parent circuit breaker when allocating an empty bucket {es-pull}89568[#89568] (issue: {es-issue}80789[#80789]) - -Authorization:: -* Remove magic string for the `"__empty"` role {es-pull}89766[#89766] - -CCR:: -* Order follower backing indices for data streams by original name {es-pull}90850[#90850] (issue: {es-issue}90820[#90820]) - -Cluster Coordination:: -* Capture deprecation warnings in batched master tasks {es-pull}85525[#85525] (issue: {es-issue}85506[#85506]) -* Check circuit breaker before sending join request {es-pull}89318[#89318] (issue: {es-issue}85003[#85003]) - -Distributed:: -* Fork `TransportClusterStateAction` to `MANAGEMENT` thread pool {es-pull}90996[#90996] - -Engine:: -* Fix `"path.conf'"` typo in `Security.java` {es-pull}89248[#89248] (issue: {es-issue}89327[#89327]) - -Geo:: -* Buffer H3Polygon2D bounding boxes to avoid edge precision issues {es-pull}89196[#89196] (issues: {es-issue}89868[#89868], {es-issue}87391[#87391]) -* Fix date histogram range edge case {es-pull}88957[#88957] -* Format runtime `geo_points` {es-pull}85449[#85449] (issue: {es-issue}85245[#85245]) -* Generate error if scripts load `_source` with synthetic `_source` enabled {es-pull}88334[#88334] - -Graph:: -* Fix race condition in timeout {es-pull}88946[#88946] (issue: {es-issue}55396[#55396]) - -Health:: -* Modify disk indicator details to provide an overview of the disk space health {es-pull}90189[#90189] -* Fix disk indicator impacts and diagnosis {es-pull}90262[#90262] -* Fix the details calculation of the disk indicator {es-pull}90869[#90869] -* Report impact and diagnosis data nodes without disk space and no blocked indices {es-pull}90772[#90772] (issue: {es-issue}90442[#90442]) -* Update minimum version for health node reporting to 8.5 {es-pull}90365[#90365] (issue: {es-issue}90359[#90359]) - -Indices APIs:: -* Avoid capturing per-task `RolloverResult` {es-pull}90626[#90626] (issue: {es-issue}90620[#90620]) -* Fix shard splitting for `nested` {es-pull}89351[#89351] (issue: {es-issue}88109[#88109]) - -Infra/Core:: -* Fix `allow_no_indices` request option in special cases {es-pull}89622[#89622] -* Fix repeated error save loops in File Settings Service {es-pull}90271[#90271] (issue: {es-issue}90222[#90222]) -* Fix date rounding for date math parsing {es-pull}90458[#90458] (issue: {es-issue}90187[#90187]) -* Fix disabling APM tracing for `CancellableTask` in `TrainedModelAssignmentNodeService` {es-pull}90972[#90972] (issue: {es-issue}89850[#89850]) -* Support camel case dates on 7.x indices {es-pull}88914[#88914] (issue: {es-issue}84199[#84199]) - -Infra/Scripting:: -* Fix true/false accumulation bug in boolean `source` fallback {es-pull}90895[#90895] - -Ingest Node:: -* Set the enrich maintenance cluster lifecycle listener only once {es-pull}90486[#90486] - -Machine Learning:: -* Require correct tier processors when multiple AZs are present {es-pull}90903[#90903] -* Return 408 instead of 500 when open/start APIs time out {es-pull}89775[#89775] (issue: {es-issue}89585[#89585]) - -Mapping:: -* Fix duplication bug for `source` fallback in numeric types {es-pull}89352[#89352] -* Include runtime fields in total fields count {es-pull}89251[#89251] (issue: {es-issue}88265[#88265]) -* Fix `aggregate_metric_double` multi-values exception {es-pull}90290[#90290] -* Validate field names when subobjects are disabled {es-pull}90950[#90950] - -Monitoring:: -* Add fields to fix {ls} cgroup graphs {es-pull}90493[#90493] - -Network:: -* Fix `RecyclerBytesStreamOutput` allocating unlimited heap for some capacities {es-pull}90632[#90632] - -Recovery:: -* Fix overcounting recovered bytes after network disconnect {es-pull}90477[#90477] (issue: {es-issue}90441[#90441]) - -Search:: -* Add support for predefined char class regexp on wildcard fields {es-pull}90064[#90064] -* Deduplicate fetching doc-values fields {es-pull}89094[#89094] -* Don't shortcut the total hit count for text fields {es-pull}90341[#90341] (issue: {es-issue}89760[#89760]) -* Safeguard `RegExp` use against `StackOverflowError` {es-pull}84624[#84624] (issue: {es-issue}82923[#82923]) -* Use MB rather than GB to calculate max boolean clauses {es-pull}90309[#90309] (issue: {es-issue}86136[#86136]) - -Snapshot/Restore:: -* Fix incorrect failed shards count in APIs for current snapshots {es-pull}89534[#89534] -* Fix over-allocation of mounted indices on a cold/frozen node {es-pull}86331[#86331] -* Fix quadratic complexity in `SnapshotStatus` serialization {es-pull}90795[#90795] -* Fork building snapshot status response off of transport thread {es-pull}90651[#90651] -* Make sure listener is resolved when file queue is cleared {es-pull}89929[#89929] -* Re-register a corrupt repository to unblock it {es-pull}89719[#89719] (issue: {es-issue}89130[#89130]) -* Reject unknown request body fields in mount API {es-pull}88987[#88987] (issue: {es-issue}75982[#75982]) - -TSDS:: -* Fix segment stats in TSDS {es-pull}89754[#89754] (issue: {es-issue}89609[#89609]) -* Fix extra fields in `GET` request for synthetic `_source` {es-pull}89778[#89778] -* Fix `scaled_float` rounding for synthetic `_source` {es-pull}88916[#88916] (issue: {es-issue}88854[#88854]) - -Transform:: -* Don't fail a transform on a ClusterBlockException, this may be due to ILM closing an index {es-pull}90396[#90396] (issue: {es-issue}89802[#89802]) -* Fix NPE in transform scheduling {es-pull}90347[#90347] (issues: {es-issue}90356[#90356], {es-issue}88203[#88203], {es-issue}90301[#90301], {es-issue}90255[#90255]) -* Improve error handling in state persistence {es-pull}88910[#88910] (issue: {es-issue}88905[#88905]) -* Return `408` instead of `500` when the start API times out {es-pull}89774[#89774] - -Vector Search:: -* Fix bug for `kNN` with filtered aliases {es-pull}89621[#89621] - -Watcher:: -* Allow `xpack.notification.email.account.domain_allowlist` to be set dynamically {es-pull}90426[#90426] (issue: {es-issue}89913[#89913]) -* Handling timeout exceptions on watcher startup {es-pull}90421[#90421] (issue: {es-issue}44981[#44981]) - -[[deprecation-8.5.0]] -[float] -=== Deprecations - -Infra/Plugins:: -* Deprecate network plugins {es-pull}88924[#88924] -* Deprecate overriding `DiscoveryPlugin` internals {es-pull}88925[#88925] - -[[enhancement-8.5.0]] -[float] -=== Enhancements - -Authentication:: -* Add more accurate error message for LDAP user modes {es-pull}89492[#89492] - -Authorization:: -* Add indices permissions to {ents} service account {es-pull}89869[#89869] -* Add information of resolved roles in denial messages {es-pull}89680[#89680] - -Autoscaling:: -* Centralize the concept of processors configuration {es-pull}89662[#89662] - -Cluster Coordination:: -* Preemptively compute `RoutingNodes` and the indices lookup during publication {es-pull}89005[#89005] -* Preemptively initialize routing nodes and indices lookup on all node types {es-pull}89032[#89032] - -Distributed:: -* Batch index delete cluster state updates {es-pull}90033[#90033] (issue: {es-issue}90022[#90022]) -* Increase the minimum size of the management pool to `2` {es-pull}90193[#90193] - -Health:: -* Add IDs to health API diagnoses and impacts {es-pull}90072[#90072] -* Add a check to the master stability health API when there is no master and the current node is not master eligible {es-pull}89219[#89219] -* Add logic to `master_is_stable` indicator to check for discovery problems {es-pull}88020[#88020] -* Poll for cluster diagnostics information {es-pull}89014[#89014] -* Update SLM health diagnosis message to include unhealthy policy details {es-pull}89138[#89138] - -Highlighting:: -* Improve efficiency of `BoundedBreakIteratorScanner` fragmentation algorithm {es-pull}89041[#89041] (issues: {es-issue}73569[#73569], {es-issue}73785[#73785]) - -ILM+SLM:: -* Add validations for the downsampling ILM action {es-pull}90295[#90295] -* Ensure that ILM does not roll over empty indices {es-pull}89557[#89557] (issue: {es-issue}86203[#86203]) -* Reuse informational message in lifecycle step {es-pull}89419[#89419] -* Move log-related logic into log block in `IndexLifecycleRunner` {es-pull}89292[#89292] - -Infra/Core:: -* Add reserved `/_snapshot/repo` file based settings {es-pull}89601[#89601] -* Add `upgrade_status` attributes to Fleet Agents {es-pull}89845[#89845] -* Add support for `/_autoscaling/policy` for file based settings {es-pull}89708[#89708] -* Add support for `/_security/role_mapping` for file based settings {es-pull}89667[#89667] -* Add support for support for `/_slm/policy` in file based settings {es-pull}89567[#89567] -* Retry file watch registration {es-pull}90537[#90537] (issue: {es-issue}89500[#89500]) - -Infra/Node Lifecycle:: -* Distinguish no shutdowns case in `NodeShutdownAllocationDecider` {es-pull}89851[#89851] (issue: {es-issue}89823[#89823]) - -Infra/Plugins:: -* Add deprecation message for deprecated plugin APIs {es-pull}88961[#88961] -* Register stable plugins in `ActionModule` {es-pull}90067[#90067] -* Load plugin named components {es-pull}89969[#89969] - -Infra/Scripting:: -* Initial code to support binary expression scripts {es-pull}89895[#89895] -* Protect `_source` inside update scripts {es-pull}88733[#88733] -* Reindex and `UpdateByQuery` metadata {es-pull}88665[#88665] -* Add write Field API `NestedDocument` support {es-pull}90021[#90021] -* Add write Field API path manipulation {es-pull}89889[#89889] -* Add write Field API with basic path resolution {es-pull}89738[#89738] -* Add write Fields API for reindex, update, and update by query {es-pull}90145[#90145] - -Infra/Settings:: -* Introduce max headroom for disk watermark stages {es-pull}88639[#88639] (issue: {es-issue}81406[#81406]) - -License:: -* License check for user profile collaboration feature {es-pull}89990[#89990] - -Machine Learning:: -* Add measure of non-cache hit inference count {es-pull}90464[#90464] -* Add new `text_similarity` nlp task {es-pull}88439[#88439] -* Add new trained model deployment cache clear API {es-pull}89074[#89074] -* Add processor autoscaling decider {es-pull}89645[#89645] -* Distribute trained model allocations across availability zones {es-pull}89822[#89822] -* Use a bitset for deduplication of frequent items {es-pull}88943[#88943] -* Optimize frequent items transaction lookup {es-pull}89062[#89062] -* Release native inference functionality as beta {es-pull}90418[#90418] -* Return `408` when the start deployment API times out {es-pull}89612[#89612] -* Skip renormalization after calling the node shutdown API {es-pull}89347[#89347] -* Compute outlier feature influence via the Gateaux derivative to improve attribution for high dimension vectors {ml-pull}2256[#2256] -* Improve classification and regression model train runtimes for data sets with many numeric features {ml-pull}2380[#2380], {ml-pull}2388[#2388], {ml-pull}2390[#2390], {ml-pull}2401[#2401] -* Increase the limit on the maximum number of classes to `100` for training classification models {ml-pull}2395[#2395] (issue: {ml-issue}2246[#2246]) - -Mapping:: -* Add `synthetic_source` support to `aggregate_metric_double` fields {es-pull}88909[#88909] -* Add `source` fallback for keyword fields using operation {es-pull}88735[#88735] -* Add `source` fallback support for `match_only_text` mapped type {es-pull}89473[#89473] -* Add `source` fallback support for date and `date_nanos` mapped types {es-pull}89440[#89440] -* Add `source` fallback support for unsigned long mapped type {es-pull}89349[#89349] -* Add support for `source` fallback with scaled float field type {es-pull}89053[#89053] -* Add support for `source` fallback with the boolean field type {es-pull}89052[#89052] -* Add text field support in the Painless scripting fields API {es-pull}89396[#89396] -* Clarify that fielddata is not supported for text fields error message {es-pull}89770[#89770] (issue: {es-issue}89485[#89485]) -* Add new mappings for Fleet Agent `last_checkin_message` and components fields {es-pull}89599[#89599] -* Support `source` fallback for `byte`, `short`, and `long` fields {es-pull}88954[#88954] -* Support `source` fallback for `double`, `float`, and `half_float` field types {es-pull}89010[#89010] - -Network:: -* Use chunked REST serialization for large REST responses {es-pull}88311[#88311] - -Recovery:: -* Disable recovering from snapshots in searchable snapshots {es-pull}86388[#86388] - -SQL:: -* Implement `DATE_FORMAT` function {es-pull}88388[#88388] (issue: {es-issue}55065[#55065]) -* Set `track_total_hits` to false when not needed {es-pull}89106[#89106] (issue: {es-issue}88764[#88764]) - -Search:: -* Enable `BloomFilter` for `_id` of non-datastream indices {es-pull}88409[#88409] -* In the field capabilities API, renew support for fields in the request body {es-pull}88972[#88972] (issue: {es-issue}86875[#86875]) - -Security:: -* Add usage stats report for user profiles {es-pull}90123[#90123] -* Implement grace period for user profile activation {es-pull}89566[#89566] -* Return limited-by role descriptors in Get/QueryApiKey response {es-pull}89273[#89273] -* Add option to return profile uid in `GetUser` response {es-pull}89570[#89570] -* Return `400` error for `GetUserPrivileges` call with API keys {es-pull}89333[#89333] -* Show assigned role descriptors in Get/QueryApiKey response {es-pull}89166[#89166] -* Add detailed errors in `hasPrivileges` response {es-pull}89224[#89224] -* Add support for multiple UIDs to the `GetProfile` API {es-pull}89023[#89023] - -Snapshot/Restore:: -* Add support for comparing `SnapshotsInProgress` {es-pull}89619[#89619] (issue: {es-issue}88732[#88732]) -* Prioritize shard snapshot tasks over file snapshot tasks and limit the number of the concurrently running snapshot tasks {es-pull}88209[#88209] (issue: {es-issue}83408[#83408]) - -Stats:: -* Introduce node mappings stats {es-pull}89807[#89807] - -TSDS:: -* Support `match_only_text` for synthetic `_source` {es-pull}89516[#89516] -* Support histogram field for synthetic `_source` {es-pull}89833[#89833] -* Support version field type for synthetic `_source` {es-pull}89706[#89706] -* Build `_id` without reparsing {es-pull}88789[#88789] -* Return metric fields in the field caps API {es-pull}88695[#88695] - -Transform:: -* Add an unattended mode setting to transform {es-pull}89212[#89212] - -[[feature-8.5.0]] -[float] -=== New features - -Authorization:: -* Introduce the new `read_security` cluster privilege {es-pull}89790[#89790] (issue: {es-issue}89245[#89245]) - -Health:: -* Enable the health node and the disk health indicator {es-pull}90085[#90085] (issue: {es-issue}84811[#84811]) - -Infra/Core:: -* Provide tracing implementation using OpenTelemetry and APM Java agent {es-pull}88443[#88443] (issue: {es-issue}84369[#84369]) - -Infra/Plugins:: -* Add the stable Plugin API module and analysis interfaces {es-pull}88775[#88775] - -Machine Learning:: -* Make `bucket_correlation` aggregation generally available {es-pull}88655[#88655] -* Make `bucket_count_ks_test` aggregation generally available {es-pull}88657[#88657] - -Security:: -* Support bulk updates of API keys {es-pull}88856[#88856] - -TSDS:: -* Add a TSID global ordinal to `TimeSeriesIndexSearcher` {es-pull}90035[#90035] -* Release time series data stream functionality {es-pull}90116[#90116] (issue: {es-issue}74660[#74660]) -* Add synthetic `_source` support for the `ignore_above` parameter on `keyword` fields {es-pull}89466[#89466] - -Vector Search:: -* Add synthetic `_source` support for `dense_vector` {es-pull}89840[#89840] - -[[regression-8.5.0]] -[float] -=== Regressions - -Infra/Scripting:: -* Fix fields API caching regression {es-pull}90017[#90017] - -[[upgrade-8.5.0]] -[float] -=== Upgrades - -Client:: -* Upgrade Apache Commons Logging to 1.2 {es-pull}85745[#85745] (issue: {es-issue}40305[#40305]) - -Packaging:: -* Upgrade bundled JDK to Java 19 {es-pull}90571[#90571] diff --git a/docs/reference/release-notes/8.5.1.asciidoc b/docs/reference/release-notes/8.5.1.asciidoc deleted file mode 100644 index 89ccaec8af21e..0000000000000 --- a/docs/reference/release-notes/8.5.1.asciidoc +++ /dev/null @@ -1,76 +0,0 @@ -[[release-notes-8.5.1]] -== {es} version 8.5.1 - - -Also see <>. - -[[known-issues-8.5.1]] -[float] -=== Known issues - -include::8.4.0.asciidoc[tag=ml-pre-7-datafeeds-known-issue] - -include::8.4.0.asciidoc[tag=file-based-settings-deadlock-known-issue] - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] - -include::8.4.0.asciidoc[tag=ingest-processor-log4j-cluster-instability-known-issue] - -[[bug-8.5.1]] -[float] -=== Bug fixes - -Audit:: -* Fix NPE in auditing `authenticationSuccess` for non-existing run-as user {es-pull}91171[#91171] - -Authentication:: -* Ensure PKI's `delegated_by_realm` metadata respect run-as {es-pull}91173[#91173] - -Authorization:: -* Ensure `TermsEnum` action works correctly with API keys {es-pull}91170[#91170] - -Cluster Coordination:: -* Fix corrupted Metadata from index and alias having the same name {es-pull}91456[#91456] - -EQL:: -* Fix EQLSearchRequest serialization (bwc) {es-pull}91402[#91402] - -Geo:: -* Fix handling indexed envelopes crossing the dateline in mvt API {es-pull}91105[#91105] (issue: {es-issue}91060[#91060]) - -Infra/CLI:: -* Fix carriage return removal when reading a long line from terminal {es-pull}91131[#91131] (issue: {es-issue}89227[#89227]) - -Infra/Core:: -* Fix APM configuration file delete {es-pull}91058[#91058] (issue: {es-issue}89439[#89439]) - -Machine Learning:: -* Allow NLP truncate option to be updated when span is set {es-pull}91224[#91224] -* Interim buckets should not count towards the total bucket count {es-pull}91288[#91288] - -Network:: -* Fix `TransportActionProxy` for local execution {es-pull}91289[#91289] - -Transform:: -* Make transform `_preview` request cancellable {es-pull}91313[#91313] (issue: {es-issue}91286[#91286]) - -[[enhancement-8.5.1]] -[float] -=== Enhancements - -Authorization:: -* Add privileges for connectors index creation {es-pull}91026[#91026] -* Add privileges for crawler logs indices in Enterprise Search service account {es-pull}91094[#91094] - -Infra/Core:: -* Allow legacy index settings on legacy indices {es-pull}90264[#90264] (issue: {es-issue}84992[#84992]) -* Check for unassigned shards on node shutdown {es-pull}91297[#91297] (issue: {es-issue}88635[#88635]) - -[[upgrade-8.5.1]] -[float] -=== Upgrades - -Packaging:: -* Update bundled JDK to Java 19.0.1 {es-pull}91025[#91025] (issue: {es-issue}91010[#91010]) - - diff --git a/docs/reference/release-notes/8.5.2.asciidoc b/docs/reference/release-notes/8.5.2.asciidoc deleted file mode 100644 index b6f8ec1646496..0000000000000 --- a/docs/reference/release-notes/8.5.2.asciidoc +++ /dev/null @@ -1,51 +0,0 @@ -[[release-notes-8.5.2]] -== {es} version 8.5.2 - - -Also see <>. - -[[known-issues-8.5.2]] -[float] -=== Known issues - -include::8.4.0.asciidoc[tag=ml-pre-7-datafeeds-known-issue] - -include::8.4.0.asciidoc[tag=file-based-settings-deadlock-known-issue] - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] - -include::8.4.0.asciidoc[tag=ingest-processor-log4j-cluster-instability-known-issue] - -[[bug-8.5.2]] -[float] -=== Bug fixes - -Authorization:: -* Avoid potential unsupported operation exception in doc bitset cache {es-pull}91490[#91490] - -EQL:: -* Refine bwc version checks on `EqlSearchRequest` {es-pull}91510[#91510] - -Health:: -* SLM uneahlthy policies diagnosis recommends correct URL in action {es-pull}91506[#91506] - -Ingest Node:: -* Refactor `DatabaseNodeService` as a cluster state listener {es-pull}91567[#91567] (issue: {es-issue}86999[#86999]) - -Stats:: -* Fix NPE in IndexService getNodeMappingStats {es-pull}91334[#91334] (issue: {es-issue}91259[#91259]) - -Transform:: -* Fix failure when resolving indices from CCS {es-pull}91622[#91622] (issue: {es-issue}91550[#91550]) - -[[enhancement-8.5.2]] -[float] -=== Enhancements - -EQL:: -* Remove version limitations for CCS {es-pull}91409[#91409] - -Ingest Node:: -* Refactor enrich maintenance coordination logic {es-pull}90931[#90931] - - diff --git a/docs/reference/release-notes/8.5.3.asciidoc b/docs/reference/release-notes/8.5.3.asciidoc deleted file mode 100644 index 70d92d42a038c..0000000000000 --- a/docs/reference/release-notes/8.5.3.asciidoc +++ /dev/null @@ -1,55 +0,0 @@ -[[release-notes-8.5.3]] -== {es} version 8.5.3 - -Also see <>. - -[[known-issues-8.5.3]] -[float] -=== Known issues - -include::8.4.0.asciidoc[tag=ml-pre-7-datafeeds-known-issue] - -include::8.4.0.asciidoc[tag=file-based-settings-deadlock-known-issue] - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] - -include::8.4.0.asciidoc[tag=ingest-processor-log4j-cluster-instability-known-issue] - -[[bug-8.5.3]] -[float] -=== Bug fixes - -Infra/Core:: -* Add `trace.id` to request trace logs {es-pull}91772[#91772] (issue: {es-issue}88174[#88174]) -* `DoPrivileged` in `ElasticsearchEncaughtExceptionHandler` and check modify thread {es-pull}91704[#91704] (issue: {es-issue}91650[#91650]) - -Ingest Node:: -* Handle any exception thrown while generating source for an `IngestDocument` {es-pull}91981[#91981] - -Machine Learning:: -* ML stats failures should not stop the usage API working {es-pull}91917[#91917] (issue: {es-issue}91893[#91893]) - -Stats:: -* Fix NPE in IndexService getNodeMappingStats {es-pull}91334[#91334] (issue: {es-issue}91259[#91259]) - -Transform:: -* Fix failure when resolving indices from CCS {es-pull}91622[#91622] (issue: {es-issue}91550[#91550]) - -[[enhancement-8.5.3]] -[float] -=== Enhancements - -Ingest Node:: -* Refactor enrich maintenance coordination logic {es-pull}90931[#90931] - -TLS:: -* Support SAN/dnsName for restricted trust {es-pull}91946[#91946] - -[[upgrade-8.5.3]] -[float] -=== Upgrades - -Engine:: -* Upgrade Lucene to version 9.4.2 {es-pull}91823[#91823] - - diff --git a/docs/reference/release-notes/8.6.0.asciidoc b/docs/reference/release-notes/8.6.0.asciidoc deleted file mode 100644 index 1e30c4a4a9c49..0000000000000 --- a/docs/reference/release-notes/8.6.0.asciidoc +++ /dev/null @@ -1,295 +0,0 @@ -[[release-notes-8.6.0]] -== {es} version 8.6.0 - -Also see <>. - -[[known-issues-8.6.0]] -[float] -=== Known issues - -include::8.4.0.asciidoc[tag=file-based-settings-deadlock-known-issue] - -include::8.0.0.asciidoc[tag=jackson-filtering-bug] - -include::8.4.0.asciidoc[tag=ingest-processor-log4j-cluster-instability-known-issue] - -// tag::reconciliation-imbalance-known-issue[] -* Shard rebalancing may temporarily unbalance cluster -+ -From 8.6.0 onwards the default shard rebalancing algorithm will compute the -final desired balance and then make shard movements to reconcile the current -state of the cluster with the desired state. However the order in which the -shard movements take place may be skewed towards certain nodes, causing the -cluster to become temporarily unbalanced while the reconciliation is ongoing. -As always, once a node reaches a disk watermark it will not accept any -additional shards, but this skew may result in nodes reaching their disk -watermarks more often than expected in normal operation. Once the -reconciliation process completes, the cluster will be balanced again. -+ -To avoid this problem, upgrade to 8.8.0 or later. -// end::reconciliation-imbalance-known-issue[] - -[[bug-8.6.0]] -[float] -=== Bug fixes - -Aggregations:: -* GeoBoundsAggregations reject sub aggregations {es-pull}91073[#91073] (issue: {es-issue}91072[#91072]) - -Allocation:: -* Avoid NPE when disassociateDeadNodes is executed for a node present in the desired balance {es-pull}91659[#91659] -* Check `NodesShutdownMetadata` type before assuming restart {es-pull}90807[#90807] - -Authentication:: -* Fix time unit for connection request timeout of JWKs reload {es-pull}92080[#92080] -* Improve performance for role mapping with DNs {es-pull}92074[#92074] -* Improve robustness of `JwkSet` reloading {es-pull}92081[#92081] -* Support stored authentication headers prior to version 6.7 {es-pull}92221[#92221] - -Authorization:: -* Make adding auth info to REST responses more robust {es-pull}92168[#92168] -* Security remove datemath special handling {es-pull}91047[#91047] - -Cluster Coordination:: -* Fix `TransportMasterNodeAction` holding a CS reference needlessly {es-pull}90724[#90724] (issue: {es-issue}89220[#89220]) -* Include last-committed data in publication {es-pull}92259[#92259] (issue: {es-issue}90158[#90158]) -* Unsafe bootstrap memory optimization {es-pull}92493[#92493] - -EQL:: -* EQL sequences: support join on multi-values {es-pull}89965[#89965] - -Graph:: -* Fix potential issue with graph api's timed out field in response {es-pull}91006[#91006] - -Health:: -* Don't account for the unassigned reason when diagnosing NO_VALID_SHARD_COPY {es-pull}92416[#92416] -* Fix NPE when evaluating the disk health for non-data nodes {es-pull}92643[#92643] -* Use https in the short URLs for the `shards_availability` indicator {es-pull}92310[#92310] - -Indices APIs:: -* Trigger index settings providers when updating component templates {es-pull}91615[#91615] (issue: {es-issue}91592[#91592]) - -Infra/Core:: -* Check reserved state in Metadata.isGlobalStateEquals {es-pull}92124[#92124] -* Datastream unavailable exception metadata {es-pull}91461[#91461] -* Fix `BytesRefArray` on append empty `BytesRef` {es-pull}91364[#91364] -* Fix index expression options for requests with a single name or pattern {es-pull}91231[#91231] -* Force init of Unbox in log4j {es-pull}92377[#92377] (issue: {es-issue}91964[#91964]) -* In file based settings, wait until security index is ready for role mappings {es-pull}92173[#92173] (issue: {es-issue}91939[#91939]) -* Index expression exclusions never trigger "not found" {es-pull}90902[#90902] -* Update error states from inside the main state executor {es-pull}90346[#90346] (issue: {es-issue}90337[#90337]) - -Infra/Scripting:: -* Fix compile with hex literals ending with d/f {es-pull}91501[#91501] (issue: {es-issue}88614[#88614]) - -Ingest Node:: -* Fixing a race condition in `EnrichCoordinatorProxyAction` that can leave an item stuck in its queue {es-pull}90688[#90688] (issue: {es-issue}90598[#90598]) - -Machine Learning:: -* Copy more settings when creating DF analytics destination index {es-pull}91546[#91546] (issue: {es-issue}89795[#89795]) -* Fix for 'No statistics' error message {ml-pull}2410[#2410] -* Fix for 'No counts available' error message {ml-pull}2414[#2414] -* Guard against input sequences that are too long for Question Answering models {es-pull}91924[#91924] -* Improve performance of closing files before spawning {ml-pull}2424[#2424] -* Skip remote clusters when performing up front privileges validation for datafeeds {es-pull}91895[#91895] (issue: {es-issue}87832[#87832]) -* Support fields with commas in data frame analytics `analyzed_fields` {es-pull}91710[#91710] (issue: {es-issue}72541[#72541]) -* Validate rule filters are present on open anomaly detection api {es-pull}92207[#92207] - - -Mapping:: -* Consolidate field name validation when parsing mappings and documents {es-pull}91328[#91328] -* Fix handling empty key case in the terms aggregation {es-pull}90822[#90822] - -Monitoring:: -* Fix logstash loadavg (xpack cases) {es-pull}90494[#90494] -* [Stack Monitoring] Update ES module mappings {es-pull}90649[#90649] - -Network:: -* Clean up on exception while chunking XContent {es-pull}92024[#92024] -* Fix Chunked APIs sending incorrect responses to HEAD requests {es-pull}92042[#92042] (issue: {es-issue}92032[#92032]) -* Reject connection attempts while closing {es-pull}92465[#92465] - -SQL:: -* Fix NPE on logging when not tracking total hits {es-pull}92425[#92425] - -Search:: -* Allow different decay values depending on the score function {es-pull}91195[#91195] (issue: {es-issue}78887[#78887]) -* Fix timing bug with DFS profiling {es-pull}92421[#92421] - -Snapshot/Restore:: -* Simplify and optimize deduplication of `RepositoryData` for a non-caching repository instance {es-pull}91851[#91851] (issue: {es-issue}89952[#89952]) - -Store:: -* Fix numOpenOutputs and modCount in ByteSizeCachingDirectory {es-pull}92440[#92440] (issue: {es-issue}92434[#92434]) - -Transform:: -* Skip remote clusters when performing up front privileges validation {es-pull}91788[#91788] - -Vector Search:: -* Make `knn` search requests fully cancellable {es-pull}90612[#90612] - -[[deprecation-8.6.0]] -[float] -=== Deprecations - -Allocation:: -* Deprecate state field in /_cluster/reroute response {es-pull}90399[#90399] -* Ensure balance threshold is at least 1 {es-pull}92100[#92100] - -Ingest Node:: -* Deprecate 'remove_binary' default of false for ingest attachment processor {es-pull}90460[#90460] - -Mapping:: -* Deprecate silently ignoring type, fields, copy_to and boost in metadata field definition {es-pull}90989[#90989] (issue: {es-issue}35389[#35389]) - -[[enhancement-8.6.0]] -[float] -=== Enhancements - -Allocation:: -* Clear up forecasted write load and shard size from previous write index during rollovers {es-pull}91590[#91590] -* Forecast average shard size during rollovers {es-pull}91561[#91561] -* Forecast write load during rollovers {es-pull}91425[#91425] -* Improve shard balancing {es-pull}91603[#91603] -* Introduce desired-balance allocator {es-pull}91343[#91343] -* Limit shard realocation retries {es-pull}90296[#90296] -* Prevalidate node removal API {es-pull}88952[#88952] -* Set default `cluster.routing.allocation.balance.disk_usage` {es-pull}91951[#91951] -* Store write load in the `IndexMetadata` during data streams rollovers {es-pull}91019[#91019] -* Update the default `cluster.routing.allocation.balance.disk_usage` {es-pull}92065[#92065] -* `DesiredBalance:` expose it via _internal/desired_balance {es-pull}91038[#91038] (issue: {es-issue}90583[#90583]) - -Authorization:: -* [Fleet] Added logs-elastic_agent* read privileges to `kibana_system` {es-pull}91701[#91701] - -CRUD:: -* Keep track of average shard write load {es-pull}90768[#90768] (issue: {es-issue}90102[#90102]) - -Geo:: -* Centroid aggregation for cartesian points and shapes {es-pull}89216[#89216] (issue: {es-issue}90156[#90156]) -* Improve H3#hexRing logic and add H3#areNeighborCells method {es-pull}91140[#91140] -* Move SpatialUtils to geo library {es-pull}88088[#88088] (issue: {es-issue}86607[#86607]) -* Reduce number of object allocations in H3#geoToH3 and speed up computations {es-pull}91492[#91492] -* Support `cartesian_bounds` aggregation on point and shape {es-pull}91298[#91298] (issue: {es-issue}90157[#90157]) - -ILM+SLM:: -* ILM: Get policy support wildcard name {es-pull}89238[#89238] - -Infra/Core:: -* Handle APM global labels as affix setting {es-pull}91438[#91438] (issue: {es-issue}91278[#91278]) -* Improve date math exclusions in expressions {es-pull}90298[#90298] -* Introduce a phase to use String.equals on constant strings, rather than def equality {es-pull}91362[#91362] (issue: {es-issue}91235[#91235]) -* More actionable error for ancient indices {es-pull}91243[#91243] -* Operator/index templates {es-pull}90143[#90143] -* Operator/ingest {es-pull}89735[#89735] -* Transport threads and `_hot_threads` {es-pull}90482[#90482] (issue: {es-issue}90334[#90334]) -* Upgrade XContent to Jackson 2.14.0 and enable Fast Double Parser {es-pull}90553[#90553] - -Infra/Plugins:: -* Create placeholder plugin when loading stable plugins {es-pull}90870[#90870] -* Example stable plugin {es-pull}90805[#90805] -* Make `extendedPlugins,` `HasNativeController` and `moduleName` optional in plugin descriptor {es-pull}90835[#90835] -* Rename `NamedComponent` name parameter to value {es-pull}91306[#91306] - -Infra/Scripting:: -* Use an explicit null check for null receivers in painless, rather than an NPE {es-pull}91347[#91347] (issue: {es-issue}91236[#91236]) - -Machine Learning:: -* Add a filter parameter to frequent items {es-pull}91137[#91137] -* Add a regex to the output of the `categorize_text` aggregation {es-pull}90723[#90723] -* Add ability to filter and sort buckets by `change_point` numeric values {es-pull}91299[#91299] -* Add api to update trained model deployment `number_of_allocations` {es-pull}90728[#90728] -* Alias timestamp to @timestamp in anomaly detection results index {es-pull}90812[#90812] -* Allow `model_aliases` to be used with Pytorch trained models {es-pull}91296[#91296] -* Allow overriding timestamp field to null in file structure finder {es-pull}90764[#90764] -* Audit a message every day the datafeed has seen no data {es-pull}91774[#91774] -* Low priority trained model deployments {es-pull}91234[#91234] (issue: {es-issue}91024[#91024]) -* Provide additional information about anomaly score factors {es-pull}90675[#90675] - -Mapping:: -* Don't create IndexCaps objects when recording unmapped fields {es-pull}90806[#90806] (issue: {es-issue}90796[#90796]) -* aggregate metric double add a max min validation {es-pull}90381[#90381] - -Recovery:: -* Remove resize index settings once shards are started {es-pull}90391[#90391] (issue: {es-issue}90127[#90127]) - -Rollup:: -* Test downsample runtime fields and security {es-pull}90593[#90593] - -Search:: -* Add LimitedOffsetsEnum to Limited offset token {es-pull}86110[#86110] (issue: {es-issue}86109[#86109]) -* Add profiling and documentation for dfs phase {es-pull}90536[#90536] (issue: {es-issue}89713[#89713]) -* Bulk merge field-caps responses using mapping hash {es-pull}86323[#86323] -* Enhance nested depth tracking when parsing queries {es-pull}90425[#90425] -* Expose telemetry about search usage {es-pull}91528[#91528] -* Return docs when using nested mappings in archive indices {es-pull}90585[#90585] (issue: {es-issue}90523[#90523]) -* Use `IndexOrDocValues` query for IP range queries {es-pull}90303[#90303] (issue: {es-issue}83658[#83658]) - -Snapshot/Restore:: -* Increase snaphot pool max size to 10 {es-pull}90282[#90282] (issue: {es-issue}89608[#89608]) -* Tie snapshot speed to node bandwidth settings {es-pull}91021[#91021] (issue: {es-issue}57023[#57023]) - -Store:: -* Allow plugins to wrap Lucene directories created by the `IndexModule` {es-pull}91556[#91556] - -TLS:: -* Add certificate start/expiry dates to SSL Diagnostic message {es-pull}89461[#89461] - -TSDB:: -* Generate 'index.routing_path' from dynamic mapping templates {es-pull}90552[#90552] (issue: {es-issue}90528[#90528]) -* Support malformed numbers in synthetic `_source` {es-pull}90428[#90428] -* Support synthetic `_source` for `_doc_count` field {es-pull}91465[#91465] -* Synthetic _source: support `field` in many cases {es-pull}89950[#89950] -* Synthetic `_source`: `ignore_malformed` for `ip` {es-pull}90038[#90038] -* Synthetic `_source`: support `wildcard` field {es-pull}90196[#90196] - -Transform:: -* Add a health section to transform stats {es-pull}90760[#90760] -* Support `aggregate_metric_double` field type in transform aggregations {es-pull}91045[#91045] - -Vector Search:: -* Add profiling information for knn vector queries {es-pull}90200[#90200] - -[[feature-8.6.0]] -[float] -=== New features - -Distributed:: -* Add "index" and "search" node roles with feature flag and setting {es-pull}90993[#90993] - -EQL:: -* EQL samples {es-pull}91312[#91312] - -Health:: -* Use chunked encoding for `RestGetHealthAction` {es-pull}91515[#91515] (issue: {es-issue}90223[#90223]) -* [HealthAPI] Use the `RestCancellableNodeClient` infrastructure {es-pull}91587[#91587] - -Machine Learning:: -* Make `categorize_text` aggregation GA {es-pull}88600[#88600] - -Vector Search:: -* Add fielddata and scripting support for byte-sized vectors {es-pull}91184[#91184] -* Add support for indexing byte-sized knn vectors {es-pull}90774[#90774] - -[[regression-8.6.0]] -[float] -=== Regressions - -Infra/Core:: -* Revert "Remove `ImmutableOpenMap` from snapshot services" {es-pull}90287[#90287] - -[[upgrade-8.6.0]] -[float] -=== Upgrades - -Infra/Logging:: -* Upgrade to log4j 2.19.0 {es-pull}90589[#90589] (issue: {es-issue}90584[#90584]) - -Network:: -* Upgrade to Netty 4.1.82.Final {es-pull}90604[#90604] -* Upgrade to Netty 4.1.84 {es-pull}91271[#91271] - -Snapshot/Restore:: -* Upgrade GCS SDK to 2.13.1 {es-pull}92327[#92327] - - diff --git a/docs/reference/release-notes/8.6.1.asciidoc b/docs/reference/release-notes/8.6.1.asciidoc deleted file mode 100644 index cad0a466606be..0000000000000 --- a/docs/reference/release-notes/8.6.1.asciidoc +++ /dev/null @@ -1,43 +0,0 @@ -[[release-notes-8.6.1]] -== {es} version 8.6.1 - -Also see <>. - -[[known-issues-8.6.1]] -[float] -=== Known issues - -include::8.6.0.asciidoc[tag=reconciliation-imbalance-known-issue] - -include::8.4.0.asciidoc[tag=ingest-processor-log4j-cluster-instability-known-issue] - -[[bug-8.6.1]] -[float] -=== Bug fixes - -Data streams:: -* Fix wildcard expansion for delete-by-query on data streams {es-pull}92891[#92891] -* Fix wildcard expansion for update-by-query on data streams {es-pull}92717[#92717] (issue: {es-issue}90272[#90272]) -* Patch jackson-core with locally modified class #92984 -This fixes an issue in jackson parsing (issue: {es-issue}92480[#92480]) - -Distributed:: -* Fix `ByteArrayIndexInput` with nonzero offset {es-pull}93205[#93205] - -ILM+SLM:: -* Get repository metadata from the cluster state doesn't throw an exception if a repo is missing {es-pull}92914[#92914] - -Infra/Core:: -* Don't announce ready until file settings are applied {es-pull}92856[#92856] (issue: {es-issue}92812[#92812]) - -Machine Learning:: -* Utilise parallel allocations where the inference request contains multiple documents {es-pull}92359[#92359] - -Mapping:: -* Fix `_bulk` api `dynamic_templates` and explicit `op_type` {es-pull}92687[#92687] - -Search:: -* Avoid doing I/O when fetching min and max for keyword fields {es-pull}92026[#92026] -* Reduce memory required for search responses when many shards are unavailable {es-pull}91365[#91365] (issue: {es-issue}90622[#90622]) - - diff --git a/docs/reference/release-notes/8.6.2.asciidoc b/docs/reference/release-notes/8.6.2.asciidoc deleted file mode 100644 index 8da2bd2c5e0e5..0000000000000 --- a/docs/reference/release-notes/8.6.2.asciidoc +++ /dev/null @@ -1,33 +0,0 @@ -[[release-notes-8.6.2]] -== {es} version 8.6.2 - -Also see <>. - -[[known-issues-8.6.2]] -[float] -=== Known issues - -include::8.6.0.asciidoc[tag=reconciliation-imbalance-known-issue] - -include::8.4.0.asciidoc[tag=ingest-processor-log4j-cluster-instability-known-issue] - -[[bug-8.6.2]] -[float] -=== Bug fixes - -Allocation:: -* Only simulate legal desired moves {es-pull}93635[#93635] (issue: {es-issue}93271[#93271]) - -Health:: -* Fix the reporting of initializing shards in the Health API {es-pull}93502[#93502] (issue: {es-issue}90327[#90327]) - -Infra/Core:: -* Don't report MIGRATION_NEEDED for 7.x indices {es-pull}93666[#93666] - -Ingest Node:: -* Fix geo ip database file leak when processing IP arrays {es-pull}93177[#93177] - -Machine Learning:: -* Use long inference timeout at ingest {es-pull}93731[#93731] - - diff --git a/docs/reference/release-notes/8.7.0.asciidoc b/docs/reference/release-notes/8.7.0.asciidoc deleted file mode 100644 index 75cc1d80987aa..0000000000000 --- a/docs/reference/release-notes/8.7.0.asciidoc +++ /dev/null @@ -1,402 +0,0 @@ -[[release-notes-8.7.0]] -== {es} version 8.7.0 - -Also see <>. - -[[known-issues-8.7.0]] -[float] -=== Known issues - -include::8.6.0.asciidoc[tag=reconciliation-imbalance-known-issue] - -[[breaking-8.7.0]] -[float] -=== Breaking changes - -Ingest Node:: -* Making `JsonProcessor` stricter so that it does not silently drop data {es-pull}93179[#93179] (issue: {es-issue}92898[#92898]) - -Indices APIs:: -* The <> API implementation was adjusted to use the -same index resolution mechanism as other similar APIs, adding support for the -`ignore_unavailable` and `allow_no_indices` flags and the `_all` meta-index. If -there are no matching indices then earlier versions of this API would return an -empty result with the `200 OK` HTTP response code, but from 8.7.0 onwards by -default it returns an `IndexNotFoundException` with the `404 Not Found` HTTP -response code. To recover the old behaviour, add the query parameter -`?ignore_unavailable=true` ({es-pull}92820[#92820]). - -[[bug-8.7.0]] -[float] -=== Bug fixes - -Aggregations:: -* Don't create a new `DoubleHistogram` instance for empty buckets {es-pull}92547[#92547] -* Fix: do not allow map key types other than String {es-pull}88686[#88686] (issue: {es-issue}66057[#66057]) - -Allocation:: -* Fallback to the actual shard size when forecast is not available {es-pull}93461[#93461] -* Skip `DiskThresholdMonitor` when cluster state is not recovered {es-pull}93699[#93699] -* Suppress response headers in `AllocationActionMultiListener` {es-pull}93777[#93777] (issue: {es-issue}93773[#93773]) - -Authentication:: -* Correctly remove domain from realm when rewriting `Authentication` for compatibility with node versions that don't -support domains {es-pull}93276[#93276] - -Authorization:: -* Fix Security's expression resolver to not remove unavailable but authorized names {es-pull}92625[#92625] - -CCR:: -* Deduplicate Heavy CCR Repository CS Requests {es-pull}91398[#91398] - -CRUD:: -* Avoid NPE in Stateless Get/mGet {es-pull}94164[#94164] -* Do not refresh all indices in `TransportBulkAction` {es-pull}93417[#93417] - -Cluster Coordination:: -* Delay master task failure notifications until commit {es-pull}92693[#92693] (issue: {es-issue}92677[#92677]) - -Data streams:: -* Allow different filters per `DataStream` in a `DataStreamAlias` {es-pull}92692[#92692] (issue: {es-issue}92050[#92050]) - -Geo:: -* Build index qualified name in cross cluster vector tile search {es-pull}94574[#94574] (issue: {es-issue}94557[#94557]) -* Check `GeohexGrid` bounds on geopoint using spherical coordinates {es-pull}92460[#92460] -* Fix bug when clipping Geometry collections in vector tiles {es-pull}93562[#93562] - -Health:: -* Take into account `max_headroom` in disk watermark calculations {es-pull}93157[#93157] (issue: {es-issue}93155[#93155]) - -ILM+SLM:: -* Allow ILM step transition to the phase terminal step {es-pull}91754[#91754] -* Avoiding `BulkProcessor` deadlock in ILMHistoryStore {es-pull}91238[#91238] (issues: {es-issue}68468[#68468], {es-issue}50440[#50440]) -* Fixed changing only the `forceMerge` flag in `SearchableSnapshotAction` wouldn't update the policy {es-pull}93847[#93847] -* Preventing ILM and SLM runtime state from being stored in a snapshot {es-pull}92252[#92252] - -Infra/CLI:: -* Restore printing bootstrap checks as errors {es-pull}93178[#93178] (issue: {es-issue}93074[#93074]) - -Infra/Core:: -* Add `jdk.internal.reflect` permission to es codebase {es-pull}92387[#92387] (issue: {es-issue}92356[#92356]) -* Add checks for exception loops through suppressed exceptions only {es-pull}93944[#93944] (issue: {es-issue}93943[#93943]) -* Ensure one-shot wrappers release their delegates {es-pull}92928[#92928] -* Fix `InputStream#readAllBytes` on `InputStreamIndexInput` {es-pull}92680[#92680] -* Fix indices resolver for datemath with colon {es-pull}92973[#92973] -* Make `FilterStreamInput` less trappy {es-pull}92422[#92422] - -Infra/Plugins:: -* Ensure ordering of plugin initialization {es-pull}93882[#93882] (issue: {es-issue}93851[#93851]) -* Fix unclosed directory stream in `ClassReaders` {es-pull}92890[#92890] (issue: {es-issue}92866[#92866]) -* Update the version of asm used by plugin scanner {es-pull}92784[#92784] (issue: {es-issue}92782[#92782]) - -Infra/REST API:: -* [Rest Api Compatibility] Format response media type with parameters {es-pull}92695[#92695] - -Infra/Scripting:: -* Fix NPE when method was called on an array type {es-pull}91713[#91713] (issue: {es-issue}87562[#87562]) - -Infra/Settings:: -* Fix parse failures for ILM operator settings {es-pull}94477[#94477] (issue: {es-issue}94465[#94465]) - -Ingest Node:: -* Better names and types for ingest stats {es-pull}93533[#93533] (issue: {es-issue}80763[#80763]) -* Correctly handle an exception case for ingest failure {es-pull}92455[#92455] -* Disable ingest-attachment logging {es-pull}93878[#93878] -* Download the geoip databases only when needed {es-pull}92335[#92335] (issue: {es-issue}90673[#90673]) -* Forwarding simulate calls to ingest nodes {es-pull}92171[#92171] -* Grok returns a list of matches for repeated pattern names #92092 {es-pull}92586[#92586] (issue: {es-issue}92092[#92092]) -* Handle a default/request pipeline and a final pipeline with minimal additional overhead {es-pull}93329[#93329] (issues: {es-issue}92843[#92843], {es-issue}81244[#81244], {es-issue}93118[#93118]) -* Ingest-attachment module tika dependency versions {es-pull}93755[#93755] -* More accurate total ingest stats {es-pull}91730[#91730] (issue: {es-issue}91358[#91358]) -* Speed up ingest geoip processors {es-pull}92372[#92372] -* Speed up ingest set and append processors {es-pull}92395[#92395] - -Machine Learning:: -* Allocate trained models if zone awareness attributes not set {es-pull}94128[#94128] (issue: {es-issue}94123[#94123]) -* Fix data counts race condition when starting a datafeed {es-pull}93324[#93324] (issue: {es-issue}93298[#93298]) -* Fix tokenization bug when handling normalization in BERT and MPNet {es-pull}92329[#92329] -* Free resources correctly when model loading is cancelled {es-pull}92204[#92204] -* Stop the `frequent_items` aggregation reporting a subset when a superset exists {es-pull}92239[#92239] -* Use long inference timeout at ingest {es-pull}93731[#93731] - -Mapping:: -* Fix dynamic mapping detection for invalid dates {es-pull}94115[#94115] (issue: {es-issue}93888[#93888]) -* No length check for source-only keyword fields {es-pull}93299[#93299] (issue: {es-issue}9304[#9304]) - -Network:: -* Delay Connection#onRemoved while pending {es-pull}92546[#92546] -* Fix fransport handshake starting before tls handshake completes {es-pull}90534[#90534] (issue: {es-issue}77999[#77999]) -* Protect `NodeConnectionsService` from stale conns {es-pull}92558[#92558] (issue: {es-issue}92029[#92029]) - -Recovery:: -* Disable recovery monitor before recovery start {es-pull}93551[#93551] (issue: {es-issue}93542[#93542]) -* Fix potential leak in `RemoteRecoveryHandler` {es-pull}91802[#91802] -* Report recovered files as recovered from snapshot for fully mounted searchable snapshots {es-pull}92976[#92976] - -Rollup:: -* Downsampling unmapped text fields {es-pull}94387[#94387] (issue: {es-issue}94346[#94346]) -* Propagate timestamp format and convert nanoseconds to milliseconds {es-pull}94141[#94141] (issue: {es-issue}94085[#94085]) -* Stop processing `TransportDownsampleAction` on failure {es-pull}94624[#94624] -* Support downsampling of histogram as labels {es-pull}93445[#93445] (issue: {es-issue}93263[#93263]) - -Search:: -* Add null check for sort fields over collapse fields {es-pull}94546[#94546] (issue: {es-issue}94407[#94407]) -* Annotated highlighter does not match when search contains both annotation and annotated term {es-pull}92920[#92920] (issue: {es-issue}91944[#91944]) -* Clear field caps index responses on cancelled {es-pull}93716[#93716] (issue: {es-issue}93029[#93029]) -* Do not include frozen indices in PIT by default {es-pull}94377[#94377] -* Fix NPE thrown by prefix query in strange scenarios {es-pull}94369[#94369] -* Fix _id field fetch issue. {es-pull}94528[#94528] (issue: {es-issue}94515[#94515]) -* Fix metadata `_size` when it comes to stored fields extraction {es-pull}94483[#94483] (issue: {es-issue}94468[#94468]) -* Fix missing override for matches in `ProfileWeight` {es-pull}92360[#92360] -* Nested path info shouldn't be added during `copy_to` {es-pull}93340[#93340] (issue: {es-issue}93117[#93117]) -* Use all profiling events on startup {es-pull}92087[#92087] -* Use keyword analyzer for untokenized fields in `TermVectorsService` {es-pull}94518[#94518] -* [Profiling] Adjust handling of last data slice {es-pull}94283[#94283] -* [Profiling] Ensure responses are only sent once {es-pull}93692[#93692] (issue: {es-issue}93691[#93691]) -* [Profiling] Handle response processing errors {es-pull}93860[#93860] - -Snapshot/Restore:: -* Fix unhandled exception when blobstore repository contains unexpected file {es-pull}93914[#93914] -* Support for GCS proxies everywhere in the GCS API {es-pull}92192[#92192] (issue: {es-issue}91952[#91952]) - -Stats:: -* Avoid capturing cluster state in TBbNA {es-pull}92255[#92255] - -TSDB:: -* Fix synthetic `_source` for sparse `_doc_count` field {es-pull}91769[#91769] (issue: {es-issue}91731[#91731]) - -Task Management:: -* Fix context leak in list tasks API {es-pull}93431[#93431] (issue: {es-issue}93428[#93428]) - -Transform:: -* Integrate "sourceHasChanged" call into failure handling and retry logic {es-pull}92762[#92762] (issue: {es-issue}92133[#92133]) - -Vector Search:: -* Fix `maxScore` calculation for kNN search {es-pull}93875[#93875] -* Fix explain for kNN search matches {es-pull}93876[#93876] - -[[enhancement-8.7.0]] -[float] -=== Enhancements - -Aggregations:: -* Optimize composite agg with leading global ordinal value source {es-pull}92197[#92197] - -Allocation:: -* Add `forecasted_write_load` and `forecasted_shard_size_in_bytes` to the endpoint {es-pull}92303[#92303] -* Expose tier balancing stats via internal endpoint {es-pull}92199[#92199] -* Introduce ShardRouting.Role {es-pull}92668[#92668] -* Prevalidate node removal API (pt. 2) {es-pull}91256[#91256] (issue: {es-issue}87776[#87776]) -* Simulate moves using cluster_concurrent_rebalance=2 {es-pull}93977[#93977] -* Unpromotables skip replication and peer recovery {es-pull}93210[#93210] - -Authentication:: -* Add new `token_type` setting to JWT realm {es-pull}91536[#91536] -* JWT realm - Initial support for access tokens {es-pull}91781[#91781] -* JWT realm - Simplify token principal calculation {es-pull}92315[#92315] -* JWT realm - add support for required claims {es-pull}92314[#92314] -* Support custom PBKDF2 password hashes {es-pull}92871[#92871] - -Authorization:: -* Allowed indices matcher supports nested limited roles {es-pull}93306[#93306] -* Extra `kibana_system` privileges for Fleet transform upgrades {es-pull}91499[#91499] -* Pre-authorize child search transport actions {es-pull}91886[#91886] - -Cluster Coordination:: -* Add links to troubleshooting docs {es-pull}92755[#92755] (issue: {es-issue}92741[#92741]) -* Improve node-{join,left} logging for troubleshooting {es-pull}92742[#92742] -* Repeat `cluster.initial_master_nodes` log warning {es-pull}92744[#92744] - -EQL:: -* EQL Samples: add support for multiple samples per key {es-pull}91783[#91783] - -Engine:: -* Add commits listener for `InternalEngine` and `CombinedDeletionPolicy` {es-pull}92017[#92017] -* Add primary term supplier to Engine.IndexCommitListener {es-pull}92101[#92101] -* Adjust range of allowed percentages of deletes in an index {es-pull}93188[#93188] -* Diff the list of filenames that are added by each new commit {es-pull}92238[#92238] -* Set a fixed compound file threshold of 1GB {es-pull}92659[#92659] - -Geo:: -* Add methods to H3#hexRing to prevent allocating long arrays {es-pull}92711[#92711] -* Add methods to prevent allocating long arrays during child navigation on H3 api {es-pull}92099[#92099] -* Add new H3 api method #h3ToNoChildrenIntersecting {es-pull}91673[#91673] -* In H3, compute destination point from distance and azimuth using planar 3d math" {es-pull}93084[#93084] -* Protect H3 library against integer overflow {es-pull}92829[#92829] -* Reduce number of object allocations in H3#h3ToGeoBoundary {es-pull}91586[#91586] -* Speed H3 library by using `FastMath` implementation for trigonometric functions {es-pull}91839[#91839] - -Health:: -* Expose Health Api telemetry via xpack {es-pull}91708[#91708] (issue: {es-issue}90877[#90877]) -* Health api stats {es-pull}91559[#91559] - -Indices APIs:: -* Add `ignore_missing_component_templates` config option {es-pull}92436[#92436] (issue: {es-issue}92426[#92426]) - -Infra/CLI:: -* Scan stable plugins for named components upon install {es-pull}92528[#92528] - -Infra/Core:: -* Add log level for JVM logs {es-pull}92382[#92382] -* Added new field `rollout_duration_seconds` to fleet-actions {es-pull}92640[#92640] -* Bind the readiness service to the wildcard address {es-pull}91329[#91329] (issue: {es-issue}90997[#90997]) -* Provide locally mounted secure settings implementation {es-pull}93392[#93392] - -Infra/Plugins:: -* Check stable plugin version at install and load time {es-pull}91780[#91780] -* Example stable plugins with settings {es-pull}92334[#92334] -* Load stable plugins as synthetic modules {es-pull}91869[#91869] -* Settings api for stable plugins {es-pull}91467[#91467] - -Infra/Scripting:: -* Script: Metadata `validateMetadata` optimization {es-pull}93333[#93333] -* Short-circuit painless def equality {es-pull}92102[#92102] -* Use primitive types rather than boxing/unboxing for iterating over primitive arrays from defs {es-pull}92025[#92025] - -Ingest Node:: -* Cache the creation of parsers within DateProcessor {es-pull}92880[#92880] -* Make `GeoIpProcessor` backing database instance pluggable {es-pull}93285[#93285] - -Machine Learning:: -* Add identification of multimodal distribution to anomaly explanations {ml-pull}2440[#2440] -* Add the ability to include and exclude values in Frequent items {es-pull}92414[#92414] -* Better error when `aggregate_metric_double` used in scrolling datafeeds {es-pull}92232[#92232] (issue: {es-issue}90592[#90592]) -* Implement extension pruning in frequent items to improve runtime {es-pull}92322[#92322] -* Improve `frequent_items` performance using global ordinals {es-pull}93304[#93304] -* Improve anomaly detection results indexing speed {es-pull}92417[#92417] -* Improve frequent items runtime {es-pull}93255[#93255] -* Increase the default timeout for the start trained model deployment API {es-pull}92328[#92328] -* Option to delete user-added annotations for the reset/delete job APIs {es-pull}91698[#91698] (issue: {es-issue}74310[#74310]) -* Persist data counts and datafeed timing stats asynchronously {es-pull}93000[#93000] -* Remove the PyTorch inference work queue as now handled in Elasticsearch {ml-pull}2456[#2456] -* Text Embedding search {es-pull}93531[#93531] -* Upgrade PyTorch to version 1.13.1 {ml-pull}2430[#2430] - - -Mapping:: -* Switch to Lucene's new `IntField/LongField/FloatField/DoubleField` {es-pull}93165[#93165] - -Monitoring:: -* Add kibana.stats.elasticsearch_client stats to the monitoring index templates. {es-pull}91508[#91508] -* Add monitoring mappings for es ingest metricset {es-pull}92950[#92950] - -Network:: -* Deserialize responses on the handling thread-pool {es-pull}91367[#91367] - -Performance:: -* Add vector distance scoring to micro benchmarks {es-pull}92340[#92340] - -Query Languages:: -* Introduce parameterized rule and executor {es-pull}92428[#92428] - -Recovery:: -* Make clean up files step configurable for peer-recovery of replicas {es-pull}92490[#92490] - -Search:: -* Access term dictionary more efficiently {es-pull}92269[#92269] -* Add `term` query support to `rank_features` mapped field {es-pull}93247[#93247] -* Add new `query_vector_builder` option to knn search clause {es-pull}93331[#93331] -* Add profiling plugin {es-pull}91640[#91640] -* Enable profiling plugin by default {es-pull}92787[#92787] -* Get stackframes and executables more concurrently {es-pull}93559[#93559] -* Improve the false positive rate of the bloom filter by setting 7 hash functions {es-pull}93283[#93283] -* Increase the number of threads of GET threadpool {es-pull}92309[#92309] -* Instrument Weight#count in ProfileWeight {es-pull}85656[#85656] (issue: {es-issue}85203[#85203]) -* Reduce memory usage of match all bitset {es-pull}92777[#92777] -* Runtime fields to optionally ignore script errors {es-pull}92380[#92380] -* Speed up retrieval of data for flamegraphs {es-pull}93448[#93448] -* Support retrieving inlined stack frames {es-pull}92863[#92863] -* [Profiling] Reduce GC pressure {es-pull}93590[#93590] - -Security:: -* Configurable retention period for invalidated or expired API keys {es-pull}92219[#92219] -* Record timestamp on API key invalidation {es-pull}91873[#91873] - -Snapshot/Restore:: -* Make `RecoveryPlannerService` optional {es-pull}92489[#92489] - -TSDB:: -* Enable bloom filter for `_id` field in tsdb indices {es-pull}92115[#92115] -* Improve downsampling performance by removing map lookups {es-pull}92494[#92494] (issue: {es-issue}90226[#90226]) -* Minor TSDB parsing speedup {es-pull}92276[#92276] -* Skip duplicate checks on segments that don't contain the document's timestamp {es-pull}92456[#92456] -* Support `fields` in synthetic source in last cases {es-pull}91595[#91595] - -Task Management:: -* `TransportGetTaskAction:` Wait for the task asynchronously {es-pull}93375[#93375] -* `TransportListTaskAction:` wait for tasks to finish asynchronously {es-pull}90977[#90977] (issue: {es-issue}89564[#89564]) - -Transform:: -* Add from parameter to Transform Start API {es-pull}91116[#91116] (issue: {es-issue}88646[#88646]) -* Support "offset" parameter in `DateHistogramGroupSource` {es-pull}93203[#93203] -* Trigger state persistence based on time {es-pull}93221[#93221] - -Vector Search:: -* Allow `null` to be provided for `dense_vector` field values {es-pull}93388[#93388] -* Allow more than one KNN search clause {es-pull}92118[#92118] (issue: {es-issue}91187[#91187]) - -Watcher:: -* Add ability for Watcher's webhook actions to send additional header {es-pull}93426[#93426] - -[[feature-8.7.0]] -[float] -=== New features - -Distributed:: -* Secure settings that can fall back to yml in Stateless {es-pull}91925[#91925] - -Geo:: -* Geohex aggregation on `geo_shape` field {es-pull}91956[#91956] (issue: {es-issue}90163[#90163]) -* Support geo_grid ingest processor {es-pull}93370[#93370] (issue: {es-issue}92473[#92473]) - -Health:: -* The Health API is now generally available {es-pull}92879[#92879] -* [HealthAPI] Add size parameter that controls the number of affected resources returned {es-pull}92399[#92399] (issue: {es-issue}91930[#91930]) -* [HealthAPI] Add support for the FEATURE_STATE affected resource {es-pull}92296[#92296] (issue: {es-issue}91353[#91353]) - -Infra/Plugins:: -* [Fleet] Add files and files data index templates and ILM policies {es-pull}91413[#91413] - -Ingest Node:: -* Redact Ingest Processor {es-pull}92951[#92951] - -Machine Learning:: -* Make `frequent_item_sets` aggregation GA {es-pull}93421[#93421] -* Make native inference generally available {es-pull}92213[#92213] - -TSDB:: -* Add a TSDB rate aggregation {es-pull}90447[#90447] -* Downsampling GA {es-pull}92913[#92913] -* Release time_series and rate (on counter fields) aggegations as tech preview {es-pull}93546[#93546] -* Time series (TSDS) GA {es-pull}91519[#91519] - -Transform:: -* Transform _schedule_now API {es-pull}92948[#92948] (issue: {es-issue}44722[#44722]) - -[[upgrade-8.7.0]] -[float] -=== Upgrades - -Infra/Core:: -* Align all usages of Jackson to be 2.14.2 {es-pull}93438[#93438] - -Ingest Node:: -* Upgrading tika to 2.6.0 {es-pull}92104[#92104] - -Network:: -* Upgrade to Netty 4.1.85 {es-pull}91846[#91846] -* Upgrade to Netty 4.1.86 {es-pull}92587[#92587] - -Query Languages:: -* Upgrade antlr to 4.11.1 for ql, eql and sql {es-pull}93238[#93238] - -Search:: -* Upgrade to Lucene 9.5.0 {es-pull}93385[#93385] -* Upgrade to lucene-9.5.0-snapshot-d19c3e2e0ed {es-pull}92957[#92957] - -Snapshot/Restore:: -* Align all usages of protobuf to be 3.21.9 {es-pull}92123[#92123] -* Bump reactor netty version {es-pull}92457[#92457] -* Consolidate google-oauth-client to latest version {es-pull}91722[#91722] - - diff --git a/docs/reference/release-notes/8.7.1.asciidoc b/docs/reference/release-notes/8.7.1.asciidoc deleted file mode 100644 index 70f5e4add88ca..0000000000000 --- a/docs/reference/release-notes/8.7.1.asciidoc +++ /dev/null @@ -1,80 +0,0 @@ -[[release-notes-8.7.1]] -== {es} version 8.7.1 - -Also see <>. - -[[known-issues-8.7.1]] -[float] -=== Known issues - -* `ArrayIndexOutOfBoundsException` may be thrown while creating a transport message -+ -Certain sequences of writes and seeks to the buffer used to create a transport -message may encounter an alignment bug which results in an -`ArrayIndexOutOfBoundsException`, preventing the transport message from being -sent. -+ -This issue is fixed in 8.8.0. - -include::8.6.0.asciidoc[tag=reconciliation-imbalance-known-issue] - -// tag::no-preventive-gc-issue[] -* High Memory Pressure due to a GC JVM setting change -+ -This version of Elasticsearch is bundled with JDK 20. In JDK 20 -https://bugs.openjdk.org/browse/JDK-8293861[Preventive GC is disabled by default]. -This may lead to increased memory pressure and an increased number of CircuitBreakerExceptions when retrieving large -documents under some load patterns. (issue: {es-issue}99592[#99592]) -+ -If this change affects your use of Elasticsearch, consider re-enabling the previous behaviour -by adding the JVM arguments `-XX:+UnlockDiagnosticVMOptions -XX:+G1UsePreventiveGC` (reference: -https://www.oracle.com/java/technologies/javase/20-relnote-issues.html#JDK-8293861[JDK 20 release notes]). It is -important to note that this workaround is temporary and works only with JDK 20, which is bundled with Elasticsearch up -to version 8.10.2 inclusive. Successive versions are bundling JDK 21+, where this setting -https://bugs.openjdk.org/browse/JDK-8297639[has been removed]. Specifying those JVM arguments will prevent the -JVM (and therefore Elasticsearch Nodes) from starting. -// end::no-preventive-gc-issue[] - -[[bug-8.7.1]] -[float] -=== Bug fixes - -Allocation:: -* Compute balancer threshold based on max shard size {es-pull}95090[#95090] -* Use applied state after `DiskThresholdMonitor` reroute {es-pull}94916[#94916] -* Weaken node-replacement decider during reconciliation {es-pull}95070[#95070] - -ILM+SLM:: -* Downsample ILM action should skip non-time-series indices {es-pull}94835[#94835] (issue: {es-issue}93123[#93123]) - -Ingest Node:: -* Fix async enrich execution prematurely releases enrich policy lock {es-pull}94702[#94702] (issue: {es-issue}94690[#94690]) - -Network:: -* Fix off-by-one bug in `RecyclerBytesStreamOutput` {es-pull}95036[#95036] - -Recovery:: -* Async creation of `IndexShard` instances {es-pull}94545[#94545] - -Search:: -* Return 200 when closing empty PIT or scroll {es-pull}94708[#94708] - -Stats:: -* Fix _cluster/stats `.nodes.fs` deduplication {es-pull}94798[#94798] (issue: {es-issue}24472[#24472]) -* Fix `FsInfo` device deduplication {es-pull}94744[#94744] - -[[enhancement-8.7.1]] -[float] -=== Enhancements - -Authorization:: -* Reuse `FieldPermissionsCache` in Role parsing {es-pull}94931[#94931] - -[[upgrade-8.7.1]] -[float] -=== Upgrades - -Packaging:: -* Upgrade bundled JDK to Java 20 {es-pull}94600[#94600] - - diff --git a/docs/reference/release-notes/8.8.0.asciidoc b/docs/reference/release-notes/8.8.0.asciidoc deleted file mode 100644 index da47bd5e386bb..0000000000000 --- a/docs/reference/release-notes/8.8.0.asciidoc +++ /dev/null @@ -1,310 +0,0 @@ -[[release-notes-8.8.0]] -== {es} version 8.8.0 - -Also see <>. - -[[bug-8.8.0]] -[float] -=== Bug fixes - -Aggregations:: -* Merge two histograms using the higher number of digits among all histograms {es-pull}93704[#93704] (issue: {es-issue}92822[#92822]) - -Allocation:: -* Avoid copying during iteration of all shards in routing table {es-pull}94417[#94417] -* Avoid duplicate application of `RoutingTable` diff {es-pull}94379[#94379] -* Balance priorities during reconciliation {es-pull}95454[#95454] -* Fix `RebalanceOnlyWhenActiveAllocationDecider` {es-pull}96025[#96025] -* Streamline `AsyncShardFetch#getNumberOfInFlightFetches` {es-pull}93632[#93632] (issue: {es-issue}93631[#93631]) - -Application:: -* Check if an analytics event data stream exists before installing pipeline {es-pull}95621[#95621] -* [Behavioral Analytics] Use a client with ent-search origin in the `BulkProcessorFactory` {es-pull}95614[#95614] - -Authorization:: -* Fix role transformation to include missing properties {es-pull}94714[#94714] -* [Fleet] Add read privileges to `profiling-*` for symbolization support {es-pull}95596[#95596] - -CRUD:: -* Avoid null `Location` in post write refresh {es-pull}95229[#95229] - -Cluster Coordination:: -* Read register current term asynchronously in `StoreHeartbeatService` {es-pull}95351[#95351] - -DLM:: -* Remove rollover cluster setting validator {es-pull}94447[#94447] -* [DLM] Fix the new endpoint rest-api specification {es-pull}95665[#95665] - -Data streams:: -* Allow deletion of component templates that are specified in the `ignore_missing_component_templates` array {es-pull}95527[#95527] -* Fix searching a filtered and unfiltered data stream alias {es-pull}95865[#95865] (issue: {es-issue}95786[#95786]) - -Distributed:: -* Check shard availability before including in stats {es-pull}96015[#96015] (issues: {es-issue}96000[#96000], {es-issue}87001[#87001]) -* Fix `GetPipelineResponse` equality {es-pull}93695[#93695] - -Engine:: -* Ensure refresh to return the latest commit generation {es-pull}94249[#94249] - -Geo:: -* Adjust `BoundedGeoHexGridTiler#FACTOR` to prevent missing hits {es-pull}96088[#96088] (issue: {es-issue}96057[#96057]) -* Fix bug where `geo_line` does not respect `sort_order` {es-pull}94734[#94734] (issue: {es-issue}94733[#94733]) - -ILM+SLM:: -* Retry downsample ILM action using a new target index {es-pull}94965[#94965] (issue: {es-issue}93580[#93580]) -* Strip disallowed chars from generated snapshot name {es-pull}95767[#95767] (issue: {es-issue}95593[#95593]) -* [ILM] Fix the migrate to tiers service and migrate action tiers configuration {es-pull}95934[#95934] - -Infra/Core:: -* Fix race condition in `NodeEnvironment.close()` {es-pull}94677[#94677] (issue: {es-issue}94672[#94672]) -* Use double wildcards for filtered excludes properly {es-pull}94195[#94195] (issue: {es-issue}92632[#92632]) - -Infra/REST API:: -* Add level parameter validation in REST layer {es-pull}94136[#94136] (issue: {es-issue}93981[#93981]) - -Infra/Scripting:: -* Allow low level paging in `LeafDocLookup` {es-pull}93711[#93711] -* Revert usage of `SafeMustacheFactory` in `CustomMustacheFactory` {es-pull}95557[#95557] - -Ingest Node:: -* Fix `Grok.match()` with offset and suffix pattern {es-pull}95003[#95003] (issue: {es-issue}95002[#95002]) -* Fix bug in verbose simulations of the ingest pipeline API {es-pull}95232[#95232] - -Machine Learning:: -* Avoid expensive source parsing by using doc values when querying model definition meta fields {es-pull}95590[#95590] - -Mapping:: -* Longer timeout for mapping update during resize {es-pull}95221[#95221] - -Network:: -* Fix `RecyclerBytesStreamOutput` corrupting when ending write on page boundary {es-pull}95114[#95114] -* Fix maximum seek limit `RecyclerBytesStreamOutput` {es-pull}95133[#95133] - -Ranking:: -* Fix versioning for tests cases using a randomly generated rank builder {es-pull}95514[#95514] - -Search:: -* Fix `_terms_enum` display values {es-pull}94080[#94080] (issue: {es-issue}94041[#94041]) -* Support ignore malformed in boolean fields {es-pull}93239[#93239] (issue: {es-issue}89542[#89542]) -* Support search template api explain query string argument {es-pull}94832[#94832] (issue: {es-issue}83363[#83363]) - -Snapshot/Restore:: -* Cancel cold cache prewarming tasks if store is closing {es-pull}95891[#95891] (issue: {es-issue}95504[#95504]) -* Fix 0 default value for repo snapshot speed {es-pull}95854[#95854] (issue: {es-issue}95561[#95561]) -* Fix Azure `InputStream#read` method {es-pull}96034[#96034] -* Stop sorting indices in get-snapshots API {es-pull}94890[#94890] - -Transform:: -* Call listener in order to prevent the request from hanging {es-pull}96221[#96221] -* Do not fail upon `ResourceAlreadyExistsException` during destination index creation {es-pull}96274[#96274] (issue: {es-issue}95310[#95310]) -* Fix privileges check failures by adding `allow_restricted_indices` flag {es-pull}95187[#95187] -* Secondary credentials used with transforms should only require source and destination index privileges, not transform privileges {es-pull}94420[#94420] -* Use monotonic time in `TransformScheduler` {es-pull}95456[#95456] (issue: {es-issue}95445[#95445]) - -[[deprecation-8.8.0]] -[float] -=== Deprecations - -Allocation:: -* Deprecate `cluster.routing.allocation.type` {es-pull}94066[#94066] - -[[enhancement-8.8.0]] -[float] -=== Enhancements - -Aggregations:: -* Add `keyed` parameter to filters agg, allowing the user to get non-keyed buckets of named filters agg {es-pull}89256[#89256] (issue: {es-issue}83957[#83957]) -* Add global ordinal info to stats APIs {es-pull}94500[#94500] -* Don't create many `Rounding.Prepared` instances when checking for empty buckets in date_histogram aggregator. {es-pull}94649[#94649] - -Analysis:: -* Add origin of synonym rules to exception message {es-pull}93702[#93702] - -Application:: -* Behavioral Analytics event ingest tuning {es-pull}95405[#95405] -* [Behavioral Analytics] Add geo ip and user agent to events {es-pull}95433[#95433] -* [Behavioral analytics] Implement search filters into events {es-pull}95212[#95212] - -Authentication:: -* Do not fail node if SAML HTTP metadata is unavailable {es-pull}92810[#92810] (issue: {es-issue}37608[#37608]) -* Finer control over authentication metadata serialization {es-pull}93726[#93726] - -Authorization:: -* Add permissions to `kibana_system` for TI package transforms to support IOC expiration {es-pull}94506[#94506] (issue: {es-issue}94505[#94505]) -* Ensure checking indices privileges works with `nested-limited-role` {es-pull}95170[#95170] - -Cluster Coordination:: -* Improve master service batching queues {es-pull}92021[#92021] (issue: {es-issue}81626[#81626]) - -DLM:: -* Adding origination date to DLM {es-pull}95113[#95113] - -Engine:: -* Increase the merge factor to 32 for time-based data {es-pull}94134[#94134] -* Reduce the likelihood of writing small segments due to an oversize translog {es-pull}93524[#93524] (issue: {es-issue}75611[#75611]) -* Sort segments on timestamp in read only engine {es-pull}93576[#93576] -* Use `LogByteSizeMergePolicy` instead of `TieredMergePolicy` for time-based data {es-pull}92684[#92684] -* Use mmap for temporary files {es-pull}93595[#93595] - -Geo:: -* Allow docvalues-only search on `geo_shape` {es-pull}94396[#94396] -* Support for store parameter in `geo_shape` field {es-pull}94418[#94418] (issue: {es-issue}83655[#83655]) - -Highlighting:: -* Use `storedFieldsSpec` to load stored fields for highlighting {es-pull}91841[#91841] - -ILM+SLM:: -* Implicitly rollover data streams / aliases based on `max_primary_shard_docs` {es-pull}94065[#94065] (issue: {es-issue}87246[#87246]) -* Sort ILM explain output by natural index name {es-pull}94879[#94879] (issue: {es-issue}94768[#94768]) - -Indices APIs:: -* Adding initial public and internal serverless scopes to data management rest handlers {es-pull}93990[#93990] -* Servlerless API protection with annotations {es-pull}93607[#93607] - -Infra/Core:: -* Allow preserving specific headers on thread context stash {es-pull}94680[#94680] - -Infra/Plugins:: -* Improve module/plugin loading logging message. {es-pull}93952[#93952] (issue: {es-issue}93881[#93881]) - -Infra/Transport API:: -* Add `transport_version` to node info JSON {es-pull}94669[#94669] - -Ingest Node:: -* Add `reroute` processor {es-pull}76511[#76511] -* Introduce redirect method on `IngestDocument` {es-pull}94000[#94000] (issue: {es-issue}83653[#83653]) -* [Ingest Processor] Add `ignore_missing` param to the `uri_parts` ingest processor {es-pull}95068[#95068] - -Machine Learning:: -* Add `_meta` field to data frame analytics config {es-pull}94529[#94529] -* Add `embedding_size` to text embedding config {es-pull}95176[#95176] -* Include model definition install status for Pytorch models {es-pull}95271[#95271] -* Integrate ELSER model download into put trained model API {es-pull}95281[#95281] -* Start, stop and infer of a trained model can now optionally use a deployment ID that is different to the model ID {es-pull}95168[#95168] -* [ML] Get trained model stats by deployment id or model id {es-pull}95440[#95440] - -Mapping:: -* Cut over from Field to `StringField` when applicable {es-pull}94540[#94540] -* Enable `_terms_enum` on `ip` fields {es-pull}94322[#94322] (issue: {es-issue}89933[#89933]) -* Enable synthetic source for malformed booleans {es-pull}94121[#94121] -* Index sequence numbers via a single Lucene field {es-pull}94504[#94504] -* Use a combined field to index terms and doc values on keyword fields {es-pull}93579[#93579] - -Monitoring:: -* Add `event_loop_utilization` Kibana stats to the monitoring index templates {es-pull}95388[#95388] - -Network:: -* Add request/response body logging to HTTP tracer {es-pull}93133[#93133] -* Avoid deserializing responses in proxy node {es-pull}93799[#93799] -* Report transport message size per action {es-pull}94543[#94543] (issue: {es-issue}88151[#88151]) -* Retain underlying error on proxy mode connection failure {es-pull}94998[#94998] - -SQL:: -* Add `WildcardLike/Pattern` to QL {es-pull}95357[#95357] - -Search:: -* Adding initial public and internal serverless scopes to Search team REST handlers {es-pull}94035[#94035] -* Enable `_terms_enum` on version fields {es-pull}93839[#93839] (issue: {es-issue}83403[#83403]) -* Introduce `DocumentParsingException` {es-pull}92646[#92646] (issue: {es-issue}85083[#85083]) -* Leverage `Weight#count` when size is set to 0 {es-pull}94858[#94858] -* Make `SourceProvider` using stored fields segment-thread-safe {es-pull}95082[#95082] -* Shortcut total hit count when `terminate_after` is used {es-pull}94889[#94889] -* [Profiling] Map stack frames more efficiently {es-pull}94327[#94327] -* [Profiling] Parallelize response handling {es-pull}93960[#93960] - -Security:: -* Fleet: Add new mappings for `.fleet-actions` signing {es-pull}93802[#93802] - -Snapshot/Restore:: -* Add register analysis to repo analysis API {es-pull}93955[#93955] -* Add snapshot activity in cluster stats {es-pull}93680[#93680] -* Add support for custom endpoints in the Azure repository {es-pull}94576[#94576] (issue: {es-issue}94537[#94537]) -* Failed tasks proactively cancel children tasks {es-pull}92588[#92588] (issue: {es-issue}90353[#90353]) - -TSDB:: -* Support position `time_series_metric` on `geo_point` fields {es-pull}93946[#93946] - -Transform:: -* Add `delete_destination_index` parameter to the `Delete Transform API` {es-pull}94162[#94162] -* Allow specifying destination index aliases in the Transform's `dest` config {es-pull}94943[#94943] -* Expose authorization failure as transform health issue {es-pull}94724[#94724] - -Vector Search:: -* Increase max number of vector dims to 2048 {es-pull}95257[#95257] - -Watcher:: -* Add Watcher APIs for updating/retrieving settings {es-pull}95342[#95342] (issue: {es-issue}92991[#92991]) -* Porting watcher over to `BulkProcessor2` {es-pull}94133[#94133] - -[[feature-8.8.0]] -[float] -=== New features - -Application:: -* Initial Search Application Search API with templates {es-pull}95026[#95026] -* [Behavioral Analytics] Add a `final_pipeline` to event data streams {es-pull}95198[#95198] - -Authentication:: -* GA release of the JWT realm {es-pull}95398[#95398] - -CRUD:: -* New `TransportBroadcastUnpromotableAction` action {es-pull}93600[#93600] - -DLM:: -* Add new endpoints to configure data lifecycle on a data stream level {es-pull}94590[#94590] -* Dlm add auto rollover condition max age {es-pull}94950[#94950] -* Initial implementation for `DataLifecycleService` {es-pull}94012[#94012] -* Introduce a _lifecycle/explain API for data stream backing indices {es-pull}94621[#94621] -* Introduce the `index.lifecycle.prefer_ilm` setting {es-pull}95423[#95423] -* [DLM] Extend the template to simulate api to support include defaults {es-pull}94861[#94861] -* [DLM] Introduce default rollover cluster setting & expose it via APIs {es-pull}94240[#94240] - -Health:: -* Add new `ShardsCapacity` Health Indicator Service {es-pull}94552[#94552] -* Add to `HealthMetadata` information about `ShardLimits` {es-pull}94116[#94116] - -Ingest Node:: -* Add license checking to the redact processor {es-pull}95477[#95477] - -Machine Learning:: -* Text Expansion Query {es-pull}93694[#93694] - -Ranking:: -* Add support for Reciprocal Rank Fusion to the search API {es-pull}93396[#93396] - -Search:: -* Add Enterprise Search Module {es-pull}94381[#94381] -* Add new `similarity` field to `knn` clause in `_search` {es-pull}94828[#94828] -* Add the ability to return the score of the named queries {es-pull}94564[#94564] (issue: {es-issue}29606[#29606]) -* Implements behavioral analytics events ingest API {es-pull}95027[#95027] - -TSDB:: -* Encode using 40, 48 and 56 bits per value {es-pull}93371[#93371] -* Flattened field synthetic support {es-pull}94842[#94842] -* Support flattened fields as time series dimension fields {es-pull}95273[#95273] - -[[upgrade-8.8.0]] -[float] -=== Upgrades - -Engine:: -* Upgrade to `lucene-9.6-snapshot-dcc2154a1d3` {es-pull}94955[#94955] - -Infra/Core:: -* Upgrade Jackson xml to 2.15.0 {es-pull}95641[#95641] - -Ingest Node:: -* Upgrading tika to `2.7.0` {es-pull}93759[#93759] - -Network:: -* Upgrade to Netty `4.1.89` {es-pull}94179[#94179] - -Packaging:: -* Bump bundled JDK to Java `20.0.1` {es-pull}95359[#95359] - -Search:: -* Upgrade Lucene to the final 9.6.0 release {es-pull}95967[#95967] -* Upgrade to `lucene-9.6.0-snapshot-8a815153fbe` {es-pull}94635[#94635] -* Upgrade to `lucene-9.6.0-snapshot-f5d1e1c787c` {es-pull}94494[#94494] - - diff --git a/docs/reference/release-notes/8.8.1.asciidoc b/docs/reference/release-notes/8.8.1.asciidoc deleted file mode 100644 index 249c351241bdd..0000000000000 --- a/docs/reference/release-notes/8.8.1.asciidoc +++ /dev/null @@ -1,34 +0,0 @@ -[[release-notes-8.8.1]] -== {es} version 8.8.1 - -Also see <>. - -[[bug-8.8.1]] -[float] -=== Bug fixes - -Data streams:: -* Allow the removal of an in-use template if there are other ones matching the dependent data streams {es-pull}96286[#96286] - -Geo:: -* API rest compatibility for type parameter in `geo_bounding_box` query {es-pull}96317[#96317] - -Rollup:: -* Do not copy `index.default_pipeline` and `index.final_pipeline` {es-pull}96494[#96494] (issue: {es-issue}96478[#96478]) - -TSDB:: -* Expand start and end time to nanoseconds during coordinator rewrite when needed {es-pull}96035[#96035] (issue: {es-issue}96030[#96030]) -* Fix NPE when indexing a document that just has been deleted in a tsdb index {es-pull}96461[#96461] - -Transform:: -* Improve error message on transform `_update` conflict {es-pull}96432[#96432] -* Report version conflict on concurrent updates {es-pull}96293[#96293] (issue: {es-issue}96311[#96311]) - -[[enhancement-8.8.1]] -[float] -=== Enhancements - -Query Languages:: -* Reduce nesting of same bool queries {es-pull}96265[#96265] (issue: {es-issue}96236[#96236]) - - diff --git a/docs/reference/release-notes/8.8.2.asciidoc b/docs/reference/release-notes/8.8.2.asciidoc deleted file mode 100644 index 8a24ae2e8d4ef..0000000000000 --- a/docs/reference/release-notes/8.8.2.asciidoc +++ /dev/null @@ -1,46 +0,0 @@ -[[release-notes-8.8.2]] -== {es} version 8.8.2 - -Also see <>. - -[[known-issues-8.8.2]] -[float] -=== Known issues -include::8.7.1.asciidoc[tag=no-preventive-gc-issue] - -[[bug-8.8.2]] -[float] -=== Bug fixes - -Aggregations:: -* Fix iteration of empty percentiles throwing Null Pointer Exception {es-pull}96668[#96668] (issue: {es-issue}96626[#96626]) - -Health:: -* Uses `ClusterSettings` instead of Node `Settings` in `HealthMetadataService` {es-pull}96843[#96843] (issue: {es-issue}96219[#96219]) - -Ingest Node:: -* Support dotted field notations in the reroute processor {es-pull}96243[#96243] - -Machine Learning:: -* Ensure NLP model inference queue is always cleared after shutdown or failure {es-pull}96738[#96738] - -SQL:: -* Fix translation of queries involving Version vals {es-pull}96540[#96540] (issue: {es-issue}96509[#96509]) - -Search:: -* Increase concurrent request of opening point-in-time {es-pull}96782[#96782] - -TSDB:: -* The get data stream api incorrectly prints warning log for upgraded tsdb data streams {es-pull}96606[#96606] - -[[enhancement-8.8.2]] -[float] -=== Enhancements - -TSDB:: -* Change rollup thread pool settings {es-pull}96821[#96821] (issue: {es-issue}96758[#96758]) - -Transform:: -* Adding null check to fix potential NPE {es-pull}96785[#96785] (issue: {es-issue}96781[#96781]) - - diff --git a/docs/reference/release-notes/8.9.0.asciidoc b/docs/reference/release-notes/8.9.0.asciidoc deleted file mode 100644 index c49eac9f0327c..0000000000000 --- a/docs/reference/release-notes/8.9.0.asciidoc +++ /dev/null @@ -1,286 +0,0 @@ -[[release-notes-8.9.0]] -== {es} version 8.9.0 - -Also see <>. - -[[known-issues-8.9.0]] -[float] -=== Known issues - -* Question Answering fails on long input text. If the context supplied to the -task is longer than the model's max_sequence_length and truncate is set to none -then inference fails with the message `question answering result has -invalid dimension`. (issue: {es-issue}97917[#97917]) - -include::8.7.1.asciidoc[tag=no-preventive-gc-issue] - -[[breaking-8.9.0]] -[float] -=== Breaking changes - -Aggregations:: -* Switch TDigestState to use `HybridDigest` by default {es-pull}96904[#96904] - -[[bug-8.9.0]] -[float] -=== Bug fixes - -Allocation:: -* Attempt to fix delay allocation {es-pull}95921[#95921] -* Fix NPE in Desired Balance API {es-pull}97775[#97775] -* Fix autoexpand during node replace {es-pull}96281[#96281] - -Authorization:: -* Resolving wildcard application names without prefix query {es-pull}96479[#96479] (issue: {es-issue}96465[#96465]) - -CRUD:: -* Fix `retry_on_conflict` parameter in update API to not retry indefinitely {es-pull}96262[#96262] -* Handle failure in `TransportUpdateAction#handleUpdateFailureWithRetry` {es-pull}97290[#97290] (issue: {es-issue}97286[#97286]) - -Cluster Coordination:: -* Avoid `getStateForMasterService` where possible {es-pull}97304[#97304] -* Become candidate on publication failure {es-pull}96490[#96490] (issue: {es-issue}96273[#96273]) -* Fix cluster settings update task acknowledgment {es-pull}97111[#97111] - -Data streams:: -* Accept timestamp as object at root level {es-pull}97401[#97401] - -Geo:: -* Fix bug when creating empty `geo_lines` {es-pull}97509[#97509] (issue: {es-issue}97311[#97311]) -* Fix time-series geo_line to include reduce phase in MergedGeoLines {es-pull}96953[#96953] (issue: {es-issue}96983[#96983]) -* Support for Byte and Short as vector tiles features {es-pull}97619[#97619] (issue: {es-issue}97612[#97612]) - -ILM+SLM:: -* Limit the details field length we store for each SLM invocation {es-pull}97038[#97038] (issue: {es-issue}96918[#96918]) - -Infra/CLI:: -* Initialise ES logging in CLI {es-pull}97353[#97353] (issue: {es-issue}97350[#97350]) - -Infra/Core:: -* Capture max processors in static init {es-pull}97119[#97119] (issue: {es-issue}97088[#97088]) -* Interpret microseconds cpu stats from cgroups2 properly as nanos {es-pull}96924[#96924] (issue: {es-issue}96089[#96089]) - -Infra/Logging:: -* Add slf4j-nop in order to prevent startup warnings {es-pull}95459[#95459] - -Infra/REST API:: -* Fix tchar pattern in `RestRequest` {es-pull}96406[#96406] - -Infra/Scripting:: -* Fix Painless method lookup over unknown super interfaces {es-pull}97062[#97062] (issue: {es-issue}97022[#97022]) - -Infra/Settings:: -* Enable validation for `versionSettings` {es-pull}95874[#95874] (issue: {es-issue}95873[#95873]) - -Ingest Node:: -* Fixing `DateProcessor` when the format is `epoch_millis` {es-pull}95996[#95996] -* Fixing `GeoIpDownloaderStatsAction$NodeResponse` serialization by defensively copying inputs {es-pull}96777[#96777] (issue: {es-issue}96438[#96438]) -* Trim field references in reroute processor {es-pull}96941[#96941] (issue: {es-issue}96939[#96939]) - -Machine Learning:: -* Catch exceptions thrown during inference and report as errors {ml-pull}2542[#2542] -* Fix `WordPiece` tokenization where stripping accents results in an empty string {es-pull}97354[#97354] -* Improve model downloader robustness {es-pull}97274[#97274] -* Prevent high memory usage by evaluating batch inference singularly {ml-pull}2538[#2538] - -Mapping:: -* Avoid stack overflow while parsing mapping {es-pull}95705[#95705] (issue: {es-issue}52098[#52098]) -* Fix mapping parsing logic to determine synthetic source is active {es-pull}97355[#97355] (issue: {es-issue}97320[#97320]) - -Ranking:: -* Fix `sub_searches` serialization bug {es-pull}97587[#97587] - -Recovery:: -* Promptly fail recovery from snapshot {es-pull}96421[#96421] (issue: {es-issue}95525[#95525]) - -Search:: -* Prevent instantiation of `top_metrics` when sub-aggregations are present {es-pull}96180[#96180] (issue: {es-issue}95663[#95663]) -* Set new providers before building `FetchSubPhaseProcessors` {es-pull}97460[#97460] (issue: {es-issue}96284[#96284]) - -Snapshot/Restore:: -* Fix blob cache races/assertion errors {es-pull}96458[#96458] -* Fix reused/recovered bytes for files that are only partially recovered from cache {es-pull}95987[#95987] (issues: {es-issue}95970[#95970], {es-issue}95994[#95994]) -* Fix reused/recovered bytes for files that are recovered from cache {es-pull}97278[#97278] (issue: {es-issue}95994[#95994]) -* Refactor `RestoreClusterStateListener` to use `ClusterStateObserver` {es-pull}96662[#96662] (issue: {es-issue}96425[#96425]) - -TSDB:: -* Error message for misconfigured TSDB index {es-pull}96956[#96956] (issue: {es-issue}96445[#96445]) -* Min score for time series {es-pull}96878[#96878] - -Task Management:: -* Improve cancellability in `TransportTasksAction` {es-pull}96279[#96279] - -Transform:: -* Improve reporting status of the transform that is about to finish {es-pull}95672[#95672] - -[[enhancement-8.9.0]] -[float] -=== Enhancements - -Aggregations:: -* Add cluster setting to `SearchExecutionContext` to configure `TDigestExecutionHint` {es-pull}96943[#96943] -* Add support for dynamic pruning to cardinality aggregations on low-cardinality keyword fields {es-pull}92060[#92060] -* Make TDigestState configurable {es-pull}96794[#96794] -* Skip `SortingDigest` when merging a large digest in `HybridDigest` {es-pull}97099[#97099] -* Support value retrieval in `top_hits` {es-pull}95828[#95828] - -Allocation:: -* Take into account `expectedShardSize` when initializing shard in simulation {es-pull}95734[#95734] - -Analysis:: -* Create `.synonyms` system index {es-pull}95548[#95548] - -Application:: -* Add template parameters to Search Applications {es-pull}95674[#95674] -* Chunk profiling stacktrace response {es-pull}96340[#96340] -* [Profiling] Add status API {es-pull}96272[#96272] -* [Profiling] Allow to upgrade managed ILM policy {es-pull}96550[#96550] -* [Profiling] Introduce ILM for K/V indices {es-pull}96268[#96268] -* [Profiling] Require POST to retrieve stacktraces {es-pull}96790[#96790] -* [Profiling] Tweak default ILM policy {es-pull}96516[#96516] -* [Search Applications] Support arrays in stored mustache templates {es-pull}96197[#96197] - -Authentication:: -* Header validator with Security {es-pull}95112[#95112] - -Authorization:: -* Add Search ALC filter index prefix to the enterprise search user {es-pull}96885[#96885] -* Ensure checking application privileges work with nested-limited roles {es-pull}96970[#96970] - -Autoscaling:: -* Add shard explain info to `ReactiveReason` about unassigned shards {es-pull}88590[#88590] (issue: {es-issue}85243[#85243]) - -DLM:: -* Add auto force merge functionality to DLM {es-pull}95204[#95204] -* Adding `data_lifecycle` to the _xpack/usage API {es-pull}96177[#96177] -* Adding `manage_data_stream_lifecycle` index privilege and expanding `view_index_metadata` for access to data stream lifecycle APIs {es-pull}95512[#95512] -* Allow for the data lifecycle and the retention to be explicitly nullified {es-pull}95979[#95979] - -Data streams:: -* Add support for `logs@custom` component template for `logs-*-* data streams {es-pull}95481[#95481] (issue: {es-issue}95469[#95469]) -* Adding ECS dynamic mappings component and applying it to logs data streams by default {es-pull}96171[#96171] (issue: {es-issue}95538[#95538]) -* Adjust ECS dynamic templates to support `subobjects: false` {es-pull}96712[#96712] -* Automatically parse log events in logs data streams, if their `message` field contains JSON content {es-pull}96083[#96083] (issue: {es-issue}95522[#95522]) -* Change default of `ignore_malformed` to `true` in `logs-*-*` data streams {es-pull}95329[#95329] (issue: {es-issue}95224[#95224]) -* Set `@timestamp` for documents in logs data streams if missing and add support for custom pipeline {es-pull}95971[#95971] (issues: {es-issue}95537[#95537], {es-issue}95551[#95551]) -* Update data streams implicit timestamp `ignore_malformed` settings {es-pull}96051[#96051] - -Engine:: -* Cache modification time of translog writer file {es-pull}95107[#95107] -* Trigger refresh when shard becomes search active {es-pull}96321[#96321] (issue: {es-issue}95544[#95544]) - -Geo:: -* Add brute force approach to `GeoHashGridTiler` {es-pull}96863[#96863] -* Asset tracking - geo_line in time-series aggregations {es-pull}94954[#94954] - -ILM+SLM:: -* Chunk the GET _ilm/policy response {es-pull}97251[#97251] (issue: {es-issue}96569[#96569]) -* Move get lifecycle API to Management thread pool and make cancellable {es-pull}97248[#97248] (issue: {es-issue}96568[#96568]) -* Reduce WaitForNoFollowersStep requests indices shard stats {es-pull}94510[#94510] - -Indices APIs:: -* Bootstrap profiling indices at startup {es-pull}95666[#95666] - -Infra/Node Lifecycle:: -* SIGTERM node shutdown type {es-pull}95430[#95430] - -Ingest Node:: -* Add mappings for enrich fields {es-pull}96056[#96056] -* Ingest: expose reroute inquiry/reset via Elastic-internal API bridge {es-pull}96958[#96958] - -Machine Learning:: -* Improved compliance with memory limitations {ml-pull}2469[#2469] -* Improve detection of calendar cyclic components with long bucket lengths {ml-pull}2493[#2493] -* Improve detection of time shifts, for example for daylight saving {ml-pull}2479[#2479] - -Mapping:: -* Allow unsigned long field to use decay functions {es-pull}96394[#96394] (issue: {es-issue}89603[#89603]) - -Ranking:: -* Add multiple queries for ranking to the search endpoint {es-pull}96224[#96224] - -Recovery:: -* Implement `StartRecoveryRequest#getDescription` {es-pull}95731[#95731] - -Search:: -* Add search shards endpoint {es-pull}94534[#94534] -* Don't generate stacktrace in `EarlyTerminationException` and `TimeExceededException` {es-pull}95910[#95910] -* Feature/speed up binary vector decoding {es-pull}96716[#96716] -* Improve brute force vector search speed by using Lucene functions {es-pull}96617[#96617] -* Include search idle info to shard stats {es-pull}95740[#95740] (issue: {es-issue}95727[#95727]) -* Integrate CCS with new `search_shards` API {es-pull}95894[#95894] (issue: {es-issue}93730[#93730]) -* Introduce a filtered collector manager {es-pull}96824[#96824] -* Introduce minimum score collector manager {es-pull}96834[#96834] -* Skip shards when querying constant keyword fields {es-pull}96161[#96161] (issue: {es-issue}95541[#95541]) -* Support CCS minimize round trips in async search {es-pull}96012[#96012] -* Support for patter_replace filter in keyword normalizer {es-pull}96588[#96588] -* Support null_value for rank_feature field type {es-pull}95811[#95811] - -Security:: -* Add "_storage" internal user {es-pull}95694[#95694] - -Snapshot/Restore:: -* Reduce overhead in blob cache service get {es-pull}96399[#96399] - -Stats:: -* Add `ingest` information to the cluster info endpoint {es-pull}96328[#96328] (issue: {es-issue}95392[#95392]) -* Add `script` information to the cluster info endpoint {es-pull}96613[#96613] (issue: {es-issue}95394[#95394]) -* Add `thread_pool` information to the cluster info endpoint {es-pull}96407[#96407] (issue: {es-issue}95393[#95393]) - -TSDB:: -* Feature: include unit support for time series rate aggregation {es-pull}96605[#96605] (issue: {es-issue}94630[#94630]) - -Vector Search:: -* Leverage SIMD hardware instructions in Vector Search {es-pull}96453[#96453] (issue: {es-issue}96370[#96370]) - -[[feature-8.9.0]] -[float] -=== New features - -Application:: -* Enable analytics geoip in behavioral analytics {es-pull}96624[#96624] - -Authorization:: -* Support restricting access of API keys to only certain workflows {es-pull}96744[#96744] - -Data streams:: -* Adding ability to auto-install ingest pipelines and refer to them from index templates {es-pull}95782[#95782] - -Geo:: -* Geometry simplifier {es-pull}94859[#94859] - -ILM+SLM:: -* Enhance ILM Health Indicator {es-pull}96092[#96092] - -Infra/Node Lifecycle:: -* Gracefully shutdown elasticsearch {es-pull}96363[#96363] - -Infra/Plugins:: -* [Fleet] Add `.fleet-secrets` system index {es-pull}95625[#95625] (issue: {es-issue}95143[#95143]) - -Machine Learning:: -* Add support for `xlm_roberta` tokenized models {es-pull}94089[#94089] -* Removes the technical preview admonition from query_vector_builder docs {es-pull}96735[#96735] - -Snapshot/Restore:: -* Add repo throttle metrics to node stats api response {es-pull}96678[#96678] (issue: {es-issue}89385[#89385]) - -Stats:: -* New HTTP info endpoint {es-pull}96198[#96198] (issue: {es-issue}95391[#95391]) - -[[upgrade-8.9.0]] -[float] -=== Upgrades - -Infra/Transport API:: -* Bump `TransportVersion` to the first non-release version number. Transport protocol is now versioned independently of release version. {es-pull}95286[#95286] - -Network:: -* Upgrade Netty to 4.1.92 {es-pull}95575[#95575] -* Upgrade Netty to 4.1.94.Final {es-pull}97112[#97112] - -Search:: -* Upgrade Lucene to a 9.7.0 snapshot {es-pull}96433[#96433] -* Upgrade to new lucene snapshot 9.7.0-snapshot-a8602d6ef88 {es-pull}96741[#96741] - - diff --git a/docs/reference/release-notes/8.9.1.asciidoc b/docs/reference/release-notes/8.9.1.asciidoc deleted file mode 100644 index 680860622c1bb..0000000000000 --- a/docs/reference/release-notes/8.9.1.asciidoc +++ /dev/null @@ -1,56 +0,0 @@ -[[release-notes-8.9.1]] -== {es} version 8.9.1 - -Also see <>. - -[[known-issues-8.9.1]] -[float] -=== Known issues -include::8.7.1.asciidoc[tag=no-preventive-gc-issue] - -[[bug-8.9.1]] -[float] -=== Bug fixes - -Aggregations:: -* `GlobalAggregator` should call rewrite() before `createWeight()` {es-pull}98091[#98091] (issue: {es-issue}98076[#98076]) - -Cluster Coordination:: -* Improve exception handling in Coordinator#publish {es-pull}97840[#97840] (issue: {es-issue}97798[#97798]) - -EQL:: -* Backport fix for async missing events and re-enable the feature {es-pull}98130[#98130] - -ILM+SLM:: -* Ignore the `total_shards_per_node` setting on searchable snapshots in frozen {es-pull}97979[#97979] -* Migrate to data tiers routing configures correct default for mounted indices {es-pull}97936[#97936] (issue: {es-issue}97898[#97898]) - -Infra/Core:: -* Fix APM trace start time {es-pull}98113[#98113] - -Infra/Logging:: -* Add Configuration to `PatternLayout` {es-pull}97679[#97679] - -Machine Learning:: -* Fix failure processing Question Answering model output where the input has been spanned over multiple sequences {es-pull}98167[#98167] (issue: {es-issue}97917[#97917]) - -Search:: -* `UnmappedFieldFetcher` should ignore nested fields {es-pull}97987[#97987] (issue: {es-issue}97684[#97684]) - -[[enhancement-8.9.1]] -[float] -=== Enhancements - -Authentication:: -* Upgrade xmlsec to 2.1.8 {es-pull}97741[#97741] - -Infra/Core:: -* Enhance regex performance with duplicate wildcards {es-pull}98176[#98176] - -Machine Learning:: -* Add setting to scale the processor count used in the model assignment planner {es-pull}98296[#98296] - -Search:: -* Refactor nested field handling in `FieldFetcher` {es-pull}97683[#97683] - - diff --git a/docs/reference/release-notes/8.9.2.asciidoc b/docs/reference/release-notes/8.9.2.asciidoc deleted file mode 100644 index 8464d21e1ccc4..0000000000000 --- a/docs/reference/release-notes/8.9.2.asciidoc +++ /dev/null @@ -1,43 +0,0 @@ -[[release-notes-8.9.2]] -== {es} version 8.9.2 - -Also see <>. - -[[known-issues-8.9.2]] -[float] -=== Known issues -include::8.7.1.asciidoc[tag=no-preventive-gc-issue] - -[float] -[[security-updates-8.9.2]] -=== Security updates - -* {es} generally filters out sensitive information and credentials before -logging to the audit log. It was found that this filtering was not applied when -requests to {es} use certain deprecated `_xpack/security` URIs for APIs. The -impact of this flaw is that sensitive information, such as passwords and tokens, -might be printed in cleartext in {es} audit logs. Note that audit logging is -disabled by default and needs to be explicitly enabled. Even when audit logging -is enabled, request bodies that could contain sensitive information are not -printed to the audit log unless explicitly configured. -+ -The issue is resolved in {es} 8.9.2. -+ -For more information, see our related -https://discuss.elastic.co/t/elasticsearch-8-9-2-and-7-17-13-security-update/342479[security -announcement]. - -[[bug-8.9.2]] -[float] -=== Bug fixes - -Data streams:: -* Avoid lifecycle NPE in the data stream lifecycle usage API {es-pull}98260[#98260] - -Geo:: -* Fix mvt error when returning partial results {es-pull}98765[#98765] (issue: {es-issue}98730[#98730]) - -Ingest Node:: -* Revert "Add mappings for enrich fields" {es-pull}98683[#98683] - - diff --git a/docs/reference/release-notes/9.0.0.asciidoc b/docs/reference/release-notes/9.0.0.asciidoc new file mode 100644 index 0000000000000..af26fd57385e3 --- /dev/null +++ b/docs/reference/release-notes/9.0.0.asciidoc @@ -0,0 +1,557 @@ +// THIS IS A GENERATED FILE. DO NOT EDIT DIRECTLY. +// The content generated here are is not correct and most has been manually commented out until it can be fixed. +// See ES-9931 for more details. +[[release-notes-9.0.0]] +== {es} version 9.0.0 + +coming[9.0.0] + +Also see <>. + +[[breaking-9.0.0]] +[float] +=== Breaking changes + +// Allocation:: +// * Remove cluster state from `/_cluster/reroute` response {es-pull}114231[#114231] (issue: {es-issue}88978[#88978]) +// +// Analysis:: +// * Set lenient to true by default when using updateable synonyms {es-pull}110901[#110901] +// * Snowball stemmers have been upgraded {es-pull}114146[#114146] +// * The 'german2' stemmer is now an alias for the 'german' snowball stemmer {es-pull}113614[#113614] +// * The 'persian' analyzer has stemmer by default {es-pull}113482[#113482] (issue: {es-issue}113050[#113050]) +// * The Korean dictionary for Nori has been updated {es-pull}114124[#114124] +// +// Cluster Coordination:: +// * Remove unsupported legacy value for `discovery.type` {es-pull}112903[#112903] +// +// Data streams:: +// * Update data stream lifecycle telemetry to track global retention {es-pull}112451[#112451] +// +// ES|QL:: +// * ESQL: Entirely remove META FUNCTIONS {es-pull}113967[#113967] +// +// Indices APIs:: +// * Remove deprecated local attribute from alias APIs {es-pull}115393[#115393] +// +// Mapping:: +// * JDK locale database change {es-pull}113975[#113975] +// +// Search:: +// * Adding breaking change entry for retrievers {es-pull}115399[#115399] + +[[bug-9.0.0]] +[float] +=== Bug fixes +// +// Aggregations:: +// * Always check the parent breaker with zero bytes in `PreallocatedCircuitBreakerService` {es-pull}115181[#115181] +// * Force using the last centroid during merging {es-pull}111644[#111644] (issue: {es-issue}111065[#111065]) +// +// Authentication:: +// * Check for disabling own user in Put User API {es-pull}112262[#112262] (issue: {es-issue}90205[#90205]) +// * Expose cluster-state role mappings in APIs {es-pull}114951[#114951] +// +// Authorization:: +// * Fix DLS & FLS sometimes being enforced when it is disabled {es-pull}111915[#111915] (issue: {es-issue}94709[#94709]) +// * Fix DLS using runtime fields and synthetic source {es-pull}112341[#112341] +// +// CRUD:: +// * Don't fail retention lease sync actions due to capacity constraints {es-pull}109414[#109414] (issue: {es-issue}105926[#105926]) +// * Preserve thread context when waiting for segment generation in RTG {es-pull}114623[#114623] +// * Standardize error code when bulk body is invalid {es-pull}114869[#114869] +// +// Cluster Coordination:: +// * Ensure clean thread context in `MasterService` {es-pull}114512[#114512] +// +// Data streams:: +// * Adding support for data streams with a match-all template {es-pull}111311[#111311] (issue: {es-issue}111204[#111204]) +// * Exclude internal data streams from global retention {es-pull}112100[#112100] +// * Fix verbose get data stream API not requiring extra privileges {es-pull}112973[#112973] +// * OTel mappings: avoid metrics to be rejected when attributes are malformed {es-pull}114856[#114856] +// * [otel-data] Add more kubernetes aliases {es-pull}115429[#115429] +// * logs-apm.error-*: define log.level field as keyword {es-pull}112440[#112440] +// +// Distributed:: +// * Handle `InternalSendException` inline for non-forking handlers {es-pull}114375[#114375] +// +// EQL:: +// * Don't use a `BytesStreamOutput` to copy keys in `BytesRefBlockHash` {es-pull}114819[#114819] (issue: {es-issue}114599[#114599]) +// * Fix validation of TEXT fields with case insensitive comparison {es-pull}111238[#111238] (issue: {es-issue}111235[#111235]) +// +// ES|QL:: +// * ESQL: Add Values aggregation tests, fix `ConstantBytesRefBlock` memory handling {es-pull}111367[#111367] +// * ESQL: Align year diffing to the rest of the units in DATE_DIFF: chronological {es-pull}113103[#113103] (issue: {es-issue}112482[#112482]) +// * ESQL: Disable pushdown of WHERE past STATS {es-pull}115308[#115308] (issue: {es-issue}115281[#115281]) +// * ESQL: Fix CASE when conditions are multivalued {es-pull}112401[#112401] (issue: {es-issue}112359[#112359]) +// * ESQL: Fix Double operations returning infinite {es-pull}111064[#111064] (issue: {es-issue}111026[#111026]) +// * ESQL: Fix `REVERSE` with backspace character {es-pull}115245[#115245] (issues: {es-issue}114372[#114372], {es-issue}115227[#115227], {es-issue}115228[#115228]) +// * ESQL: Fix a bug in `MV_PERCENTILE` {es-pull}112218[#112218] (issues: {es-issue}112193[#112193], {es-issue}112180[#112180], {es-issue}112187[#112187], {es-issue}112188[#112188]) +// * ESQL: Fix filtered grouping on ords {es-pull}115312[#115312] (issue: {es-issue}114897[#114897]) +// * ESQL: Fix grammar changes around per agg filtering {es-pull}114848[#114848] +// * ESQL: Fix serialization during `can_match` {es-pull}111779[#111779] (issues: {es-issue}111701[#111701], {es-issue}111726[#111726]) +// * ESQL: Fix synthetic attribute pruning {es-pull}111413[#111413] (issue: {es-issue}105821[#105821]) +// * ESQL: don't lose the original casting error message {es-pull}111968[#111968] (issue: {es-issue}111967[#111967]) +// * ESQL: fix for missing indices error message {es-pull}111797[#111797] (issue: {es-issue}111712[#111712]) +// * ES|QL: Fix stats by constant expression {es-pull}114899[#114899] +// * ES|QL: Restrict sorting for `_source` and counter field types {es-pull}114638[#114638] (issues: {es-issue}114423[#114423], {es-issue}111976[#111976]) +// * ES|QL: better validation for GROK patterns {es-pull}110574[#110574] (issue: {es-issue}110533[#110533]) +// * ES|QL: better validation for RLIKE patterns {es-pull}112489[#112489] (issue: {es-issue}112485[#112485]) +// * ES|QL: better validation of GROK patterns {es-pull}112200[#112200] (issue: {es-issue}112111[#112111]) +// * Fix ST_CENTROID_AGG when no records are aggregated {es-pull}114888[#114888] (issue: {es-issue}106025[#106025]) +// * Fix TDigestState.read CB leaks {es-pull}114303[#114303] (issue: {es-issue}114194[#114194]) +// * Spatial search functions support multi-valued fields in compute engine {es-pull}112063[#112063] (issues: {es-issue}112102[#112102], {es-issue}112505[#112505], {es-issue}110830[#110830]) +// * [ES|QL] Check expression resolved before checking its data type in `ImplicitCasting` {es-pull}113314[#113314] (issue: {es-issue}113242[#113242]) +// * [ES|QL] Simplify patterns for subfields {es-pull}111118[#111118] +// * [ES|QL] Simplify syntax of named parameter for identifier and pattern {es-pull}115061[#115061] +// * [ES|QL] Skip validating remote cluster index names in parser {es-pull}114271[#114271] +// * [ES|QL] Use `RangeQuery` and String in `BinaryComparison` on datetime fields {es-pull}110669[#110669] (issue: {es-issue}107900[#107900]) +// * [ES|QL] add tests for stats by constant {es-pull}110593[#110593] (issue: {es-issue}105383[#105383]) +// * [ES|QL] make named parameter for identifier and pattern snapshot {es-pull}114784[#114784] +// * [ES|QL] validate `mv_sort` order {es-pull}110021[#110021] (issue: {es-issue}109910[#109910]) +// +// Geo:: +// * Fix cases of collections with one point {es-pull}111193[#111193] (issue: {es-issue}110982[#110982]) +// +// Health:: +// * Set `replica_unassigned_buffer_time` in constructor {es-pull}112612[#112612] +// +// ILM+SLM:: +// * Make `SnapshotLifecycleStats` immutable so `SnapshotLifecycleMetadata.EMPTY` isn't changed as side-effect {es-pull}111215[#111215] +// +// Indices APIs:: +// * Revert "Add `ResolvedExpression` wrapper" {es-pull}115317[#115317] +// +// Infra/Core:: +// * Fix max file size check to use `getMaxFileSize` {es-pull}113723[#113723] (issue: {es-issue}113705[#113705]) +// * Guard blob store local directory creation with `doPrivileged` {es-pull}115459[#115459] +// * Handle `BigInteger` in xcontent copy {es-pull}111937[#111937] (issue: {es-issue}111812[#111812]) +// * Report JVM stats for all memory pools (97046) {es-pull}115117[#115117] (issue: {es-issue}97046[#97046]) +// * `ByteArrayStreamInput:` Return -1 when there are no more bytes to read {es-pull}112214[#112214] +// +// Infra/Logging:: +// * Only emit product origin in deprecation log if present {es-pull}111683[#111683] (issue: {es-issue}81757[#81757]) +// +// Infra/Metrics:: +// * Make `randomInstantBetween` always return value in range [minInstant, `maxInstant]` {es-pull}114177[#114177] +// +// Infra/REST API:: +// * Fixed a `NullPointerException` in `_capabilities` API when the `path` parameter is null. {es-pull}113413[#113413] (issue: {es-issue}113413[#113413]) +// +// Infra/Settings:: +// * GET _cluster/settings with include_defaults returns the expected fallback value if defined in elasticsearch.yml {es-pull}110816[#110816] (issue: {es-issue}110815[#110815]) +// +// Ingest Node:: +// * Add warning headers for ingest pipelines containing special characters {es-pull}114837[#114837] (issue: {es-issue}104411[#104411]) +// * Fix IPinfo geolocation schema {es-pull}115147[#115147] +// * Fix `getDatabaseType` for unusual MMDBs {es-pull}112888[#112888] +// * Reducing error-level stack trace logging for normal events in `GeoIpDownloader` {es-pull}114924[#114924] +// +// License:: +// * Fix Start Trial API output acknowledgement header for features {es-pull}111740[#111740] (issue: {es-issue}111739[#111739]) +// * Fix `TokenService` always appearing used in Feature Usage {es-pull}112263[#112263] (issue: {es-issue}61956[#61956]) +// +// Logs:: +// * Do not expand dots when storing objects in ignored source {es-pull}113910[#113910] +// * Fix `ignore_above` handling in synthetic source when index level setting is used {es-pull}113570[#113570] (issue: {es-issue}113538[#113538]) +// * Fix synthetic source for flattened field when used with `ignore_above` {es-pull}113499[#113499] (issue: {es-issue}112044[#112044]) +// +// Machine Learning:: +// * Avoid `ModelAssignment` deadlock {es-pull}109684[#109684] +// * Fix NPE in Get Deployment Stats {es-pull}115404[#115404] +// * Fix bug in ML serverless autoscaling which prevented trained model updates from triggering a scale up {es-pull}110734[#110734] +// * Ignore unrecognized openai sse fields {es-pull}114715[#114715] +// * Mitigate IOSession timeouts {es-pull}115414[#115414] (issues: {es-issue}114385[#114385], {es-issue}114327[#114327], {es-issue}114105[#114105], {es-issue}114232[#114232]) +// * Prevent NPE if model assignment is removed while waiting to start {es-pull}115430[#115430] +// * Send mid-stream errors to users {es-pull}114549[#114549] +// * Temporarily return both `modelId` and `inferenceId` for GET /_inference until we migrate clients to only `inferenceId` {es-pull}111490[#111490] +// * Warn for model load failures if they have a status code <500 {es-pull}113280[#113280] +// * [Inference API] Remove unused Cohere rerank service settings fields in a BWC way {es-pull}110427[#110427] +// * [ML] Create Inference API will no longer return model_id and now only return inference_id {es-pull}112508[#112508] +// +// Mapping:: +// * Fix `MapperBuilderContext#isDataStream` when used in dynamic mappers {es-pull}110554[#110554] +// * Fix synthetic source field names for multi-fields {es-pull}112850[#112850] +// * Retrieve the source for objects and arrays in a separate parsing phase {es-pull}113027[#113027] (issue: {es-issue}112374[#112374]) +// * Two empty mappings now are created equally {es-pull}107936[#107936] (issue: {es-issue}107031[#107031]) +// +// Ranking:: +// * Fix MLTQuery handling of custom term frequencies {es-pull}110846[#110846] +// * Fix RRF validation for `rank_constant` < 1 {es-pull}112058[#112058] +// * Fix score count validation in reranker response {es-pull}111212[#111212] (issue: {es-issue}111202[#111202]) +// +// Search:: +// * Allow for querries on `_tier` to skip shards in the `can_match` phase {es-pull}114990[#114990] (issue: {es-issue}114910[#114910]) +// * Allow out of range term queries for numeric types {es-pull}112916[#112916] +// * Do not exclude empty arrays or empty objects in source filtering {es-pull}112250[#112250] (issue: {es-issue}109668[#109668]) +// * Fix synthetic source handling for `bit` type in `dense_vector` field {es-pull}114407[#114407] (issue: {es-issue}114402[#114402]) +// * Improve DateTime error handling and add some bad date tests {es-pull}112723[#112723] (issue: {es-issue}112190[#112190]) +// * Improve date expression/remote handling in index names {es-pull}112405[#112405] (issue: {es-issue}112243[#112243]) +// * Make "too many clauses" throw IllegalArgumentException to avoid 500s {es-pull}112678[#112678] (issue: {es-issue}112177[#112177]) +// * Make empty string searches be consistent with case (in)sensitivity {es-pull}110833[#110833] +// * Prevent flattening of ordered and unordered interval sources {es-pull}114234[#114234] +// * Remove needless forking to GENERIC in `TransportMultiSearchAction` {es-pull}110796[#110796] +// * Search/Mapping: KnnVectorQueryBuilder support for allowUnmappedFields {es-pull}107047[#107047] (issue: {es-issue}106846[#106846]) +// * Span term query to convert to match no docs when unmapped field is targeted {es-pull}113251[#113251] +// * Speedup `CanMatchPreFilterSearchPhase` constructor {es-pull}110860[#110860] +// * Updated Date Range to Follow Documentation When Assuming Missing Values {es-pull}112258[#112258] (issue: {es-issue}111484[#111484]) +// +// Security:: +// * Updated the transport CA name in Security Auto-Configuration. {es-pull}106520[#106520] (issue: {es-issue}106455[#106455]) +// +// Snapshot/Restore:: +// * Retry throttled snapshot deletions {es-pull}113237[#113237] +// +// TSDB:: +// * Implement `parseBytesRef` for `TimeSeriesRoutingHashFieldType` {es-pull}113373[#113373] (issue: {es-issue}112399[#112399]) +// +// Task Management:: +// * Improve handling of failure to create persistent task {es-pull}114386[#114386] +// +// Transform:: +// * Allow task canceling of validate API calls {es-pull}110951[#110951] +// * Include reason when no nodes are found {es-pull}112409[#112409] (issue: {es-issue}112404[#112404]) +// +// Vector Search:: +// * Fix dim validation for bit `element_type` {es-pull}114533[#114533] +// * Support semantic_text in object fields {es-pull}114601[#114601] (issue: {es-issue}114401[#114401]) +// +// Watcher:: +// * Truncating watcher history if it is too large {es-pull}111245[#111245] (issue: {es-issue}94745[#94745]) +// * Watch Next Run Interval Resets On Shard Move or Node Restart {es-pull}115102[#115102] (issue: {es-issue}111433[#111433]) +// +// [[deprecation-9.0.0]] +// [float] +// === Deprecations +// +// Analysis:: +// * Deprecate dutch_kp and lovins stemmer as they are removed in Lucene 10 {es-pull}113143[#113143] +// * deprecate `edge_ngram` side parameter {es-pull}110829[#110829] +// +// CRUD:: +// * Deprecate dot-prefixed indices and composable template index patterns {es-pull}112571[#112571] +// +// Machine Learning:: +// * [Inference API] Deprecate elser service {es-pull}113216[#113216] +// +// Search:: +// * Adding deprecation warnings for rrf using rank and `sub_searches` {es-pull}114854[#114854] +// * Deprecate legacy params from range query {es-pull}113286[#113286] +// +// [[enhancement-9.0.0]] +// [float] +// === Enhancements +// +// Aggregations:: +// * Account for `DelayedBucket` before reduction {es-pull}113013[#113013] +// * Add protection for OOM during aggregations partial reduction {es-pull}110520[#110520] +// * Deduplicate `BucketOrder` when deserializing {es-pull}112707[#112707] +// * Lower the memory footprint when creating `DelayedBucket` {es-pull}112519[#112519] +// * Reduce heap usage for `AggregatorsReducer` {es-pull}112874[#112874] +// * Remove reduce and `reduceContext` from `DelayedBucket` {es-pull}112547[#112547] +// +// Allocation:: +// * Add link to flood-stage watermark exception message {es-pull}111315[#111315] +// * Always allow rebalancing by default {es-pull}111015[#111015] +// * Only publish desired balance gauges on master {es-pull}115383[#115383] +// +// Application:: +// * [Profiling] add `container.id` field to event index template {es-pull}111969[#111969] +// +// Authorization:: +// * Add manage roles privilege {es-pull}110633[#110633] +// * Add privileges required for CDR misconfiguration features to work on AWS SecurityHub integration {es-pull}112574[#112574] +// * [Security Solution] Add `create_index` to `kibana_system` role for index/DS `.logs-endpoint.action.responses-*` {es-pull}115241[#115241] +// +// CRUD:: +// * Suppress merge-on-recovery for older indices {es-pull}113462[#113462] +// +// Codec:: +// * Remove zstd feature flag for index codec best compression {es-pull}112665[#112665] +// +// Data streams:: +// * Add 'verbose' flag retrieving `maximum_timestamp` for get data stream API {es-pull}112303[#112303] +// * Display effective retention in the relevant data stream APIs {es-pull}112019[#112019] +// * Expose global retention settings via data stream lifecycle API {es-pull}112210[#112210] +// * Make ecs@mappings work with OTel attributes {es-pull}111600[#111600] +// +// Distributed:: +// * Add link to Max Shards Per Node exception message {es-pull}110993[#110993] +// * Use Azure blob batch API to delete blobs in batches {es-pull}114566[#114566] +// +// EQL:: +// * ESQL: Delay construction of warnings {es-pull}114368[#114368] +// +// ES|QL:: +// * Add EXP ES|QL function {es-pull}110879[#110879] +// * Add `CircuitBreaker` to TDigest, Step 3: Connect with ESQL CB {es-pull}113387[#113387] +// * Add `CircuitBreaker` to TDigest, Step 4: Take into account shallow classes size {es-pull}113613[#113613] (issue: {es-issue}113916[#113916]) +// * Collect and display execution metadata for ES|QL cross cluster searches {es-pull}112595[#112595] (issue: {es-issue}112402[#112402]) +// * ESQL: Add support for multivalue fields in Arrow output {es-pull}114774[#114774] +// * ESQL: BUCKET: allow numerical spans as whole numbers {es-pull}111874[#111874] (issues: {es-issue}104646[#104646], {es-issue}109340[#109340], {es-issue}105375[#105375]) +// * ESQL: Have BUCKET generate friendlier intervals {es-pull}111879[#111879] (issue: {es-issue}110916[#110916]) +// * ESQL: Profile more timing information {es-pull}111855[#111855] +// * ESQL: Push down filters even in case of renames in Evals {es-pull}114411[#114411] +// * ESQL: Remove parent from `FieldAttribute` {es-pull}112881[#112881] +// * ESQL: Speed up CASE for some parameters {es-pull}112295[#112295] +// * ESQL: Speed up grouping by bytes {es-pull}114021[#114021] +// * ESQL: Support INLINESTATS grouped on expressions {es-pull}111690[#111690] +// * ESQL: Use less memory in listener {es-pull}114358[#114358] +// * ES|QL: Add support for cached strings in plan serialization {es-pull}112929[#112929] +// * ES|QL: add Telemetry API and track top functions {es-pull}111226[#111226] +// * ES|QL: add metrics for functions {es-pull}114620[#114620] +// * Enhance SORT push-down to Lucene to cover references to fields and ST_DISTANCE function {es-pull}112938[#112938] (issue: {es-issue}109973[#109973]) +// * Siem ea 9521 improve test {es-pull}111552[#111552] +// * Support multi-valued fields in compute engine for ST_DISTANCE {es-pull}114836[#114836] (issue: {es-issue}112910[#112910]) +// * [ESQL] Add `SPACE` function {es-pull}112350[#112350] +// * [ESQL] Add finish() elapsed time to aggregation profiling times {es-pull}113172[#113172] (issue: {es-issue}112950[#112950]) +// * [ESQL] Make query wrapped by `SingleValueQuery` cacheable {es-pull}110116[#110116] +// * [ES|QL] Add hypot function {es-pull}114382[#114382] +// * [ES|QL] Cast mixed numeric types to a common numeric type for Coalesce and In at Analyzer {es-pull}111917[#111917] (issue: {es-issue}111486[#111486]) +// * [ES|QL] Combine Disjunctive CIDRMatch {es-pull}111501[#111501] (issue: {es-issue}105143[#105143]) +// * [ES|QL] Create `Range` in `PushFiltersToSource` for qualified pushable filters on the same field {es-pull}111437[#111437] +// * [ES|QL] Name parameter with leading underscore {es-pull}111950[#111950] (issue: {es-issue}111821[#111821]) +// * [ES|QL] Named parameter for field names and field name patterns {es-pull}112905[#112905] +// * [ES|QL] Validate index name in parser {es-pull}112081[#112081] +// * [ES|QL] add reverse function {es-pull}113297[#113297] +// * [ES|QL] explicit cast a string literal to `date_period` and `time_duration` in arithmetic operations {es-pull}109193[#109193] +// +// Experiences:: +// * Integrate IBM watsonx to Inference API for text embeddings {es-pull}111770[#111770] +// +// Geo:: +// * Add support for spatial relationships in point field mapper {es-pull}112126[#112126] +// * Small performance improvement in h3 library {es-pull}113385[#113385] +// * Support docvalues only query in shape field {es-pull}112199[#112199] +// +// Health:: +// * (API) Cluster Health report `unassigned_primary_shards` {es-pull}112024[#112024] +// * Do not treat replica as unassigned if primary recently created and unassigned time is below a threshold {es-pull}112066[#112066] +// * Increase `replica_unassigned_buffer_time` default from 3s to 5s {es-pull}112834[#112834] +// +// ILM+SLM:: +// * ILM: Add `total_shards_per_node` setting to searchable snapshot {es-pull}112972[#112972] (issue: {es-issue}112261[#112261]) +// * PUT slm policy should only increase version if actually changed {es-pull}111079[#111079] +// * Preserve Step Info Across ILM Auto Retries {es-pull}113187[#113187] +// * Register SLM run before snapshotting to save stats {es-pull}110216[#110216] +// * SLM interval schedule followup - add back `getFieldName` style getters {es-pull}112123[#112123] +// +// Infra/Circuit Breakers:: +// * Add link to Circuit Breaker "Data too large" exception message {es-pull}113561[#113561] +// +// Infra/Core:: +// * Add nanos support to `ZonedDateTime` serialization {es-pull}111689[#111689] (issue: {es-issue}68292[#68292]) +// * Extend logging for dropped warning headers {es-pull}111624[#111624] (issue: {es-issue}90527[#90527]) +// * Give the kibana system user permission to read security entities {es-pull}114363[#114363] +// +// Infra/Metrics:: +// * Add `TaskManager` to `pluginServices` {es-pull}112687[#112687] +// * Add `ensureGreen` test method for use with `adminClient` {es-pull}113425[#113425] +// +// Infra/REST API:: +// * Optimize the loop processing of URL decoding {es-pull}110237[#110237] (issue: {es-issue}110235[#110235]) +// +// Infra/Scripting:: +// * Add a `mustache.max_output_size_bytes` setting to limit the length of results from mustache scripts {es-pull}114002[#114002] +// * Expose `HexFormat` in Painless {es-pull}112412[#112412] +// +// Infra/Settings:: +// * Improve exception message for bad environment variable placeholders in settings {es-pull}114552[#114552] (issue: {es-issue}110858[#110858]) +// * Reprocess operator file settings when settings service starts, due to node restart or master node change {es-pull}114295[#114295] +// +// Ingest Node:: +// * Add `size_in_bytes` to enrich cache stats {es-pull}110578[#110578] +// * Add support for templates when validating mappings in the simulate ingest API {es-pull}111161[#111161] +// * Adding `index_template_substitutions` to the simulate ingest API {es-pull}114128[#114128] +// * Adding component template substitutions to the simulate ingest API {es-pull}113276[#113276] +// * Adding mapping validation to the simulate ingest API {es-pull}110606[#110606] +// * Adding support for additional mapping to simulate ingest API {es-pull}114742[#114742] +// * Adding support for simulate ingest mapping adddition for indices with mappings that do not come from templates {es-pull}115359[#115359] +// * Adds example plugin for custom ingest processor {es-pull}112282[#112282] (issue: {es-issue}111539[#111539]) +// * Fix unnecessary mustache template evaluation {es-pull}110986[#110986] (issue: {es-issue}110191[#110191]) +// * Listing all available databases in the _ingest/geoip/database API {es-pull}113498[#113498] +// * Make enrich cache based on memory usage {es-pull}111412[#111412] (issue: {es-issue}106081[#106081]) +// * Tag redacted document in ingest metadata {es-pull}113552[#113552] +// * Verify Maxmind database types in the geoip processor {es-pull}114527[#114527] +// +// Logs:: +// * Add validation for synthetic source mode in logs mode indices {es-pull}110677[#110677] +// * Store original source for keywords using a normalizer {es-pull}112151[#112151] +// +// Machine Learning:: +// * Add Completion Inference API for Alibaba Cloud AI Search Model {es-pull}112512[#112512] +// * Add DeBERTa-V2/V3 tokenizer {es-pull}111852[#111852] +// * Add Streaming Inference spec {es-pull}113812[#113812] +// * Add chunking settings configuration to `CohereService,` `AmazonBedrockService,` and `AzureOpenAiService` {es-pull}113897[#113897] +// * Add chunking settings configuration to `ElasticsearchService/ELSER` {es-pull}114429[#114429] +// * Add custom rule parameters to force time shift {es-pull}110974[#110974] +// * Adding chunking settings to `GoogleVertexAiService,` `AzureAiStudioService,` and `AlibabaCloudSearchService` {es-pull}113981[#113981] +// * Adding chunking settings to `MistralService,` `GoogleAiStudioService,` and `HuggingFaceService` {es-pull}113623[#113623] +// * Adds a new Inference API for streaming responses back to the user. {es-pull}113158[#113158] +// * Create `StreamingHttpResultPublisher` {es-pull}112026[#112026] +// * Create an ml node inference endpoint referencing an existing model {es-pull}114750[#114750] +// * Default inference endpoint for ELSER {es-pull}113873[#113873] +// * Default inference endpoint for the multilingual-e5-small model {es-pull}114683[#114683] +// * Enable OpenAI Streaming {es-pull}113911[#113911] +// * Filter empty task settings objects from the API response {es-pull}114389[#114389] +// * Increase default `queue_capacity` to 10_000 and decrease max `queue_capacity` to 100_000 {es-pull}115041[#115041] +// * Migrate Inference to `ChunkedToXContent` {es-pull}111655[#111655] +// * Register Task while Streaming {es-pull}112369[#112369] +// * Server-Sent Events for Inference response {es-pull}112565[#112565] +// * Stream Anthropic Completion {es-pull}114321[#114321] +// * Stream Azure Completion {es-pull}114464[#114464] +// * Stream Bedrock Completion {es-pull}114732[#114732] +// * Stream Cohere Completion {es-pull}114080[#114080] +// * Stream Google Completion {es-pull}114596[#114596] +// * Stream OpenAI Completion {es-pull}112677[#112677] +// * Support sparse embedding models in the elasticsearch inference service {es-pull}112270[#112270] +// * Switch default chunking strategy to sentence {es-pull}114453[#114453] +// * Upgrade to AWS SDK v2 {es-pull}114309[#114309] (issue: {es-issue}110590[#110590]) +// * Use the same chunking configurations for models in the Elasticsearch service {es-pull}111336[#111336] +// * Validate streaming HTTP Response {es-pull}112481[#112481] +// * Wait for allocation on scale up {es-pull}114719[#114719] +// * [Inference API] Add Alibaba Cloud AI Search Model support to Inference API {es-pull}111181[#111181] +// * [Inference API] Add Docs for AlibabaCloud AI Search Support for the Inference API {es-pull}111181[#111181] +// * [Inference API] Introduce Update API to change some aspects of existing inference endpoints {es-pull}114457[#114457] +// * [Inference API] Prevent inference endpoints from being deleted if they are referenced by semantic text {es-pull}110399[#110399] +// * [Inference API] alibabacloud ai search service support chunk infer to support semantic_text field {es-pull}110399[#110399] +// +// Mapping:: +// * Add Field caps support for Semantic Text {es-pull}111809[#111809] +// * Add Lucene segment-level fields stats {es-pull}111123[#111123] +// * Add Search Inference ID To Semantic Text Mapping {es-pull}113051[#113051] +// * Add object param for keeping synthetic source {es-pull}113690[#113690] +// * Add support for multi-value dimensions {es-pull}112645[#112645] (issue: {es-issue}110387[#110387]) +// * Allow dimension fields to have multiple values in standard and logsdb index mode {es-pull}112345[#112345] (issues: {es-issue}112232[#112232], {es-issue}112239[#112239]) +// * Allow fields with dots in sparse vector field mapper {es-pull}111981[#111981] (issue: {es-issue}109118[#109118]) +// * Allow querying `index_mode` {es-pull}110676[#110676] +// * Configure keeping source in `FieldMapper` {es-pull}112706[#112706] +// * Control storing array source with index setting {es-pull}112397[#112397] +// * Introduce mode `subobjects=auto` for objects {es-pull}110524[#110524] +// * Update `semantic_text` field to support indexing numeric and boolean data types {es-pull}111284[#111284] +// * Use ELSER By Default For Semantic Text {es-pull}113563[#113563] +// * Use fallback synthetic source for `copy_to` and doc_values: false cases {es-pull}112294[#112294] (issues: {es-issue}110753[#110753], {es-issue}110038[#110038], {es-issue}109546[#109546]) +// +// Network:: +// * Add links to network disconnect troubleshooting {es-pull}112330[#112330] +// +// Ranking:: +// * Add timeout and cancellation check to rescore phase {es-pull}115048[#115048] +// +// Recovery:: +// * Trigger merges after recovery {es-pull}113102[#113102] +// +// Relevance:: +// * Add a query rules tester API call {es-pull}114168[#114168] +// +// Search:: +// * Add initial support for `semantic_text` field type {es-pull}113920[#113920] +// * Add more `dense_vector` details for cluster stats field stats {es-pull}113607[#113607] +// * Add range and regexp Intervals {es-pull}111465[#111465] +// * Adding support for `allow_partial_search_results` in PIT {es-pull}111516[#111516] +// * Allow incubating Panama Vector in simdvec, and add vectorized `ipByteBin` {es-pull}112933[#112933] +// * Avoid using concurrent collector manager in `LuceneChangesSnapshot` {es-pull}113816[#113816] +// * Bool query early termination should also consider `must_not` clauses {es-pull}115031[#115031] +// * Deduplicate Kuromoji User Dictionary {es-pull}112768[#112768] +// * Multi term intervals: increase max_expansions {es-pull}112826[#112826] (issue: {es-issue}110491[#110491]) +// * Search coordinator uses `event.ingested` in cluster state to do rewrites {es-pull}111523[#111523] +// * Update cluster stats for retrievers {es-pull}114109[#114109] +// +// Security:: +// * (logger) change from error to warn for short circuiting user {es-pull}112895[#112895] +// * Add asset criticality indices for `kibana_system_user` {es-pull}113588[#113588] +// * Add tier preference to security index settings allowlist {es-pull}111818[#111818] +// * [Service Account] Add `AutoOps` account {es-pull}111316[#111316] +// +// Snapshot/Restore:: +// * Add `max_multipart_parts` setting to S3 repository {es-pull}113989[#113989] +// * Add support for Azure Managed Identity {es-pull}111344[#111344] +// * Add telemetry for repository usage {es-pull}112133[#112133] +// * Add workaround for missing shard gen blob {es-pull}112337[#112337] +// * Clean up dangling S3 multipart uploads {es-pull}111955[#111955] (issues: {es-issue}101169[#101169], {es-issue}44971[#44971]) +// * Execute shard snapshot tasks in shard-id order {es-pull}111576[#111576] (issue: {es-issue}108739[#108739]) +// * Include account name in Azure settings exceptions {es-pull}111274[#111274] +// * Introduce repository integrity verification API {es-pull}112348[#112348] (issue: {es-issue}52622[#52622]) +// * Retry `S3BlobContainer#getRegister` on all exceptions {es-pull}114813[#114813] +// * Track shard snapshot progress during node shutdown {es-pull}112567[#112567] +// +// Stats:: +// * Track search and fetch failure stats {es-pull}113988[#113988] +// +// TSDB:: +// * Add support for boolean dimensions {es-pull}111457[#111457] (issue: {es-issue}111338[#111338]) +// * Stop iterating over all fields to extract @timestamp value {es-pull}110603[#110603] (issue: {es-issue}92297[#92297]) +// * Support booleans in routing path {es-pull}111445[#111445] +// +// Vector Search:: +// * Dense vector field types updatable for int4 {es-pull}110928[#110928] +// * Use native scalar scorer for int8_flat index {es-pull}111071[#111071] +// +// [[feature-9.0.0]] +// [float] +// === New features +// +// Data streams:: +// * Introduce global retention in data stream lifecycle. {es-pull}111972[#111972] +// * X-pack/plugin/otel: introduce x-pack-otel plugin {es-pull}111091[#111091] +// +// ES|QL:: +// * Add ESQL match function {es-pull}113374[#113374] +// * ESQL: Add `MV_PSERIES_WEIGHTED_SUM` for score calculations used by security solution {es-pull}109017[#109017] +// * ESQL: Add async ID and `is_running` headers to ESQL async query {es-pull}111840[#111840] +// * ESQL: Add boolean support to Max and Min aggs {es-pull}110527[#110527] +// * ESQL: Add boolean support to TOP aggregation {es-pull}110718[#110718] +// * ESQL: Added `mv_percentile` function {es-pull}111749[#111749] (issue: {es-issue}111591[#111591]) +// * ESQL: INLINESTATS {es-pull}109583[#109583] (issue: {es-issue}107589[#107589]) +// * ESQL: Introduce per agg filter {es-pull}113735[#113735] +// * ESQL: Strings support for MAX and MIN aggregations {es-pull}111544[#111544] +// * ESQL: Support IP fields in MAX and MIN aggregations {es-pull}110921[#110921] +// * ESQL: TOP aggregation IP support {es-pull}111105[#111105] +// * ESQL: TOP support for strings {es-pull}113183[#113183] (issue: {es-issue}109849[#109849]) +// * ESQL: `mv_median_absolute_deviation` function {es-pull}112055[#112055] (issue: {es-issue}111590[#111590]) +// * Remove snapshot build restriction for match and qstr functions {es-pull}114482[#114482] +// * Search in ES|QL: Add MATCH operator {es-pull}110971[#110971] +// +// ILM+SLM:: +// * SLM Interval based scheduling {es-pull}110847[#110847] +// +// Inference:: +// * EIS integration {es-pull}111154[#111154] +// +// Ingest Node:: +// * Add a `terminate` ingest processor {es-pull}114157[#114157] (issue: {es-issue}110218[#110218]) +// +// Machine Learning:: +// * Inference autoscaling {es-pull}109667[#109667] +// * Telemetry for inference adaptive allocations {es-pull}110630[#110630] +// +// Relevance:: +// * [Query rules] Add `exclude` query rule type {es-pull}111420[#111420] +// +// Search:: +// * Async search: Add ID and "is running" http headers {es-pull}112431[#112431] (issue: {es-issue}109576[#109576]) +// * Cross-cluster search telemetry {es-pull}113825[#113825] +// +// Vector Search:: +// * Adding new bbq index types behind a feature flag {es-pull}114439[#114439] + +[[upgrade-9.0.0]] +[float] +=== Upgrades +// +// Infra/Core:: +// * Upgrade xcontent to Jackson 2.17.0 {es-pull}111948[#111948] +// * Upgrade xcontent to Jackson 2.17.2 {es-pull}112320[#112320] +// +// Infra/Metrics:: +// * Update APM Java Agent to support JDK 23 {es-pull}115194[#115194] (issues: {es-issue}115101[#115101], {es-issue}115100[#115100]) +// +// Search:: +// * Upgrade to Lucene 10 {es-pull}114741[#114741] +// * Upgrade to Lucene 9.12 {es-pull}113333[#113333] +// +// Snapshot/Restore:: +// * Upgrade Azure SDK {es-pull}111225[#111225] +// * Upgrade `repository-azure` dependencies {es-pull}112277[#112277] + + diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index c3f6fb43f2ffd..edecd4f727583 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -1,6 +1,8 @@ -[chapter] +// THIS IS A GENERATED FILE. DO NOT EDIT DIRECTLY. +// The content generated here are is not correct and most has been manually commented out until it can be fixed. +// See ES-9931 for more details. [[release-highlights]] -= What's new in {minor-version} +== What's new in {minor-version} coming::[{minor-version}] @@ -9,95 +11,164 @@ ifeval::["{release-state}"!="unreleased"] For detailed information about this release, see the <> and <>. -// Add previous release to the list -Other versions: - -{ref-bare}/8.16/release-highlights.html[8.16] -| {ref-bare}/8.15/release-highlights.html[8.15] -| {ref-bare}/8.14/release-highlights.html[8.14] -| {ref-bare}/8.13/release-highlights.html[8.13] -| {ref-bare}/8.12/release-highlights.html[8.12] -| {ref-bare}/8.11/release-highlights.html[8.11] -| {ref-bare}/8.10/release-highlights.html[8.10] -| {ref-bare}/8.9/release-highlights.html[8.9] -| {ref-bare}/8.8/release-highlights.html[8.8] -| {ref-bare}/8.7/release-highlights.html[8.7] -| {ref-bare}/8.6/release-highlights.html[8.6] -| {ref-bare}/8.5/release-highlights.html[8.5] -| {ref-bare}/8.4/release-highlights.html[8.4] -| {ref-bare}/8.3/release-highlights.html[8.3] -| {ref-bare}/8.2/release-highlights.html[8.2] -| {ref-bare}/8.1/release-highlights.html[8.1] -| {ref-bare}/8.0/release-highlights.html[8.0] - endif::[] - -// The notable-highlights tag marks entries that -// should be featured in the Stack Installation and Upgrade Guide: -// tag::notable-highlights[] - -[discrete] -[[esql_inlinestats]] -== ESQL: INLINESTATS -This adds the `INLINESTATS` command to ESQL which performs a STATS and -then enriches the results into the output stream. So, this query: - -[source,esql] ----- -FROM test -| INLINESTATS m=MAX(a * b) BY b -| WHERE m == a * b -| SORT a DESC, b DESC -| LIMIT 3 ----- - -Produces output like: - -| a | b | m | -| --- | --- | ----- | -| 99 | 999 | 98901 | -| 99 | 998 | 98802 | -| 99 | 997 | 98703 | - -{es-pull}109583[#109583] - -[discrete] -[[always_allow_rebalancing_by_default]] -== Always allow rebalancing by default -In earlier versions of {es} the `cluster.routing.allocation.allow_rebalance` setting defaults to -`indices_all_active` which blocks all rebalancing moves while the cluster is in `yellow` or `red` health. This was -appropriate for the legacy allocator which might do too many rebalancing moves otherwise. Today's allocator has -better support for rebalancing a cluster that is not in `green` health, and expects to be able to rebalance some -shards away from over-full nodes to avoid allocating shards to undesirable locations in the first place. From -version 8.16 `allow_rebalance` setting defaults to `always` unless the legacy allocator is explicitly enabled. - -{es-pull}111015[#111015] - -[discrete] -[[add_global_retention_in_data_stream_lifecycle]] -== Add global retention in data stream lifecycle -Data stream lifecycle now supports configuring retention on a cluster level, -namely global retention. Global retention \nallows us to configure two different -retentions: - -- `data_streams.lifecycle.retention.default` is applied to all data streams managed -by the data stream lifecycle that do not have retention defined on the data stream level. -- `data_streams.lifecycle.retention.max` is applied to all data streams managed by the -data stream lifecycle and it allows any data stream \ndata to be deleted after the `max_retention` has passed. - -{es-pull}111972[#111972] - -[discrete] -[[enable_zstandard_compression_for_indices_with_index_codec_set_to_best_compression]] -== Enable ZStandard compression for indices with index.codec set to best_compression -Before DEFLATE compression was used to compress stored fields in indices with index.codec index setting set to -best_compression, with this change ZStandard is used as compression algorithm to stored fields for indices with -index.codec index setting set to best_compression. The usage ZStandard results in less storage usage with a -similar indexing throughput depending on what options are used. Experiments with indexing logs have shown that -ZStandard offers ~12% lower storage usage and a ~14% higher indexing throughput compared to DEFLATE. - -{es-pull}112665[#112665] - -// end::notable-highlights[] - +// +// // tag::notable-highlights[] +// +// [discrete] +// [[esql_inlinestats]] +// === ESQL: INLINESTATS +// This adds the `INLINESTATS` command to ESQL which performs a STATS and +// then enriches the results into the output stream. So, this query: +// +// [source,esql] +// ---- +// FROM test +// | INLINESTATS m=MAX(a * b) BY b +// | WHERE m == a * b +// | SORT a DESC, b DESC +// | LIMIT 3 +// ---- +// +// Produces output like: +// +// | a | b | m | +// | --- | --- | ----- | +// | 99 | 999 | 98901 | +// | 99 | 998 | 98802 | +// | 99 | 997 | 98703 | +// +// {es-pull}109583[#109583] +// +// [discrete] +// [[always_allow_rebalancing_by_default]] +// === Always allow rebalancing by default +// In earlier versions of {es} the `cluster.routing.allocation.allow_rebalance` setting defaults to +// `indices_all_active` which blocks all rebalancing moves while the cluster is in `yellow` or `red` health. This was +// appropriate for the legacy allocator which might do too many rebalancing moves otherwise. Today's allocator has +// better support for rebalancing a cluster that is not in `green` health, and expects to be able to rebalance some +// shards away from over-full nodes to avoid allocating shards to undesirable locations in the first place. From +// version 8.16 `allow_rebalance` setting defaults to `always` unless the legacy allocator is explicitly enabled. +// +// {es-pull}111015[#111015] +// +// [discrete] +// [[add_global_retention_in_data_stream_lifecycle]] +// === Add global retention in data stream lifecycle +// Data stream lifecycle now supports configuring retention on a cluster level, +// namely global retention. Global retention \nallows us to configure two different +// retentions: +// +// - `data_streams.lifecycle.retention.default` is applied to all data streams managed +// by the data stream lifecycle that do not have retention defined on the data stream level. +// - `data_streams.lifecycle.retention.max` is applied to all data streams managed by the +// data stream lifecycle and it allows any data stream \ndata to be deleted after the `max_retention` has passed. +// +// {es-pull}111972[#111972] +// +// [discrete] +// [[enable_zstandard_compression_for_indices_with_index_codec_set_to_best_compression]] +// === Enable ZStandard compression for indices with index.codec set to best_compression +// Before DEFLATE compression was used to compress stored fields in indices with index.codec index setting set to +// best_compression, with this change ZStandard is used as compression algorithm to stored fields for indices with +// index.codec index setting set to best_compression. The usage ZStandard results in less storage usage with a +// similar indexing throughput depending on what options are used. Experiments with indexing logs have shown that +// ZStandard offers ~12% lower storage usage and a ~14% higher indexing throughput compared to DEFLATE. +// +// {es-pull}112665[#112665] +// +// [discrete] +// [[esql_introduce_per_agg_filter]] +// === ESQL: Introduce per agg filter +// Add support for aggregation scoped filters that work dynamically on the +// data in each group. +// +// [source,esql] +// ---- +// | STATS success = COUNT(*) WHERE 200 <= code AND code < 300, +// redirect = COUNT(*) WHERE 300 <= code AND code < 400, +// client_err = COUNT(*) WHERE 400 <= code AND code < 500, +// server_err = COUNT(*) WHERE 500 <= code AND code < 600, +// total_count = COUNT(*) +// ---- +// +// Implementation wise, the base AggregateFunction has been extended to +// allow a filter to be passed on. This is required to incorporate the +// filter as part of the aggregate equality/identity which would fail with +// the filter as an external component. +// As part of the process, the serialization for the existing aggregations +// had to be fixed so AggregateFunction implementations so that it +// delegates to their parent first. +// +// {es-pull}113735[#113735] +// +// // end::notable-highlights[] +// +// +// [discrete] +// [[esql_multi_value_fields_supported_in_geospatial_predicates]] +// === ESQL: Multi-value fields supported in Geospatial predicates +// Supporting multi-value fields in `WHERE` predicates is a challenge due to not knowing whether `ALL` or `ANY` +// of the values in the field should pass the predicate. +// For example, should the field `age:[10,30]` pass the predicate `WHERE age>20` or not? +// This ambiguity does not exist with the spatial predicates +// `ST_INTERSECTS` and `ST_DISJOINT`, because the choice between `ANY` or `ALL` +// is implied by the predicate itself. +// Consider a predicate checking a field named `location` against a test geometry named `shape`: +// +// * `ST_INTERSECTS(field, shape)` - true if `ANY` value can intersect the shape +// * `ST_DISJOINT(field, shape)` - true only if `ALL` values are disjoint from the shape +// +// This works even if the shape argument is itself a complex or compound geometry. +// +// Similar logic exists for `ST_CONTAINS` and `ST_WITHIN` predicates, but these are not as easily solved +// with `ANY` or `ALL`, because a collection of geometries contains another collection if each of the contained +// geometries is within at least one of the containing geometries. Evaluating this requires that the multi-value +// field is first combined into a single geometry before performing the predicate check. +// +// * `ST_CONTAINS(field, shape)` - true if the combined geometry contains the shape +// * `ST_WITHIN(field, shape)` - true if the combined geometry is within the shape +// +// {es-pull}112063[#112063] +// +// [discrete] +// [[enhance_sort_push_down_to_lucene_to_cover_references_to_fields_st_distance_function]] +// === Enhance SORT push-down to Lucene to cover references to fields and ST_DISTANCE function +// The most used and likely most valuable geospatial search query in Elasticsearch is the sorted proximity search, +// finding items within a certain distance of a point of interest and sorting the results by distance. +// This has been possible in ES|QL since 8.15.0, but the sorting was done in-memory, not pushed down to Lucene. +// Now the sorting is pushed down to Lucene, which results in a significant performance improvement. +// +// Queries that perform both filtering and sorting on distance are supported. For example: +// +// [source,esql] +// ---- +// FROM test +// | EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(37.7749, -122.4194)")) +// | WHERE distance < 1000000 +// | SORT distance ASC, name DESC +// | LIMIT 10 +// ---- +// +// In addition, the support for sorting on EVAL expressions has been extended to cover references to fields: +// +// [source,esql] +// ---- +// FROM test +// | EVAL ref = field +// | SORT ref ASC +// | LIMIT 10 +// ---- +// +// {es-pull}112938[#112938] +// +// [discrete] +// [[cross_cluster_search_telemetry]] +// === Cross-cluster search telemetry +// The cross-cluster search telemetry is collected when cross-cluster searches +// are performed, and is returned as "ccs" field in `_cluster/stats` output. +// It also add a new parameter `include_remotes=true` to the `_cluster/stats` API +// which will collect data from connected remote clusters. +// +// {es-pull}113825[#113825] diff --git a/docs/reference/reranking/learning-to-rank-model-training.asciidoc b/docs/reference/reranking/learning-to-rank-model-training.asciidoc index 8e0b3f9ae94ce..9a35573a0879d 100644 --- a/docs/reference/reranking/learning-to-rank-model-training.asciidoc +++ b/docs/reference/reranking/learning-to-rank-model-training.asciidoc @@ -43,7 +43,7 @@ feature_extractors=[ feature_name="title_bm25", query={"match": {"title": "{{query}}"}} ), - # We want to use the the number of matched terms in the title field as a feature: + # We want to use the number of matched terms in the title field as a feature: QueryFeatureExtractor( feature_name="title_matched_term_count", query={ diff --git a/docs/reference/rest-api/rest-api-compatibility.asciidoc b/docs/reference/rest-api/rest-api-compatibility.asciidoc index 14193fa9371bd..dbc140d1f9236 100644 --- a/docs/reference/rest-api/rest-api-compatibility.asciidoc +++ b/docs/reference/rest-api/rest-api-compatibility.asciidoc @@ -6,11 +6,6 @@ API changes, {es} provides a per-request, opt-in API compatibility mode. {es} REST APIs are generally stable across versions. However, some improvements require changes that are not compatible with previous versions. -For example, {es} 7.x supported custom mapping types in many URL paths, -but {es} 8.0+ does not (see <>). Specifying a custom type -in a request sent to {es} 8.0+ returns an error. However, if you request -REST API compatibility, {es} accepts the request even though mapping types -are no longer supported. When an API is targeted for removal or is going to be changed in a non-compatible way, the original API is deprecated for one or more releases. @@ -32,20 +27,18 @@ IMPORTANT: REST API compatibility does not guarantee the same behavior as the prior version. It instructs {es} to automatically resolve any incompatibilities so the request can be processed instead of returning an error. - REST API compatibility should be a bridge to smooth out the upgrade process, not a long term strategy. REST API compatibility is only honored across one -major version: honor 7.x requests/responses from 8.x. +major version: honor 8.x requests/responses from 9.x. When you submit requests using REST API compatibility and {es} resolves the incompatibility, a message is written to the deprecation log with the category "compatible_api". Review the deprecation log to identify any gaps in usage and fully supported features. - -For information about specific breaking changes and the impact of requesting -compatibility mode, see <> -in the migration guide. +//TODO: add this back once this is fixed: ES-9932 +// For information about specific breaking changes and the impact of requesting +// compatibility mode. See <> in the migration guide. [discrete] [[request-rest-api-compatibility]] @@ -58,19 +51,19 @@ For example: [source, text] ------------------------------------------------------------ -Accept: "application/vnd.elasticsearch+json;compatible-with=7" -Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" +Accept: "application/vnd.elasticsearch+json;compatible-with=8" +Content-Type: "application/vnd.elasticsearch+json;compatible-with=8" ------------------------------------------------------------ The Accept header is always required and the Content-Type header is only required when a body is sent with the request. The following values are -valid when communicating with a 7.x or 8.x {es} server: +valid when communicating with a 8.x or 9.x {es} server: [source, text] ------------------------------------------------------------ -"application/vnd.elasticsearch+json;compatible-with=7" -"application/vnd.elasticsearch+yaml;compatible-with=7" -"application/vnd.elasticsearch+smile;compatible-with=7" -"application/vnd.elasticsearch+cbor;compatible-with=7" +"application/vnd.elasticsearch+json;compatible-with=8" +"application/vnd.elasticsearch+yaml;compatible-with=8" +"application/vnd.elasticsearch+smile;compatible-with=8" +"application/vnd.elasticsearch+cbor;compatible-with=8" ------------------------------------------------------------ The https://www.elastic.co/guide/en/elasticsearch/client/index.html[officially supported {es} clients] can enable REST API compatibility for all requests. @@ -81,10 +74,10 @@ by {es} set the environment variable `ELASTIC_CLIENT_APIVERSIONING` to true. [discrete] === REST API compatibility workflow -To leverage REST API compatibility during an upgrade from 7.17 to {version}: +To leverage REST API compatibility during an upgrade from the last 8.x to {version}: 1. Upgrade your https://www.elastic.co/guide/en/elasticsearch/client/index.html[{es} clients] -to the latest 7.x version and enable REST API compatibility. +to the latest 8.x version and enable REST API compatibility. 2. Use the {kibana-ref-all}/{prev-major-last}/upgrade-assistant.html[Upgrade Assistant] to review all critical issues and explore the deprecation logs. Some critical issues might be mitigated by REST API compatibility. @@ -92,5 +85,6 @@ Some critical issues might be mitigated by REST API compatibility. 4. Upgrade Elasticsearch to {version}. 5. Review the deprecation logs for entries with the category `compatible_api`. Review the workflow associated with the requests that relied on compatibility mode. -6. Upgrade your {es} clients to 8.x and resolve compatibility issues manually where needed. +6. Upgrade your {es} clients to 9.x and resolve compatibility issues manually where needed. + diff --git a/docs/reference/search-application/apis/put-search-application.asciidoc b/docs/reference/search-application/apis/put-search-application.asciidoc index eb559acc8cdc7..dc5e20ec40b7f 100644 --- a/docs/reference/search-application/apis/put-search-application.asciidoc +++ b/docs/reference/search-application/apis/put-search-application.asciidoc @@ -192,7 +192,7 @@ When the above `dictionary` parameter is specified, the <> API will return an error. +If the parameters are not valid, the <> API will return an error. [source,console] ---- POST _application/search_application/my-app/_search diff --git a/docs/reference/security/securing-communications/security-minimal-setup.asciidoc b/docs/reference/security/securing-communications/security-minimal-setup.asciidoc index ee158294df03c..fd54c37d9e8fa 100644 --- a/docs/reference/security/securing-communications/security-minimal-setup.asciidoc +++ b/docs/reference/security/securing-communications/security-minimal-setup.asciidoc @@ -78,7 +78,7 @@ This command resets the password to an auto-generated value. ./bin/elasticsearch-reset-password -u elastic ---- + -If you want to set the password to a specific value, run the command with the +If you want to set the password to a specific value, run the command with the interactive (`-i`) parameter. + [source,shell] @@ -93,7 +93,7 @@ interactive (`-i`) parameter. ./bin/elasticsearch-reset-password -u kibana_system ---- -. Save the new passwords. In the next step, you'll add the the password for the +. Save the new passwords. In the next step, you'll add the password for the `kibana_system` user to {kib}. *Next*: <> diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/AbstractXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/AbstractXContentParser.java index a3b495b9c3e38..909cf808d1f34 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/AbstractXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/AbstractXContentParser.java @@ -370,7 +370,7 @@ private static void skipToListStart(XContentParser parser) throws IOException { } } - // read a list without bounds checks, assuming the the current parser is always on an array start + // read a list without bounds checks, assuming the current parser is always on an array start private static List readListUnsafe(XContentParser parser, Supplier> mapFactory) throws IOException { assert parser.currentToken() == Token.START_ARRAY; ArrayList list = new ArrayList<>(); diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/top_hits.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/top_hits.yml index ed24e1cc8404c..91e9b04e35860 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/top_hits.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/top_hits.yml @@ -356,9 +356,10 @@ synthetic _source: indices.create: index: test_synthetic body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: page: type: keyword diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml index ee6803d809087..4e31c1ba36601 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -180,9 +180,9 @@ setup: body: settings: number_of_shards: 1 + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: boolean: type: boolean @@ -5630,9 +5630,9 @@ version and sequence number synthetic _source: body: settings: number_of_shards: 1 + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: keyword: type: keyword diff --git a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/match_only_text/10_basic.yml b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/match_only_text/10_basic.yml index b4ee226f72692..4a6810faab215 100644 --- a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/match_only_text/10_basic.yml +++ b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/match_only_text/10_basic.yml @@ -285,9 +285,10 @@ synthetic_source: indices.create: index: synthetic_source_test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: foo: type: match_only_text @@ -362,9 +363,10 @@ synthetic_source with copy_to: indices.create: index: synthetic_source_test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: foo: type: match_only_text diff --git a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_feature/30_synthetic_source.yml b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_feature/30_synthetic_source.yml index 1e0b90ebb9e0f..e6df05189b6f9 100644 --- a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_feature/30_synthetic_source.yml +++ b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_feature/30_synthetic_source.yml @@ -7,9 +7,10 @@ setup: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: pagerank: type: rank_feature diff --git a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_features/20_synthetic_source.yml b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_features/20_synthetic_source.yml index c64e35cc2cea4..6ffe896408079 100644 --- a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_features/20_synthetic_source.yml +++ b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_features/20_synthetic_source.yml @@ -7,9 +7,10 @@ setup: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: tags: type: rank_features diff --git a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/search-as-you-type/30_synthetic_source.yml b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/search-as-you-type/30_synthetic_source.yml index 75397bd9e0fe9..b764e14c8ced1 100644 --- a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/search-as-you-type/30_synthetic_source.yml +++ b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/search-as-you-type/30_synthetic_source.yml @@ -7,9 +7,10 @@ setup: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: a_field: type: search_as_you_type diff --git a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/token_count/10_basic.yml b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/token_count/10_basic.yml index 03b72a2623497..a870cd4d01abc 100644 --- a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/token_count/10_basic.yml +++ b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/token_count/10_basic.yml @@ -42,9 +42,10 @@ indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: count: type: token_count diff --git a/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source.yml b/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source.yml index 12d0f1bbae6c7..5eb5739222905 100644 --- a/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source.yml +++ b/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source.yml @@ -7,9 +7,10 @@ supported: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: join_field: type: join diff --git a/modules/percolator/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml b/modules/percolator/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml index a5576d203314f..93ecaaa66bb91 100644 --- a/modules/percolator/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml +++ b/modules/percolator/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml @@ -137,9 +137,10 @@ indices.create: index: queries_index body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: query: type: percolator diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/110_synthetic_source.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/110_synthetic_source.yml index 9ae2153f89ca5..db78af6719c1f 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/110_synthetic_source.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/110_synthetic_source.yml @@ -3,9 +3,10 @@ setup: indices.create: index: synthetic body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/100_synthetic_source.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/100_synthetic_source.yml index 4329bf8ed471a..a17f29a2d52e7 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/100_synthetic_source.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/100_synthetic_source.yml @@ -3,9 +3,10 @@ update: indices.create: index: synthetic body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index 36eb1d61e21d7..1ebd6f920d518 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -335,7 +335,7 @@ public void refresh() { * Customizes {@link com.amazonaws.auth.WebIdentityTokenCredentialsProvider} * *
    - *
  • Reads the the location of the web identity token not from AWS_WEB_IDENTITY_TOKEN_FILE, but from a symlink + *
  • Reads the location of the web identity token not from AWS_WEB_IDENTITY_TOKEN_FILE, but from a symlink * in the plugin directory, so we don't need to create a hardcoded read file permission for the plugin.
  • *
  • Supports customization of the STS endpoint via a system property, so we can test it against a test fixture.
  • *
  • Supports gracefully shutting down the provider and the STS client.
  • diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/270_synthetic_source.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/270_synthetic_source.yml index 8832b3230910c..0a3e372d2d8e4 100644 --- a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/270_synthetic_source.yml +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/270_synthetic_source.yml @@ -8,9 +8,10 @@ keywords: indices.create: index: index1 body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: field1: type: keyword @@ -76,9 +77,10 @@ doubles: indices.create: index: index1 body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: field1: type: double diff --git a/muted-tests.yml b/muted-tests.yml index c781a7d30a597..d04fc426a7a6b 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -20,8 +20,6 @@ tests: - class: org.elasticsearch.nativeaccess.VectorSystemPropertyTests method: testSystemPropertyDisabled issue: https://github.com/elastic/elasticsearch/issues/110949 -- class: org.elasticsearch.multi_node.GlobalCheckpointSyncActionIT - issue: https://github.com/elastic/elasticsearch/issues/111124 - class: org.elasticsearch.packaging.test.DockerTests method: test021InstallPlugin issue: https://github.com/elastic/elasticsearch/issues/110343 @@ -100,8 +98,6 @@ tests: - class: org.elasticsearch.xpack.sql.qa.single_node.JdbcSqlSpecIT method: test {case-functions.testUcaseInline3} issue: https://github.com/elastic/elasticsearch/issues/112643 -- class: org.elasticsearch.repositories.blobstore.testkit.analyze.HdfsRepositoryAnalysisRestIT - issue: https://github.com/elastic/elasticsearch/issues/112889 - class: org.elasticsearch.xpack.sql.qa.security.JdbcSqlSpecIT method: test {case-functions.testUcaseInline1} issue: https://github.com/elastic/elasticsearch/issues/112641 @@ -210,9 +206,6 @@ tests: - class: org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityWithApmTracingRestIT method: testTracingCrossCluster issue: https://github.com/elastic/elasticsearch/issues/112731 -- class: org.elasticsearch.packaging.test.EnrollmentProcessTests - method: test20DockerAutoFormCluster - issue: https://github.com/elastic/elasticsearch/issues/114885 - class: org.elasticsearch.xpack.inference.DefaultEndPointsIT method: testInferDeploysDefaultElser issue: https://github.com/elastic/elasticsearch/issues/114913 @@ -279,14 +272,27 @@ tests: - class: org.elasticsearch.action.search.PointInTimeIT method: testPITTiebreak issue: https://github.com/elastic/elasticsearch/issues/115810 -- class: org.elasticsearch.xpack.searchablesnapshots.hdfs.SecureHdfsSearchableSnapshotsIT - issue: https://github.com/elastic/elasticsearch/issues/115995 - class: org.elasticsearch.index.reindex.ReindexNodeShutdownIT method: testReindexWithShutdown issue: https://github.com/elastic/elasticsearch/issues/115996 - class: org.elasticsearch.search.query.SearchQueryIT method: testAllDocsQueryString issue: https://github.com/elastic/elasticsearch/issues/115728 +- class: org.elasticsearch.search.basic.SearchWithRandomExceptionsIT + method: testRandomExceptions + issue: https://github.com/elastic/elasticsearch/issues/116027 +- class: org.elasticsearch.action.admin.HotThreadsIT + method: testHotThreadsDontFail + issue: https://github.com/elastic/elasticsearch/issues/115754 +- class: org.elasticsearch.search.functionscore.QueryRescorerIT + method: testScoring + issue: https://github.com/elastic/elasticsearch/issues/116050 +- class: org.elasticsearch.indexing.IndexActionIT + method: testAutoGenerateIdNoDuplicates + issue: https://github.com/elastic/elasticsearch/issues/115716 +- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT + method: test {yaml=logsdb/10_settings/logsdb with default ignore dynamic beyond limit and subobjects false} + issue: https://github.com/elastic/elasticsearch/issues/116054 # Examples: # diff --git a/plugins/mapper-annotated-text/src/yamlRestTest/resources/rest-api-spec/test/mapper_annotatedtext/20_synthetic_source.yml b/plugins/mapper-annotated-text/src/yamlRestTest/resources/rest-api-spec/test/mapper_annotatedtext/20_synthetic_source.yml index 4aac881700e15..1d5f1daad9958 100644 --- a/plugins/mapper-annotated-text/src/yamlRestTest/resources/rest-api-spec/test/mapper_annotatedtext/20_synthetic_source.yml +++ b/plugins/mapper-annotated-text/src/yamlRestTest/resources/rest-api-spec/test/mapper_annotatedtext/20_synthetic_source.yml @@ -10,9 +10,10 @@ stored annotated_text field: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: annotated_text: type: annotated_text @@ -40,9 +41,10 @@ annotated_text field with keyword multi-field: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: annotated_text: type: annotated_text @@ -72,9 +74,10 @@ multiple values in stored annotated_text field: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: annotated_text: type: annotated_text @@ -102,9 +105,10 @@ multiple values in annotated_text field with keyword multi-field: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: annotated_text: type: annotated_text @@ -135,9 +139,10 @@ multiple values in annotated_text field with stored keyword multi-field: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: annotated_text: type: annotated_text @@ -169,9 +174,10 @@ multiple values in stored annotated_text field with keyword multi-field: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: annotated_text: type: annotated_text @@ -202,9 +208,10 @@ fallback synthetic source: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: annotated_text: type: annotated_text diff --git a/plugins/mapper-murmur3/src/yamlRestTest/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml b/plugins/mapper-murmur3/src/yamlRestTest/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml index 12b23fb3b0395..6873145d39d5c 100644 --- a/plugins/mapper-murmur3/src/yamlRestTest/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml +++ b/plugins/mapper-murmur3/src/yamlRestTest/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml @@ -134,9 +134,10 @@ setup: indices.create: index: test_synthetic_source body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: foo: type: keyword diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/EnrollmentProcessTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/EnrollmentProcessTests.java index 0f0a599423092..836a6b71c4a19 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/EnrollmentProcessTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/EnrollmentProcessTests.java @@ -116,7 +116,7 @@ public void test20DockerAutoFormCluster() throws Exception { makeRequestAsElastic("https://localhost:9200/_cluster/health", "password"), containsString("\"number_of_nodes\":2") ), - 20, + 60, TimeUnit.SECONDS ); diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/FileUtils.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/FileUtils.java index 76140da810c2b..2b8fb0d4e6021 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/FileUtils.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/FileUtils.java @@ -373,7 +373,7 @@ public static String escapePath(Path path) { } /** - * Recursively copy the the source directory to the target directory, preserving permissions. + * Recursively copy the source directory to the target directory, preserving permissions. */ public static void copyDirectory(Path source, Path target) throws IOException { Files.walkFileTree(source, new SimpleFileVisitor<>() { diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml index a7600da575cd3..7ca7580b0d5ca 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml @@ -7,9 +7,10 @@ keyword: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -48,9 +49,8 @@ fetch without refresh also produces synthetic source: settings: index: refresh_interval: -1 + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: obj: properties: @@ -90,9 +90,10 @@ force_synthetic_source_ok: indices.create: index: test body: + settings: + index: + mapping.source.mode: stored mappings: - _source: - mode: stored properties: obj: properties: @@ -139,9 +140,10 @@ stored text: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: text: type: text @@ -212,9 +214,10 @@ stored keyword: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -253,9 +256,10 @@ doc values keyword with ignore_above: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -336,9 +340,10 @@ stored keyword with ignore_above: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -421,9 +426,10 @@ indexed dense vectors: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -465,9 +471,10 @@ non-indexed dense vectors: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -508,9 +515,10 @@ _source filtering: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -550,9 +558,9 @@ _doc_count: indices.create: index: test body: - mappings: - _source: - mode: synthetic + settings: + index: + mapping.source.mode: synthetic # with _doc_count - do: @@ -679,9 +687,10 @@ fields with ignore_malformed: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: ip: type: ip @@ -914,9 +923,10 @@ flattened field: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: flattened: type: flattened @@ -1006,9 +1016,10 @@ flattened field with ignore_above: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: field: type: flattened @@ -1061,9 +1072,10 @@ flattened field with ignore_above and arrays: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: field: type: flattened @@ -1117,9 +1129,10 @@ completion: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: completion: type: completion diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index 258dfeb57e00c..15a712d77a7ef 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -1,3 +1,4 @@ +--- object with unmapped fields: - requires: cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] @@ -10,13 +11,11 @@ object with unmapped fields: settings: index: mapping: + source.mode: synthetic total_fields: ignore_dynamic_beyond_limit: true limit: 1 - mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -63,13 +62,12 @@ unmapped arrays: settings: index: mapping: + source.mode: synthetic total_fields: ignore_dynamic_beyond_limit: true limit: 1 mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -110,13 +108,12 @@ nested object with unmapped fields: settings: index: mapping: + source.mode: synthetic total_fields: ignore_dynamic_beyond_limit: true limit: 3 mappings: - _source: - mode: synthetic properties: path: properties: @@ -162,13 +159,12 @@ empty object with unmapped fields: settings: index: mapping: + source.mode: synthetic total_fields: ignore_dynamic_beyond_limit: true limit: 3 mappings: - _source: - mode: synthetic properties: path: properties: @@ -204,9 +200,10 @@ disabled root object: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic enabled: false - do: @@ -241,9 +238,10 @@ disabled object: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: path: enabled: false @@ -278,9 +276,10 @@ disabled object contains array: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: path: enabled: false @@ -318,9 +317,10 @@ disabled subobject: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: path: properties: @@ -356,9 +356,10 @@ disabled subobject with array: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: path: properties: @@ -395,9 +396,10 @@ mixed disabled and enabled objects: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: path: properties: @@ -441,9 +443,10 @@ object with dynamic override: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: path_no: dynamic: false @@ -488,9 +491,10 @@ subobject with dynamic override: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: path: properties: @@ -536,9 +540,10 @@ object array in object with dynamic override: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: id: type: integer @@ -590,9 +595,10 @@ value array in object with dynamic override: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: path_no: dynamic: false @@ -633,9 +639,10 @@ nested object: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: nested_field: type: nested @@ -678,9 +685,10 @@ nested object next to regular: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: path: properties: @@ -724,9 +732,10 @@ nested object with disabled: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: obj_field: properties: @@ -812,9 +821,10 @@ doubly nested object: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: obj_field: properties: @@ -907,9 +917,10 @@ subobjects auto: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic subobjects: auto properties: id: @@ -995,9 +1006,10 @@ synthetic_source with copy_to: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: number: type: integer @@ -1131,9 +1143,10 @@ synthetic_source with disabled doc_values: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: number: type: integer @@ -1214,9 +1227,10 @@ fallback synthetic_source for text field: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: text: type: text @@ -1248,9 +1262,10 @@ synthetic_source with copy_to and ignored values: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -1316,9 +1331,10 @@ synthetic_source with copy_to field having values in source: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -1379,9 +1395,10 @@ synthetic_source with ignored source field using copy_to: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -1443,9 +1460,10 @@ synthetic_source with copy_to field from dynamic template having values in sourc indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic dynamic_templates: - copy_template: match: "k" @@ -1540,9 +1558,10 @@ synthetic_source with copy_to and invalid values for copy: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -1576,9 +1595,10 @@ synthetic_source with copy_to pointing inside object: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -1680,9 +1700,10 @@ synthetic_source with copy_to pointing to ambiguous field: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: k: type: keyword @@ -1727,9 +1748,10 @@ synthetic_source with copy_to pointing to ambiguous field and subobjects false: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic subobjects: false properties: k: @@ -1775,9 +1797,10 @@ synthetic_source with copy_to pointing to ambiguous field and subobjects auto: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic subobjects: auto properties: k: @@ -1824,9 +1847,10 @@ synthetic_source with copy_to pointing at dynamic field: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -1910,9 +1934,10 @@ synthetic_source with copy_to pointing inside dynamic object: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index f3545bb0a3f0e..4f55b52224a38 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -8,9 +8,10 @@ object param - store complex object: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: id: type: integer @@ -72,9 +73,10 @@ object param - object array: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: id: type: integer @@ -136,9 +138,10 @@ object param - object array within array: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: stored: synthetic_source_keep: arrays @@ -179,9 +182,10 @@ object param - no object array: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: stored: synthetic_source_keep: arrays @@ -221,9 +225,10 @@ object param - field ordering in object array: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: a: type: keyword @@ -270,9 +275,10 @@ object param - nested object array next to other fields: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: a: type: keyword @@ -326,9 +332,10 @@ object param - nested object with stored array: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -378,9 +385,10 @@ index param - nested array within array: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -428,9 +436,9 @@ index param - nested array within array - disabled second pass: index: synthetic_source: enable_second_doc_parsing_pass: false + mapping.source.mode: synthetic + mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -478,9 +486,8 @@ stored field under object with store_array_source: index: sort.field: "name" sort.order: "asc" + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -525,9 +532,10 @@ field param - keep root array: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: id: type: integer @@ -582,9 +590,10 @@ field param - keep nested array: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: id: type: integer @@ -650,9 +659,10 @@ field param - keep root singleton fields: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: id: type: integer @@ -739,9 +749,10 @@ field param - keep nested singleton fields: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: id: type: integer @@ -820,9 +831,10 @@ field param - nested array within array: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -866,10 +878,9 @@ index param - root arrays: settings: index: mapping: + source.mode: synthetic synthetic_source_keep: arrays mappings: - _source: - mode: synthetic properties: id: type: integer @@ -945,10 +956,9 @@ index param - dynamic root arrays: settings: index: mapping: + source.mode: synthetic synthetic_source_keep: arrays mappings: - _source: - mode: synthetic properties: id: type: integer @@ -998,10 +1008,9 @@ index param - object array within array: settings: index: mapping: + source.mode: synthetic synthetic_source_keep: arrays mappings: - _source: - mode: synthetic properties: stored: properties: @@ -1048,10 +1057,9 @@ index param - no object array: settings: index: mapping: + source.mode: synthetic synthetic_source_keep: arrays mappings: - _source: - mode: synthetic properties: stored: properties: @@ -1093,10 +1101,9 @@ index param - field ordering: settings: index: mapping: + source.mode: synthetic synthetic_source_keep: arrays mappings: - _source: - mode: synthetic properties: a: type: keyword @@ -1144,10 +1151,9 @@ index param - nested arrays: settings: index: mapping: + source.mode: synthetic synthetic_source_keep: arrays mappings: - _source: - mode: synthetic properties: a: type: keyword @@ -1212,10 +1218,9 @@ index param - nested object with stored array: settings: index: mapping: + source.mode: synthetic synthetic_source_keep: arrays mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -1264,10 +1269,9 @@ index param - flattened fields: settings: index: mapping: + source.mode: synthetic synthetic_source_keep: arrays mappings: - _source: - mode: synthetic properties: name: type: keyword diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml index 75d282d524607..67419cd18ad99 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml @@ -145,28 +145,6 @@ - is_false: test_index.mappings.properties.foo.meta.bar - match: { test_index.mappings.properties.foo.meta.baz: "quux" } ---- -"disabling synthetic source fails": - - requires: - cluster_features: ["gte_v8.4.0"] - reason: "Added in 8.4.0" - - - do: - indices.create: - index: test_index - body: - mappings: - _source: - mode: synthetic - - - do: - catch: /Cannot update parameter \[mode\] from \[synthetic\] to \[stored\]/ - indices.put_mapping: - index: test_index - body: - _source: - mode: stored - --- "enabling synthetic source from explicit succeeds": - requires: @@ -177,9 +155,9 @@ indices.create: index: test_index body: - mappings: - _source: - mode: stored + settings: + index: + mapping.source.mode: stored - do: indices.put_mapping: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml index 4439441efdd02..20c2ef63fc850 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml @@ -599,3 +599,284 @@ end time not allowed in logs mode: - match: { error.root_cause.0.type: "illegal_argument_exception" } - match: { error.type: "illegal_argument_exception" } - match: { error.reason: "[index.time_series.end_time] requires [index.mode=time_series]" } + +--- +ignore dynamic beyond limit logsdb default value: + - requires: + cluster_features: [ "mapper.logsdb_default_ignore_dynamic_beyond_limit" ] + reason: requires logsdb default value for `index.mapping.total_fields.ignore_dynamic_beyond_limit` + + - do: + indices.create: + index: test-ignore-dynamic-default + body: + settings: + index: + mode: logsdb + + - do: + indices.get_settings: + index: test-ignore-dynamic-default + include_defaults: true + + - match: { test-ignore-dynamic-default.settings.index.mode: "logsdb" } + - match: { test-ignore-dynamic-default.defaults.index.mapping.total_fields.limit: "1000" } + - match: { test-ignore-dynamic-default.defaults.index.mapping.total_fields.ignore_dynamic_beyond_limit: "true" } + +--- +ignore dynamic beyond limit logsdb override value: + - requires: + cluster_features: [ "mapper.logsdb_default_ignore_dynamic_beyond_limit" ] + reason: requires logsdb default value for `index.mapping.total_fields.ignore_dynamic_beyond_limit` + + - do: + indices.create: + index: test-ignore-dynamic-override + body: + settings: + index: + mode: logsdb + mapping: + total_fields: + ignore_dynamic_beyond_limit: false + + - do: + indices.get_settings: + index: test-ignore-dynamic-override + + - match: { test-ignore-dynamic-override.settings.index.mode: "logsdb" } + - match: { test-ignore-dynamic-override.settings.index.mapping.total_fields.ignore_dynamic_beyond_limit: "false" } + +--- +logsdb with default ignore dynamic beyond limit and default sorting: + - requires: + cluster_features: ["mapper.logsdb_default_ignore_dynamic_beyond_limit"] + reason: requires default value for ignore_dynamic_beyond_limit + + - do: + indices.create: + index: test-logsdb-default-sort + body: + settings: + index: + mode: logsdb + mapping: + # NOTE: When the index mode is set to `logsdb`, the `host.name` field is automatically injected if + # sort settings are not overridden. + # With `subobjects` set to `true` (default), this creates a `host` object field and a nested `name` + # keyword field (`host.name`). + # + # As a result, there are always at least 4 statically mapped fields (`@timestamp`, `host`, `host.name` + # and `name`). We cannot use a field limit lower than 4 because these fields are always present. + # + # Indeed, if `index.mapping.total_fields.ignore_dynamic_beyond_limit` is `true`, any dynamically + # mapped fields beyond the limit `index.mapping.total_fields.limit` are ignored, but the statically + # mapped fields are always counted. + total_fields: + limit: 4 + mappings: + properties: + "@timestamp": + type: date + name: + type: keyword + + - do: + indices.get_settings: + index: test-logsdb-default-sort + + - match: { test-logsdb-default-sort.settings.index.mode: "logsdb" } + + - do: + bulk: + index: test-logsdb-default-sort + refresh: true + body: + - '{ "index": { } }' + - '{ "@timestamp": "2024-08-13T12:30:00Z", "name": "foo", "host.name": "92f4a67c", "value": 10, "message": "the quick brown fox", "region": "us-west", "pid": 153462 }' + - '{ "index": { } }' + - '{ "@timestamp": "2024-08-13T12:01:00Z", "name": "bar", "host.name": "24eea278", "value": 20, "message": "jumps over the lazy dog", "region": "us-central", "pid": 674972 }' + - match: { errors: false } + + - do: + search: + index: test-logsdb-default-sort + body: + query: + match_all: {} + + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.name: "bar" } + - match: { hits.hits.0._source.value: 20 } + - match: { hits.hits.0._source.message: "jumps over the lazy dog" } + - match: { hits.hits.0._ignored: [ "message", "pid", "region", "value" ] } + - match: { hits.hits.1._source.name: "foo" } + - match: { hits.hits.1._source.value: 10 } + - match: { hits.hits.1._source.message: "the quick brown fox" } + - match: { hits.hits.1._ignored: [ "message", "pid", "region", "value" ] } + +--- +logsdb with default ignore dynamic beyond limit and non-default sorting: + - requires: + cluster_features: ["mapper.logsdb_default_ignore_dynamic_beyond_limit"] + reason: requires default value for ignore_dynamic_beyond_limit + + - do: + indices.create: + index: test-logsdb-non-default-sort + body: + settings: + index: + sort.field: [ "name" ] + sort.order: [ "desc" ] + mode: logsdb + mapping: + # NOTE: Here sort settings are overridden and we do not have any additional statically mapped field other + # than `name` and `timestamp`. As a result, there are only 2 statically mapped fields. + total_fields: + limit: 2 + mappings: + properties: + "@timestamp": + type: date + name: + type: keyword + + - do: + indices.get_settings: + index: test-logsdb-non-default-sort + + - match: { test-logsdb-non-default-sort.settings.index.mode: "logsdb" } + + - do: + bulk: + index: test-logsdb-non-default-sort + refresh: true + body: + - '{ "index": { } }' + - '{ "@timestamp": "2024-08-13T12:30:00Z", "name": "foo", "host.name": "92f4a67c", "value": 10, "message": "the quick brown fox", "region": "us-west", "pid": 153462 }' + - '{ "index": { } }' + - '{ "@timestamp": "2024-08-13T12:01:00Z", "name": "bar", "host.name": "24eea278", "value": 20, "message": "jumps over the lazy dog", "region": "us-central", "pid": 674972 }' + - match: { errors: false } + + - do: + search: + index: test-logsdb-non-default-sort + body: + query: + match_all: {} + + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.name: "foo" } + - match: { hits.hits.0._source.value: 10 } + - match: { hits.hits.0._source.message: "the quick brown fox" } + - match: { hits.hits.0._ignored: [ "host", "message", "pid", "region", "value" ] } + - match: { hits.hits.1._source.name: "bar" } + - match: { hits.hits.1._source.value: 20 } + - match: { hits.hits.1._source.message: "jumps over the lazy dog" } + - match: { hits.hits.1._ignored: [ "host", "message", "pid", "region", "value" ] } + +--- +logsdb with default ignore dynamic beyond limit and too low limit: + - requires: + cluster_features: ["mapper.logsdb_default_ignore_dynamic_beyond_limit"] + reason: requires default value for ignore_dynamic_beyond_limit + + - do: + catch: bad_request + indices.create: + index: test-logsdb-low-limit + body: + settings: + index: + mode: logsdb + mapping: + # NOTE: When the index mode is set to `logsdb`, the `host.name` field is automatically injected if + # sort settings are not overridden. + # With `subobjects` set to `true` (default), this creates a `host` object field and a nested `name` + # keyword field (`host.name`). + # + # As a result, there are always at least 4 statically mapped fields (`@timestamp`, `host`, `host.name` + # and `name`). We cannot use a field limit lower than 4 because these fields are always present. + # + # Indeed, if `index.mapping.total_fields.ignore_dynamic_beyond_limit` is `true`, any dynamically + # mapped fields beyond the limit `index.mapping.total_fields.limit` are ignored, but the statically + # mapped fields are always counted. + total_fields: + limit: 3 + mappings: + properties: + "@timestamp": + type: date + name: + type: keyword + - match: { error.type: "illegal_argument_exception" } + - match: { error.reason: "Limit of total fields [3] has been exceeded" } + +--- +logsdb with default ignore dynamic beyond limit and subobjects false: + - requires: + cluster_features: ["mapper.logsdb_default_ignore_dynamic_beyond_limit"] + reason: requires default value for ignore_dynamic_beyond_limit + + - do: + indices.create: + index: test-logsdb-subobjects-false + body: + settings: + index: + mode: logsdb + mapping: + # NOTE: When the index mode is set to `logsdb`, the `host.name` field is automatically injected if + # sort settings are not overridden. + # With `subobjects` set to `false` anyway, a single `host.name` keyword field is automatically mapped. + # + # As a result, there are just 3 statically mapped fields (`@timestamp`, `host.name` and `name`). + # We cannot use a field limit lower than 3 because these fields are always present. + # + # Indeed, if `index.mapping.total_fields.ignore_dynamic_beyond_limit` is `true`, any dynamically + # mapped fields beyond the limit `index.mapping.total_fields.limit` are ignored, but the statically + # mapped fields are always counted. + total_fields: + limit: 3 + mappings: + subobjects: false + properties: + "@timestamp": + type: date + name: + type: keyword + + - do: + indices.get_settings: + index: test-logsdb-subobjects-false + + - match: { test-logsdb-subobjects-false.settings.index.mode: "logsdb" } + + - do: + bulk: + index: test-logsdb-subobjects-false + refresh: true + body: + - '{ "index": { } }' + - '{ "@timestamp": "2024-08-13T12:30:00Z", "name": "foo", "host.name": "92f4a67c", "value": 10, "message": "the quick brown fox", "region": "us-west", "pid": 153462 }' + - '{ "index": { } }' + - '{ "@timestamp": "2024-08-13T12:01:00Z", "name": "bar", "host.name": "24eea278", "value": 20, "message": "jumps over the lazy dog", "region": "us-central", "pid": 674972 }' + - match: { errors: false } + + - do: + search: + index: test-logsdb-subobjects-false + body: + query: + match_all: {} + + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.name: "bar" } + - match: { hits.hits.0._source.value: 20 } + - match: { hits.hits.0._source.message: "jumps over the lazy dog" } + - match: { hits.hits.0._ignored: [ "message", "pid", "region", "value" ] } + - match: { hits.hits.1._source.name: "foo" } + - match: { hits.hits.1._source.value: 10 } + - match: { hits.hits.1._source.message: "the quick brown fox" } + - match: { hits.hits.1._ignored: [ "message", "pid", "region", "value" ] } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml index b4709a4e4d176..e77f4a4c912b7 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml @@ -30,9 +30,7 @@ stored _source mode is supported: settings: index: mode: logsdb - mappings: - _source: - mode: stored + mapping.source.mode: stored - do: indices.get: index: test-stored-source @@ -69,9 +67,7 @@ disabled _source is not supported: settings: index: mode: logsdb - mappings: - _source: - mode: disabled + mapping.source.mode: disabled - match: { error.type: "mapper_parsing_exception" } - match: { error.root_cause.0.type: "mapper_parsing_exception" } @@ -120,9 +116,9 @@ include/exclude is supported with stored _source: settings: index: mode: logsdb + mapping.source.mode: stored mappings: _source: - mode: stored includes: [a] - do: @@ -139,9 +135,9 @@ include/exclude is supported with stored _source: settings: index: mode: logsdb + mapping.source.mode: stored mappings: _source: - mode: stored excludes: [b] - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml index 2f3d2fa2f974d..c728252d98201 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml @@ -7,9 +7,10 @@ keyword: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -62,9 +63,9 @@ keyword with normalizer: type: custom filter: - lowercase + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: keyword: type: keyword @@ -144,9 +145,10 @@ stored text: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: text: type: text @@ -193,9 +195,10 @@ force_synthetic_source_ok: indices.create: index: test body: + settings: + index: + mapping.source.mode: stored mappings: - _source: - mode: stored properties: obj: properties: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/50_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/50_synthetic_source.yml index a2fd448f5044d..024ce48562281 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/50_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/50_synthetic_source.yml @@ -9,9 +9,9 @@ setup: body: settings: number_of_shards: 1 + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: foo: type: keyword @@ -21,6 +21,7 @@ setup: index_options: positions vectors: type: text + store: false term_vector: with_positions_offsets positions: type: text diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml index 27f12f394c6a4..f4a55341dff08 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml @@ -394,9 +394,10 @@ indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: ml.tokens: type: sparse_vector diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/350_binary_field.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/350_binary_field.yml index 455d06ba2a984..ced9c4fe0825d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/350_binary_field.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/350_binary_field.yml @@ -55,9 +55,10 @@ indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: binary: type: binary diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml index 0cc1796bb47de..21338c12415d8 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml @@ -7,9 +7,10 @@ keyword: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -44,9 +45,10 @@ stored text: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: text: type: text @@ -83,8 +85,6 @@ stored keyword: index: test body: mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -120,9 +120,10 @@ stored keyword without sibling fields: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -165,9 +166,10 @@ force_synthetic_source_ok: indices.create: index: test body: + settings: + index: + mapping.source.mode: stored mappings: - _source: - mode: stored properties: obj: properties: @@ -218,9 +220,10 @@ doc values keyword with ignore_above: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -286,9 +289,10 @@ stored keyword with ignore_above: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -356,9 +360,10 @@ _source filtering: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -397,9 +402,9 @@ _doc_count: indices.create: index: test body: - mappings: - _source: - mode: synthetic + settings: + index: + mapping.source.mode: synthetic - do: index: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml index 435cda637cca6..8950a378c7203 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml @@ -10,10 +10,9 @@ ignore_above mapping level setting: settings: index: mapping: + source.mode: synthetic ignore_above: 10 mappings: - _source: - mode: synthetic properties: keyword: type: keyword @@ -53,10 +52,9 @@ ignore_above mapping level setting on arrays: settings: index: mapping: + source.mode: synthetic ignore_above: 10 mappings: - _source: - mode: synthetic properties: keyword: type: keyword @@ -97,10 +95,9 @@ ignore_above mapping overrides setting: settings: index: mapping: + source.mode: synthetic ignore_above: 10 mappings: - _source: - mode: synthetic properties: keyword: type: keyword @@ -143,10 +140,9 @@ ignore_above mapping overrides setting on arrays: settings: index: mapping: + source.mode: synthetic ignore_above: 10 mappings: - _source: - mode: synthetic properties: keyword: type: keyword diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index c5669cd6414b1..4fe4a7d2e2f51 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -472,9 +472,9 @@ stored source is supported: time_series: start_time: 2021-04-28T00:00:00Z end_time: 2021-04-29T00:00:00Z + mapping: + source.mode: stored mappings: - _source: - mode: stored properties: "@timestamp": type: date @@ -510,9 +510,9 @@ disabled source is not supported: time_series: start_time: 2021-04-28T00:00:00Z end_time: 2021-04-29T00:00:00Z + mapping: + source.mode: disabled mappings: - _source: - mode: disabled properties: "@timestamp": type: date diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml index f74fde7eb2a24..c3f013395ea36 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml @@ -7,9 +7,10 @@ keyword: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -65,9 +66,10 @@ stored text: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: text: type: text diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java index fbc4e08b6b78a..de565605ff58a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java @@ -70,7 +70,7 @@ public void testSystemIndexManagerUpgradesMappings() throws Exception { } /** - * Check that if the the SystemIndexManager finds a managed index with mappings that claim to be newer than + * Check that if the SystemIndexManager finds a managed index with mappings that claim to be newer than * what it expects, then those mappings are left alone. */ public void testSystemIndexManagerLeavesNewerMappingsAlone() throws Exception { diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java index 5c4195bc7b069..43167e206bfb5 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.UnavailableShardsException; import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.action.support.RefCountingListener; import org.elasticsearch.action.support.RetryableAction; import org.elasticsearch.action.support.TransportActions; import org.elasticsearch.cluster.action.shard.ShardStateAction; @@ -55,17 +56,6 @@ public class ReplicationOperation< private final Request request; private final String opType; private final AtomicInteger totalShards = new AtomicInteger(); - /** - * The number of pending sub-operations in this operation. This is incremented when the following operations start and decremented when - * they complete: - *
      - *
    • The operation on the primary
    • - *
    • The operation on each replica
    • - *
    • Coordination of the operation as a whole. This prevents the operation from terminating early if we haven't started any replica - * operations and the primary finishes.
    • - *
    - */ - private final AtomicInteger pendingActions = new AtomicInteger(); private final AtomicInteger successfulShards = new AtomicInteger(); private final Primary primary; private final Replicas replicasProxy; @@ -74,7 +64,6 @@ public class ReplicationOperation< private final TimeValue retryTimeout; private final long primaryTerm; - // exposed for tests private final ActionListener resultListener; private volatile PrimaryResultT primaryResult = null; @@ -105,29 +94,57 @@ public ReplicationOperation( this.retryTimeout = retryTimeout; } + /** + * The execution is based on a {@link RefCountingListener} that encapsulates the pending sub-operations in this operation. A new + * listener is acquired when the following sub-operations start and triggered when they complete: + *
      + *
    • The operation on the primary
    • + *
    • The operation on each replica
    • + *
    • Coordination of the operation as a whole. This prevents the operation from terminating early if we haven't started any replica + * operations and the primary finishes.
    • + *
    + */ public void execute() throws Exception { - final String activeShardCountFailure = checkActiveShardCount(); - final ShardRouting primaryRouting = primary.routingEntry(); - final ShardId primaryId = primaryRouting.shardId(); - if (activeShardCountFailure != null) { - finishAsFailed( - new UnavailableShardsException( - primaryId, - "{} Timeout: [{}], request: [{}]", - activeShardCountFailure, - request.timeout(), - request + try (var pendingActionsListener = new RefCountingListener(ActionListener.wrap((ignored) -> { + primaryResult.setShardInfo( + ReplicationResponse.ShardInfo.of( + totalShards.get(), + successfulShards.get(), + shardReplicaFailures.toArray(ReplicationResponse.NO_FAILURES) ) ); - return; - } + resultListener.onResponse(primaryResult); + }, resultListener::onFailure))) { + final String activeShardCountFailure = checkActiveShardCount(); + final ShardRouting primaryRouting = primary.routingEntry(); + final ShardId primaryId = primaryRouting.shardId(); + if (activeShardCountFailure != null) { + pendingActionsListener.acquire() + .onFailure( + new UnavailableShardsException( + primaryId, + "{} Timeout: [{}], request: [{}]", + activeShardCountFailure, + request.timeout(), + request + ) + ); + return; + } - totalShards.incrementAndGet(); - pendingActions.incrementAndGet(); // increase by 1 until we finish all primary coordination - primary.perform(request, ActionListener.wrap(this::handlePrimaryResult, this::finishAsFailed)); + totalShards.incrementAndGet(); + var primaryCoordinationPendingActionListener = pendingActionsListener.acquire(); // triggered when we finish all coordination + primary.perform(request, primaryCoordinationPendingActionListener.delegateFailureAndWrap((l, primaryResult) -> { + handlePrimaryResult(primaryResult, l, pendingActionsListener); + })); + } } - private void handlePrimaryResult(final PrimaryResultT primaryResult) { + private void handlePrimaryResult( + final PrimaryResultT primaryResult, + final ActionListener primaryCoordinationPendingActionListener, + final RefCountingListener pendingActionsListener + ) { this.primaryResult = primaryResult; final ReplicaRequest replicaRequest = primaryResult.replicaRequest(); if (replicaRequest != null) { @@ -136,11 +153,11 @@ private void handlePrimaryResult(final PrimaryResultT primaryResult) { } final ReplicationGroup replicationGroup = primary.getReplicationGroup(); - pendingActions.incrementAndGet(); + var primaryOperationPendingActionListener = pendingActionsListener.acquire(); replicasProxy.onPrimaryOperationComplete( replicaRequest, replicationGroup.getRoutingTable(), - ActionListener.wrap(ignored -> decPendingAndFinishIfNeeded(), exception -> { + ActionListener.wrap(ignored -> primaryOperationPendingActionListener.onResponse(null), exception -> { totalShards.incrementAndGet(); shardReplicaFailures.add( new ReplicationResponse.ShardInfo.Failure( @@ -151,7 +168,7 @@ private void handlePrimaryResult(final PrimaryResultT primaryResult) { false ) ); - decPendingAndFinishIfNeeded(); + primaryOperationPendingActionListener.onResponse(null); }) ); @@ -169,8 +186,15 @@ private void handlePrimaryResult(final PrimaryResultT primaryResult) { final long maxSeqNoOfUpdatesOrDeletes = primary.maxSeqNoOfUpdatesOrDeletes(); assert maxSeqNoOfUpdatesOrDeletes != SequenceNumbers.UNASSIGNED_SEQ_NO : "seqno_of_updates still uninitialized"; final PendingReplicationActions pendingReplicationActions = primary.getPendingReplicationActions(); - markUnavailableShardsAsStale(replicaRequest, replicationGroup); - performOnReplicas(replicaRequest, globalCheckpoint, maxSeqNoOfUpdatesOrDeletes, replicationGroup, pendingReplicationActions); + markUnavailableShardsAsStale(replicaRequest, replicationGroup, pendingActionsListener); + performOnReplicas( + replicaRequest, + globalCheckpoint, + maxSeqNoOfUpdatesOrDeletes, + replicationGroup, + pendingReplicationActions, + pendingActionsListener + ); } primaryResult.runPostReplicationActions(new ActionListener<>() { @@ -181,7 +205,7 @@ public void onResponse(Void aVoid) { primary.routingEntry(), primary::localCheckpoint, primary::globalCheckpoint, - () -> decPendingAndFinishIfNeeded() + () -> primaryCoordinationPendingActionListener.onResponse(null) ); } @@ -193,20 +217,29 @@ public void onFailure(Exception e) { // We update the checkpoints since a refresh might fail but the operations could be safely persisted, in the case that the // fsync failed the local checkpoint won't advance and the engine will be marked as failed when the next indexing operation // is appended into the translog. - updateCheckPoints(primary.routingEntry(), primary::localCheckpoint, primary::globalCheckpoint, () -> finishAsFailed(e)); + updateCheckPoints( + primary.routingEntry(), + primary::localCheckpoint, + primary::globalCheckpoint, + () -> primaryCoordinationPendingActionListener.onFailure(e) + ); } }); } - private void markUnavailableShardsAsStale(ReplicaRequest replicaRequest, ReplicationGroup replicationGroup) { + private void markUnavailableShardsAsStale( + final ReplicaRequest replicaRequest, + final ReplicationGroup replicationGroup, + final RefCountingListener pendingActionsListener + ) { // if inSyncAllocationIds contains allocation ids of shards that don't exist in RoutingTable, mark copies as stale for (String allocationId : replicationGroup.getUnavailableInSyncShards()) { - pendingActions.incrementAndGet(); + var staleCopyPendingActionListener = pendingActionsListener.acquire(); replicasProxy.markShardCopyAsStaleIfNeeded( replicaRequest.shardId(), allocationId, primaryTerm, - ActionListener.wrap(r -> decPendingAndFinishIfNeeded(), ReplicationOperation.this::onNoLongerPrimary) + staleCopyPendingActionListener.delegateResponse((l, e) -> onNoLongerPrimary(e, l)) ); } } @@ -216,7 +249,8 @@ private void performOnReplicas( final long globalCheckpoint, final long maxSeqNoOfUpdatesOrDeletes, final ReplicationGroup replicationGroup, - final PendingReplicationActions pendingReplicationActions + final PendingReplicationActions pendingReplicationActions, + final RefCountingListener pendingActionsListener ) { // for total stats, add number of unassigned shards and // number of initializing shards that are not ready yet to receive operations (recovery has not opened engine yet on the target) @@ -226,7 +260,14 @@ private void performOnReplicas( for (final ShardRouting shard : replicationGroup.getReplicationTargets()) { if (shard.isSameAllocation(primaryRouting) == false) { - performOnReplica(shard, replicaRequest, globalCheckpoint, maxSeqNoOfUpdatesOrDeletes, pendingReplicationActions); + performOnReplica( + shard, + replicaRequest, + globalCheckpoint, + maxSeqNoOfUpdatesOrDeletes, + pendingReplicationActions, + pendingActionsListener + ); } } } @@ -236,94 +277,97 @@ private void performOnReplica( final ReplicaRequest replicaRequest, final long globalCheckpoint, final long maxSeqNoOfUpdatesOrDeletes, - final PendingReplicationActions pendingReplicationActions + final PendingReplicationActions pendingReplicationActions, + final RefCountingListener pendingActionsListener ) { assert shard.isPromotableToPrimary() : "only promotable shards should receive replication requests"; if (logger.isTraceEnabled()) { logger.trace("[{}] sending op [{}] to replica {} for request [{}]", shard.shardId(), opType, shard, replicaRequest); } totalShards.incrementAndGet(); - pendingActions.incrementAndGet(); - final ActionListener replicationListener = new ActionListener<>() { - @Override - public void onResponse(ReplicaResponse response) { - successfulShards.incrementAndGet(); - updateCheckPoints(shard, response::localCheckpoint, response::globalCheckpoint, () -> decPendingAndFinishIfNeeded()); - } + var replicationPendingActionListener = pendingActionsListener.acquire(); + ActionListener.run(replicationPendingActionListener, (listener) -> { + final ActionListener replicationListener = new ActionListener<>() { + @Override + public void onResponse(ReplicaResponse response) { + successfulShards.incrementAndGet(); + updateCheckPoints(shard, response::localCheckpoint, response::globalCheckpoint, () -> listener.onResponse(null)); + } - @Override - public void onFailure(Exception replicaException) { - logger.trace( - () -> format( - "[%s] failure while performing [%s] on replica %s, request [%s]", - shard.shardId(), - opType, - shard, - replicaRequest - ), - replicaException - ); - // Only report "critical" exceptions - TODO: Reach out to the master node to get the latest shard state then report. - if (TransportActions.isShardNotAvailableException(replicaException) == false) { - RestStatus restStatus = ExceptionsHelper.status(replicaException); - shardReplicaFailures.add( - new ReplicationResponse.ShardInfo.Failure( + @Override + public void onFailure(Exception replicaException) { + logger.trace( + () -> format( + "[%s] failure while performing [%s] on replica %s, request [%s]", shard.shardId(), - shard.currentNodeId(), - replicaException, - restStatus, - false - ) + opType, + shard, + replicaRequest + ), + replicaException + ); + // Only report "critical" exceptions - TODO: Reach out to the master node to get the latest shard state then report. + if (TransportActions.isShardNotAvailableException(replicaException) == false) { + RestStatus restStatus = ExceptionsHelper.status(replicaException); + shardReplicaFailures.add( + new ReplicationResponse.ShardInfo.Failure( + shard.shardId(), + shard.currentNodeId(), + replicaException, + restStatus, + false + ) + ); + } + String message = String.format(Locale.ROOT, "failed to perform %s on replica %s", opType, shard); + replicasProxy.failShardIfNeeded( + shard, + primaryTerm, + message, + replicaException, + listener.delegateResponse((l, e) -> onNoLongerPrimary(e, l)) ); } - String message = String.format(Locale.ROOT, "failed to perform %s on replica %s", opType, shard); - replicasProxy.failShardIfNeeded( - shard, - primaryTerm, - message, - replicaException, - ActionListener.wrap(r -> decPendingAndFinishIfNeeded(), ReplicationOperation.this::onNoLongerPrimary) - ); - } - @Override - public String toString() { - return "[" + replicaRequest + "][" + shard + "]"; - } - }; - - final String allocationId = shard.allocationId().getId(); - final RetryableAction replicationAction = new RetryableAction<>( - logger, - threadPool, - initialRetryBackoffBound, - retryTimeout, - replicationListener, - EsExecutors.DIRECT_EXECUTOR_SERVICE - ) { + @Override + public String toString() { + return "[" + replicaRequest + "][" + shard + "]"; + } + }; + + final String allocationId = shard.allocationId().getId(); + final RetryableAction replicationAction = new RetryableAction<>( + logger, + threadPool, + initialRetryBackoffBound, + retryTimeout, + replicationListener, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ) { - @Override - public void tryAction(ActionListener listener) { - replicasProxy.performOn(shard, replicaRequest, primaryTerm, globalCheckpoint, maxSeqNoOfUpdatesOrDeletes, listener); - } + @Override + public void tryAction(ActionListener listener) { + replicasProxy.performOn(shard, replicaRequest, primaryTerm, globalCheckpoint, maxSeqNoOfUpdatesOrDeletes, listener); + } - @Override - public void onFinished() { - super.onFinished(); - pendingReplicationActions.removeReplicationAction(allocationId, this); - } + @Override + public void onFinished() { + super.onFinished(); + pendingReplicationActions.removeReplicationAction(allocationId, this); + } - @Override - public boolean shouldRetry(Exception e) { - final Throwable cause = ExceptionsHelper.unwrapCause(e); - return cause instanceof CircuitBreakingException - || cause instanceof EsRejectedExecutionException - || cause instanceof ConnectTransportException; - } - }; + @Override + public boolean shouldRetry(Exception e) { + final Throwable cause = ExceptionsHelper.unwrapCause(e); + return cause instanceof CircuitBreakingException + || cause instanceof EsRejectedExecutionException + || cause instanceof ConnectTransportException; + } + }; - pendingReplicationActions.addPendingAction(allocationId, replicationAction); - replicationAction.run(); + pendingReplicationActions.addPendingAction(allocationId, replicationAction); + replicationAction.run(); + }); } private void updateCheckPoints( @@ -369,58 +413,60 @@ public void onAfter() { } } - private void onNoLongerPrimary(Exception failure) { - final Throwable cause = ExceptionsHelper.unwrapCause(failure); - final boolean nodeIsClosing = cause instanceof NodeClosedException; - if (nodeIsClosing) { - // We prefer not to fail the primary to avoid unnecessary warning log - // when the node with the primary shard is gracefully shutting down. - finishAsFailed( - new RetryOnPrimaryException( - primary.routingEntry().shardId(), - String.format( - Locale.ROOT, - "node with primary [%s] is shutting down while failing replica shard", - primary.routingEntry() - ), - failure - ) - ); - } else { - assert failure instanceof ShardStateAction.NoLongerPrimaryShardException : failure; - threadPool.executor(ThreadPool.Names.WRITE).execute(new AbstractRunnable() { - @Override - protected void doRun() { - // we are no longer the primary, fail ourselves and start over - final var message = String.format( - Locale.ROOT, - "primary shard [%s] was demoted while failing replica shard", - primary.routingEntry() - ); - primary.failShard(message, failure); - finishAsFailed(new RetryOnPrimaryException(primary.routingEntry().shardId(), message, failure)); - } - - @Override - public boolean isForceExecution() { - return true; - } - - @Override - public void onFailure(Exception e) { - e.addSuppressed(failure); - assert false : e; - logger.error(() -> "unexpected failure while failing primary [" + primary.routingEntry() + "]", e); - finishAsFailed( - new RetryOnPrimaryException( - primary.routingEntry().shardId(), - String.format(Locale.ROOT, "unexpected failure while failing primary [%s]", primary.routingEntry()), - e - ) - ); - } - }); - } + private void onNoLongerPrimary(Exception failure, ActionListener listener) { + ActionListener.run(listener, (l) -> { + final Throwable cause = ExceptionsHelper.unwrapCause(failure); + final boolean nodeIsClosing = cause instanceof NodeClosedException; + if (nodeIsClosing) { + // We prefer not to fail the primary to avoid unnecessary warning log + // when the node with the primary shard is gracefully shutting down. + l.onFailure( + new RetryOnPrimaryException( + primary.routingEntry().shardId(), + String.format( + Locale.ROOT, + "node with primary [%s] is shutting down while failing replica shard", + primary.routingEntry() + ), + failure + ) + ); + } else { + assert failure instanceof ShardStateAction.NoLongerPrimaryShardException : failure; + threadPool.executor(ThreadPool.Names.WRITE).execute(new AbstractRunnable() { + @Override + protected void doRun() { + // we are no longer the primary, fail ourselves and start over + final var message = String.format( + Locale.ROOT, + "primary shard [%s] was demoted while failing replica shard", + primary.routingEntry() + ); + primary.failShard(message, failure); + l.onFailure(new RetryOnPrimaryException(primary.routingEntry().shardId(), message, failure)); + } + + @Override + public boolean isForceExecution() { + return true; + } + + @Override + public void onFailure(Exception e) { + e.addSuppressed(failure); + assert false : e; + logger.error(() -> "unexpected failure while failing primary [" + primary.routingEntry() + "]", e); + l.onFailure( + new RetryOnPrimaryException( + primary.routingEntry().shardId(), + String.format(Locale.ROOT, "unexpected failure while failing primary [%s]", primary.routingEntry()), + e + ) + ); + } + }); + } + }); } /** @@ -461,32 +507,6 @@ protected String checkActiveShardCount() { } } - private void decPendingAndFinishIfNeeded() { - assert pendingActions.get() > 0 : "pending action count goes below 0 for request [" + request + "]"; - if (pendingActions.decrementAndGet() == 0) { - finish(); - } - } - - private void finish() { - if (finished.compareAndSet(false, true)) { - primaryResult.setShardInfo( - ReplicationResponse.ShardInfo.of( - totalShards.get(), - successfulShards.get(), - shardReplicaFailures.toArray(ReplicationResponse.NO_FAILURES) - ) - ); - resultListener.onResponse(primaryResult); - } - } - - private void finishAsFailed(Exception exception) { - if (finished.compareAndSet(false, true)) { - resultListener.onFailure(exception); - } - } - /** * An encapsulation of an operation that is to be performed on the primary shard */ @@ -693,7 +713,7 @@ public interface PrimaryResult> { /** * Run actions to be triggered post replication - * @param listener calllback that is invoked after post replication actions have completed + * @param listener callback that is invoked after post replication actions have completed * */ void runPostReplicationActions(ActionListener listener); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index b7777eca86179..6d2e9c37fc625 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -2402,7 +2402,7 @@ public Metadata build(boolean skipNameCollisionChecks) { assert previousIndicesLookup.equals(buildIndicesLookup(dataStreamMetadata(), indicesMap)); indicesLookup = previousIndicesLookup; } else if (skipNameCollisionChecks == false) { - // we have changes to the the entity names so we ensure we have no naming collisions + // we have changes to the entity names so we ensure we have no naming collisions ensureNoNameCollisions(aliasedIndices.keySet(), indicesMap, dataStreamMetadata()); } assert assertDataStreams(indicesMap, dataStreamMetadata()); diff --git a/server/src/main/java/org/elasticsearch/common/settings/Settings.java b/server/src/main/java/org/elasticsearch/common/settings/Settings.java index 2abfee670b950..09773ba1de35e 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -242,7 +242,7 @@ public Settings getByPrefix(String prefix) { if (prefix.isEmpty()) { return this; } - // create the the next prefix right after the given prefix, and use it as exclusive upper bound for the sub-map to filter by prefix + // create the next prefix right after the given prefix, and use it as exclusive upper bound for the sub-map to filter by prefix // below char[] toPrefixCharArr = prefix.toCharArray(); toPrefixCharArr[toPrefixCharArr.length - 1]++; diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 2919f98ee200e..440613263d441 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -129,8 +129,9 @@ private static Version parseUnchecked(String version) { public static final IndexVersion UPGRADE_TO_LUCENE_9_12 = def(8_516_00_0, Version.LUCENE_9_12_0); public static final IndexVersion ENABLE_IGNORE_ABOVE_LOGSDB = def(8_517_00_0, Version.LUCENE_9_12_0); public static final IndexVersion ADD_ROLE_MAPPING_CLEANUP_MIGRATION = def(8_518_00_0, Version.LUCENE_9_12_0); + public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT_BACKPORT = def(8_519_00_0, Version.LUCENE_9_12_0); public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_00_0, Version.LUCENE_10_0_0); - + public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT = def(9_001_00_0, Version.LUCENE_10_0_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index a5f173afffba2..4797857fc12f8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -64,7 +64,8 @@ public Set getTestFeatures() { IgnoredSourceFieldMapper.DONT_EXPAND_DOTS_IN_IGNORED_SOURCE, SourceFieldMapper.REMOVE_SYNTHETIC_SOURCE_ONLY_VALIDATION, IgnoredSourceFieldMapper.IGNORED_SOURCE_AS_TOP_LEVEL_METADATA_ARRAY_FIELD, - IgnoredSourceFieldMapper.ALWAYS_STORE_OBJECT_ARRAYS_IN_NESTED_OBJECTS + IgnoredSourceFieldMapper.ALWAYS_STORE_OBJECT_ARRAYS_IN_NESTED_OBJECTS, + MapperService.LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 08461525526b9..7f952153c6453 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -22,9 +22,12 @@ import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.AbstractIndexComponent; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -121,9 +124,21 @@ public boolean isAutoUpdate() { Property.IndexScope, Property.ServerlessPublic ); + + public static final NodeFeature LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT = new NodeFeature( + "mapper.logsdb_default_ignore_dynamic_beyond_limit" + ); public static final Setting INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING = Setting.boolSetting( "index.mapping.total_fields.ignore_dynamic_beyond_limit", - false, + settings -> { + boolean isLogsDBIndexMode = IndexSettings.MODE.get(settings) == IndexMode.LOGSDB; + final IndexVersion indexVersionCreated = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings); + boolean isNewIndexVersion = indexVersionCreated.between( + IndexVersions.LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT_BACKPORT, + IndexVersions.UPGRADE_TO_LUCENE_10_0_0 + ) || indexVersionCreated.onOrAfter(IndexVersions.LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT); + return String.valueOf(isLogsDBIndexMode && isNewIndexVersion); + }, Property.Dynamic, Property.IndexScope, Property.ServerlessPublic diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 0dff80ecc2cd6..ee24b8d9a9e91 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -3860,8 +3860,7 @@ public int getActiveOperationsCount() { * listener handles all exception cases internally. */ public final void syncAfterWrite(Translog.Location location, Consumer syncListener) { - // TODO AwaitsFix https://github.com/elastic/elasticsearch/issues/97183 - // assert indexShardOperationPermits.getActiveOperationsCount() != 0; + assert indexShardOperationPermits.getActiveOperationsCount() != 0; verifyNotClosed(); getEngine().asyncEnsureTranslogSynced(location, syncListener); } diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java index fbfed8a75a146..30fea41330038 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java @@ -954,7 +954,7 @@ private void notifyFailureOnceAllOutstandingRequestAreDone(Exception e) { void createRetentionLease(final long startingSeqNo, ActionListener listener) { updateRetentionLease(syncListener -> { - // Clone the peer recovery retention lease belonging to the source shard. We are retaining history between the the local + // Clone the peer recovery retention lease belonging to the source shard. We are retaining history between the local // checkpoint of the safe commit we're creating and this lease's retained seqno with the retention lock, and by cloning an // existing lease we (approximately) know that all our peers are also retaining history as requested by the cloned lease. If // the recovery now fails before copying enough history over then a subsequent attempt will find this lease, determine it is diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 435bf71e3b2c9..b43fe05a541f6 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -435,7 +435,7 @@ public static String getRepositoryDataBlobName(long repositoryGeneration) { /** * Flag that is set to {@code true} if this instance is started with {@link #metadata} that has a higher value for * {@link RepositoryMetadata#pendingGeneration()} than for {@link RepositoryMetadata#generation()} indicating a full cluster restart - * potentially accounting for the the last {@code index-N} write in the cluster state. + * potentially accounting for the last {@code index-N} write in the cluster state. * Note: While it is true that this value could also be set to {@code true} for an instance on a node that is just joining the cluster * during a new {@code index-N} write, this does not present a problem. The node will still load the correct {@link RepositoryData} in * all cases and simply do a redundant listing of the repository contents if it tries to load {@link RepositoryData} and falls back diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/ChunkedBlobOutputStream.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/ChunkedBlobOutputStream.java index bcbbad81f1ca1..0e2066f58e25d 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/ChunkedBlobOutputStream.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/ChunkedBlobOutputStream.java @@ -128,7 +128,7 @@ protected final void finishPart(T partId) { } /** - * Write the contents of {@link #buffer} to storage. Implementations should call {@link #finishPart} at the end to track the the chunk + * Write the contents of {@link #buffer} to storage. Implementations should call {@link #finishPart} at the end to track the chunk * of data just written and ready {@link #buffer} for the next write. */ protected abstract void flushBuffer() throws IOException; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterByFilterAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterByFilterAggregator.java index 8ebf55487ed59..cc8ecc74b5ea0 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterByFilterAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterByFilterAggregator.java @@ -132,7 +132,7 @@ final void add(QueryToFilterAdapter filter) throws IOException { } /** - * Build the the adapter or {@code null} if the this isn't a valid rewrite. + * Build the adapter or {@code null} if this isn't a valid rewrite. */ public final T build() throws IOException { if (false == valid || aggCtx.enableRewriteToFilterByFilter() == false) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorFactory.java index 1ff7529bf3188..080cac9cbfb85 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorFactory.java @@ -123,7 +123,7 @@ private static SignificantTermsAggregatorSupplier bytesSupplier() { /** * Whether the aggregation will execute. If the main query matches no documents and parent aggregation isn't a global or terms - * aggregation with min_doc_count = 0, the the aggregator will not really execute. In those cases it doesn't make sense to load + * aggregation with min_doc_count = 0, the aggregator will not really execute. In those cases it doesn't make sense to load * global ordinals. *

    * Some searches that will never match can still fall through and we endup running query that will produce no results. diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java index 4f939ea294e48..c720f3d9465a3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java @@ -556,7 +556,7 @@ public synchronized void removeReleasable(Aggregator aggregator) { // Removing an aggregator is done after calling Aggregator#buildTopLevel which happens on an executor thread. // We need to synchronize the removal because he AggregatorContext it is shared between executor threads. assert releaseMe.contains(aggregator) - : "removing non-existing aggregator [" + aggregator.name() + "] from the the aggregation context"; + : "removing non-existing aggregator [" + aggregator.name() + "] from the aggregation context"; releaseMe.remove(aggregator); } diff --git a/server/src/main/java/org/elasticsearch/snapshots/package-info.java b/server/src/main/java/org/elasticsearch/snapshots/package-info.java index 694c9c5c9062b..d73a1d9bd701a 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/package-info.java +++ b/server/src/main/java/org/elasticsearch/snapshots/package-info.java @@ -113,7 +113,7 @@ * snapshots, we load the {@link org.elasticsearch.snapshots.SnapshotInfo} for the source snapshot and check for shard snapshot * failures of the relevant indices. *

  • Once all shard counts are known and the health of all source indices data has been verified, we populate the - * {@code SnapshotsInProgress.Entry#clones} map for the clone operation with the the relevant shard clone tasks.
  • + * {@code SnapshotsInProgress.Entry#clones} map for the clone operation with the relevant shard clone tasks. *
  • After the clone tasks have been added to the {@code SnapshotsInProgress.Entry}, master executes them on its snapshot thread-pool * by invoking {@link org.elasticsearch.repositories.Repository#cloneShardSnapshot} for each shard that is to be cloned. Each completed * shard snapshot triggers a call to the {@link org.elasticsearch.snapshots.SnapshotsService#masterServiceTaskQueue} which updates the diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfoTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfoTests.java index c0bf8f7c3bf12..5fa138abca809 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfoTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfoTests.java @@ -33,7 +33,7 @@ public class NodeInfoTests extends ESTestCase { /** - * Check that the the {@link NodeInfo#getInfo(Class)} method returns null + * Check that the {@link NodeInfo#getInfo(Class)} method returns null * for absent info objects, and returns the right thing for present info * objects. */ diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinReasonServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinReasonServiceTests.java index 80aa142069358..dc46f862923e5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinReasonServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinReasonServiceTests.java @@ -144,7 +144,7 @@ public void testCleanup() { discoveryNodes[i] = randomDiscoveryNode(); } - // we stop tracking the the oldest absent node(s) when only 1/3 of the tracked nodes are present + // we stop tracking the oldest absent node(s) when only 1/3 of the tracked nodes are present final int cleanupNodeCount = (discoveryNodes.length - 2) / 3; final DiscoveryNodes.Builder cleanupNodesBuilder = new DiscoveryNodes.Builder().add(masterNode) diff --git a/server/src/test/java/org/elasticsearch/common/LocalTimeOffsetTests.java b/server/src/test/java/org/elasticsearch/common/LocalTimeOffsetTests.java index 9711fb2c0f7fd..e3af6695bde1d 100644 --- a/server/src/test/java/org/elasticsearch/common/LocalTimeOffsetTests.java +++ b/server/src/test/java/org/elasticsearch/common/LocalTimeOffsetTests.java @@ -275,7 +275,7 @@ private static long time(String time, ZoneId zone) { } /** - * The the last "fully defined" transitions in the provided {@linkplain ZoneId}. + * The last "fully defined" transitions in the provided {@linkplain ZoneId}. */ private static ZoneOffsetTransition lastTransitionIn(ZoneId zone) { List transitions = zone.getRules().getTransitions(); diff --git a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java index 610e87b50d365..0a8fbcf6d56b9 100644 --- a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java @@ -747,7 +747,7 @@ public void testAllocationBucketsBreaker() { // make sure used bytes is greater than the total circuit breaker limit breaker.addWithoutBreaking(200); - // make sure that we check on the the following call + // make sure that we check on the following call for (int i = 0; i < 1023; i++) { multiBucketConsumer.accept(0); } diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/AbstractNXYSignificanceHeuristicTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/AbstractNXYSignificanceHeuristicTestCase.java index adb9c6f1e4ca0..5e82cb7edfeac 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/AbstractNXYSignificanceHeuristicTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/AbstractNXYSignificanceHeuristicTestCase.java @@ -27,7 +27,7 @@ protected SignificanceHeuristic getHeuristic() { /** * @param includeNegatives value for this test run, should the scores include negative values. * @param backgroundIsSuperset value for this test run, indicates in NXY significant terms if the background is indeed - * a superset of the the subset, or is instead a disjoint set + * a superset of the subset, or is instead a disjoint set * @return A random instance of an NXY heuristic to test */ protected abstract SignificanceHeuristic getHeuristic(boolean includeNegatives, boolean backgroundIsSuperset); diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java index 075621e8cdccb..a253b6bdd2360 100644 --- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java +++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java @@ -64,7 +64,7 @@ public BlobCacheMetrics(MeterRegistry meterRegistry) { ), meterRegistry.registerDoubleHistogram( "es.blob_cache.population.throughput.histogram", - "The throughput observed when populating the the cache", + "The throughput observed when populating the cache", "MiB/second" ), meterRegistry.registerLongCounter( diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 1b1eff8a07b1d..81739536c6572 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -370,6 +370,9 @@ public static boolean isUnsupported(DataType from) { } public static boolean isString(DataType t) { + if (EsqlCorePlugin.SEMANTIC_TEXT_FEATURE_FLAG.isEnabled() && t == SEMANTIC_TEXT) { + return true; + } return t == KEYWORD || t == TEXT; } @@ -585,7 +588,7 @@ static Builder builder() { } public DataType noText() { - return this == TEXT ? KEYWORD : this; + return isString(this) ? KEYWORD : this; } /** diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/async/AsyncTaskManagementServiceTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/async/AsyncTaskManagementServiceTests.java index 5361f1e8d1974..2e71c2c0a1ad3 100644 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/async/AsyncTaskManagementServiceTests.java +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/async/AsyncTaskManagementServiceTests.java @@ -313,7 +313,7 @@ public void execute(TestRequest request, TestTask task, ActionListener response = getResponse(responseHolder.get().id, TimeValue.ZERO); if (success) { assertThat(response.getException(), nullValue()); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json index b110d6fd4cdd5..c587b69828170 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json @@ -68,6 +68,10 @@ }, "value": { "type": "long" + }, + "st_base64": { + "type": "semantic_text", + "inference_id": "test_sparse_inference" } } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv index c6de9a208e9a7..6cae82cfefa0a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv @@ -1,4 +1,4 @@ -_id:keyword,semantic_text_field:semantic_text,st_bool:semantic_text,st_cartesian_point:semantic_text,st_cartesian_shape:semantic_text,st_datetime:semantic_text,st_double:semantic_text,st_geopoint:semantic_text,st_geoshape:semantic_text,st_integer:semantic_text,st_ip:semantic_text,st_long:semantic_text,st_unsigned_long:semantic_text,st_version:semantic_text,st_multi_value:semantic_text,st_unicode:semantic_text,host:keyword,description:text,value:long -1,live long and prosper,false,"POINT(4297.11 -1475.53)",,1953-09-02T00:00:00.000Z,5.20128E11,"POINT(42.97109630194 14.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",23,1.1.1.1,2147483648,2147483648,1.2.3,["Hello there!", "This is a random value", "for testing purposes"],你吃饭了吗,"host1","some description1",1001 -2,all we have to decide is what to do with the time that is given to us,true,"POINT(7580.93 2272.77)",,2023-09-24T15:57:00.000Z,4541.11,"POINT(37.97109630194 21.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",122,1.1.2.1,123,2147483648.2,9.0.0,["nice to meet you", "bye bye!"],["谢谢", "对不起我的中文不好"],"host2","some description2",1002 -3,be excellent to each other,,,,,,,,,,,,,,,"host3","some description3",1003 +_id:keyword,semantic_text_field:semantic_text,st_bool:semantic_text,st_cartesian_point:semantic_text,st_cartesian_shape:semantic_text,st_datetime:semantic_text,st_double:semantic_text,st_geopoint:semantic_text,st_geoshape:semantic_text,st_integer:semantic_text,st_ip:semantic_text,st_long:semantic_text,st_unsigned_long:semantic_text,st_version:semantic_text,st_multi_value:semantic_text,st_unicode:semantic_text,host:keyword,description:text,value:long,st_base64:semantic_text +1,live long and prosper,false,"POINT(4297.11 -1475.53)",,1953-09-02T00:00:00.000Z,5.20128E11,"POINT(42.97109630194 14.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",23,1.1.1.1,2147483648,2147483648,1.2.3,["Hello there!", "This is a random value", "for testing purposes"],你吃饭了吗,"host1","some description1",1001,ZWxhc3RpYw== +2,all we have to decide is what to do with the time that is given to us,true,"POINT(7580.93 2272.77)",,2023-09-24T15:57:00.000Z,4541.11,"POINT(37.97109630194 21.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",122,1.1.2.1,123,2147483648.2,9.0.0,["nice to meet you", "bye bye!"],["谢谢", "对不起我的中文不好"],"host2","some description2",1002,aGVsbG8= +3,be excellent to each other,,,,,,,,,,,,,,,"host3","some description3",1003, diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec index 683bcdc3f7490..de2a79df06a50 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec @@ -173,3 +173,946 @@ host:keyword | semantic_text_field:semantic_text "host2" | all we have to decide is what to do with the time that is given to us "host3" | be excellent to each other ; + +case +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = case(st_ip == "1.1.1.1", "okay", "try again") +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:keyword +1 | okay +2 | try again +3 | try again +; + +coalesce +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = coalesce(st_version, st_ip, semantic_text_field) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:keyword +1 | 1.2.3 +2 | 9.0.0 +3 | be excellent to each other +; + +greatest +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = greatest(semantic_text_field, st_version) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:keyword +1 | live long and prosper +2 | all we have to decide is what to do with the time that is given to us +3 | null +; + +least +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = least(semantic_text_field, st_version) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:keyword +1 | 1.2.3 +2 | 9.0.0 +3 | null +; + +convertToBool +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_bool(st_bool) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:bool +1 | false +2 | true +3 | null +; + +convertToCartesianPoint +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_cartesianpoint(st_cartesian_point) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:cartesian_point +1 | "POINT(4297.11 -1475.53)" +2 | "POINT(7580.93 2272.77)" +3 | null +; + +convertToCartesianShape +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_cartesianshape(st_cartesian_shape) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:cartesian_shape +1 | null +2 | null +3 | null +; + +convertToDatetime +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_datetime(st_datetime) +| KEEP _id, result +| SORT _id, result +; + +_id:keyword|result:datetime +1 | 1953-09-02T00:00:00.000Z +2 | 2023-09-24T15:57:00.000Z +3 | null +; + +convertToDouble +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_double(st_double) +| KEEP _id, result +| SORT _id +; + +_id:keyword|result:double +1 | 5.20128E11 +2 | 4541.11 +3 | null +; + +convertToGeopoint +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_geopoint(st_geopoint) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:geo_point +1 | "POINT(42.97109630194 14.7552534413725)" +2 | "POINT(37.97109630194 21.7552534413725)" +3 | null +; + +convertToGeoshape +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_geoshape(st_geoshape) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:geo_shape +1 | "POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))" +2 | "POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))" +3 | null +; + +convertToInteger +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_integer(st_integer) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:integer +1 | 23 +2 | 122 +3 | null +; + +convertToIp +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_ip(st_ip) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:ip +1 | 1.1.1.1 +2 | 1.1.2.1 +3 | null +; + +convertToLong +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_long(st_long) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:long +1 | 2147483648 +2 | 123 +3 | null +; + +convertToUnsignedLong +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_unsigned_long(st_unsigned_long) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:unsigned_long +1 | 2147483648 +2 | 2147483648.2 +3 | null +; + +convertToVersion +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_version(st_version) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:version +1 | 1.2.3 +2 | 9.0.0 +3 | null +; + +concat +required_capability: semantic_text_type + +FROM semantic_text +| EVAL result = concat("", semantic_text_field, "") +| KEEP result +| SORT result +; + +result:keyword +all we have to decide is what to do with the time that is given to us +be excellent to each other +live long and prosper +; + +endsWith +required_capability: semantic_text_type + +FROM semantic_text +| WHERE ends_with(semantic_text_field, "er") +| KEEP semantic_text_field +| SORT semantic_text_field +; + +semantic_text_field:semantic_text +be excellent to each other +live long and prosper +; + +fromBase64 +required_capability: semantic_text_type +FROM semantic_text +| EVAL result = from_base64(st_base64) +| SORT result +| KEEP result +; + +result:keyword +elastic +hello +null +; + +left +required_capability: semantic_text_type + +FROM semantic_text +| EVAL result = left(semantic_text_field, 2) +| SORT result +| KEEP result +; + +result:keyword +al +be +li +; + +length +required_capability: semantic_text_type + +FROM semantic_text +| EVAL result = length(st_version) +| KEEP result +| SORT result +; + +result:integer +5 +5 +null +; + +locate +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = locate(semantic_text_field, "all") +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:integer +1 | 0 +2 | 1 +3 | 0 +; + +ltrim +required_capability: semantic_text_type + +FROM semantic_text +| EVAL result = ltrim(semantic_text_field) +| SORT result +| KEEP result +; + +result:keyword +all we have to decide is what to do with the time that is given to us +be excellent to each other +live long and prosper +; + +repeat +required_capability: semantic_text_type + +FROM semantic_text +| EVAL result = repeat(semantic_text_field, 2) +| WHERE length(semantic_text_field) < 25 +| KEEP result +; + +result:keyword +live long and prosperlive long and prosper +; + +replace +required_capability: semantic_text_type + +FROM semantic_text +| EVAL result = replace(semantic_text_field, "excellent", "good") +| WHERE length(semantic_text_field) < 30 +| KEEP result +| SORT result +; + +result:keyword +be good to each other +live long and prosper +; + +right +required_capability: semantic_text_type + +FROM semantic_text +| EVAL result = right(semantic_text_field, 2) +| KEEP result +| SORT result +; + +result:keyword +er +er +us +; + +rtrim +required_capability: semantic_text_type + +FROM semantic_text +| EVAL result = rtrim(semantic_text_field) +| KEEP result +| SORT result +; + +result:keyword +all we have to decide is what to do with the time that is given to us +be excellent to each other +live long and prosper +; + +split +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = split(st_version, ".") +| SORT _id +| KEEP result +; + +result:keyword +["1", "2", "3"] +["9", "0", "0"] +null +; + +startsWith +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = starts_with(semantic_text_field, "be") +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:bool +1 | false +2 | false +3 | true +; + +substring +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = substring(semantic_text_field, 2, 1) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:keyword +1 | i +2 | l +3 | e +; + +toBase64 +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_base64(st_integer) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:keyword +1 | MjM= +2 | MTIy +3 | null +; + +toLower +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_lower(st_cartesian_point) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:keyword +1 | point(4297.11 -1475.53) +2 | point(7580.93 2272.77) +3 | null +; + +toUpper +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_upper(semantic_text_field) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:keyword +1 | LIVE LONG AND PROSPER +2 | ALL WE HAVE TO DECIDE IS WHAT TO DO WITH THE TIME THAT IS GIVEN TO US +3 | BE EXCELLENT TO EACH OTHER +; + +trim +required_capability: semantic_text_type + +FROM semantic_text +| EVAL result = trim(semantic_text_field) +| SORT result +| KEEP result +; + +result:keyword +all we have to decide is what to do with the time that is given to us +be excellent to each other +live long and prosper +; + +mvAppend +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = mv_append(st_multi_value, st_long) +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:keyword +1 | ["Hello there!", "This is a random value", "for testing purposes", "2147483648"] +2 | ["nice to meet you", "bye bye!", "123"] +3 | null +; + +mvConcat +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = mv_concat(st_multi_value, "; ") +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:keyword +1 | Hello there!; This is a random value; for testing purposes +2 | nice to meet you; bye bye! +3 | null +; + +mvCount +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = mv_count(st_multi_value) +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:integer +1 | 3 +2 | 2 +3 | null +; + +mvDedupe +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = mv_dedupe(st_multi_value) +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:keyword +1 | ["Hello there!", "This is a random value", "for testing purposes"] +2 | ["nice to meet you", "bye bye!"] +3 | null +; + +mvFirst +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = mv_first(st_multi_value) +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:keyword +1 | Hello there! +2 | nice to meet you +3 | null +; + +mvLast +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = mv_last(st_multi_value) +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:keyword +1 | for testing purposes +2 | bye bye! +3 | null +; + +mvMax +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = mv_max(st_multi_value) +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:keyword +1 | for testing purposes +2 | nice to meet you +3 | null +; + +mvMin +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = mv_min(st_multi_value) +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:keyword +1 | Hello there! +2 | bye bye! +3 | null +; + +mvSlice +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = mv_slice(st_multi_value, 1, 2) +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:keyword +1 | ["This is a random value", "for testing purposes"] +2 | bye bye! +3 | null +; + +mvSort +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = mv_sort(st_multi_value, "ASC") +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:keyword +1 | ["Hello there!", "This is a random value", "for testing purposes"] +2 | ["bye bye!", "nice to meet you"] +3 | null +; + +mvZip +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = mv_zip(st_multi_value, st_multi_value, " + ") +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:keyword +1 | ["Hello there! + Hello there!", "This is a random value + This is a random value", "for testing purposes + for testing purposes"] +2 | ["nice to meet you + nice to meet you", "bye bye! + bye bye!"] +3 | null +; + +equalityWithConstant +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_ip == "1.1.1.1" +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | true +2 | false +3 | null +; + +equalityBetweenFields +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_long == st_unsigned_long +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | true +2 | false +3 | null +; + +inequalityWithConstant +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_ip != "1.1.1.1" +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | false +2 | true +3 | null +; + +inequalityBetweenFields +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_long != st_unsigned_long +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | false +2 | true +3 | null +; + +lessThanWithConstant +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = semantic_text_field < "bye!" +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | false +2 | true +3 | true +; + +lessThanBetweenFields +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = semantic_text_field < st_version +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | false +2 | false +3 | null +; + + +lessThanOrEqualToWithConstant +required_capability: semantic_text_type + + +FROM semantic_text METADATA _id +| EVAL result = semantic_text_field <= "be excellent to each other" +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | false +2 | true +3 | true +; + +lessThanOrEqualToBetweenFields +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_integer <= st_long +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | false +2 | true +3 | null +; + +greaterThanWithConstant +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = semantic_text_field > "bye!" +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | true +2 | false +3 | false +; + +greaterThanBetweenFields +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = semantic_text_field > st_version +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | true +2 | true +3 | null +; + +greaterThanOrEqualToWithConstant +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = semantic_text_field >= "be excellent to each other" +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | true +2 | false +3 | true +; + +greaterThanOrEqualToBetweenFields +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_integer >= st_long +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | true +2 | false +3 | null +; + +isNull +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_integer IS NULL +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | false +2 | false +3 | true +; + +isNotNull +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_integer IS NOT NULL +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | true +2 | true +3 | false +; + +cast +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_bool::BOOL +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:bool +1 | false +2 | true +3 | null +; + +in +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_integer IN ("123", "23") +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | true +2 | false +3 | null +; + +like +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = semantic_text_field LIKE "all*" +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | false +2 | true +3 | false +; + +rlike +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_version RLIKE "[0-9].[0-9].[0-9]" +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | true +2 | true +3 | null +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index fbaf43467a2e7..994ea3ecdbb0d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.analysis; import org.elasticsearch.common.logging.LoggerMessageFormat; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable; import org.elasticsearch.xpack.esql.core.expression.Alias; @@ -505,6 +506,9 @@ public static Failure validateBinaryComparison(BinaryComparison bc) { List allowed = new ArrayList<>(); allowed.add(DataType.KEYWORD); allowed.add(DataType.TEXT); + if (EsqlCapabilities.Cap.SEMANTIC_TEXT_TYPE.isEnabled()) { + allowed.add(DataType.SEMANTIC_TEXT); + } allowed.add(DataType.IP); allowed.add(DataType.DATETIME); allowed.add(DataType.VERSION); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java index 06cc993456433..ad73de7829692 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java @@ -28,6 +28,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToBoolean; @@ -44,6 +45,7 @@ public class ToBoolean extends AbstractConvertFunction { Map.entry(BOOLEAN, (field, source) -> field), Map.entry(KEYWORD, ToBooleanFromStringEvaluator.Factory::new), Map.entry(TEXT, ToBooleanFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToBooleanFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToBooleanFromDoubleEvaluator.Factory::new), Map.entry(LONG, ToBooleanFromLongEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToBooleanFromUnsignedLongEvaluator.Factory::new), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java index 60a25fc91d50d..92ae2cd0ade52 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java @@ -25,6 +25,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial; @@ -38,7 +39,8 @@ public class ToCartesianPoint extends AbstractConvertFunction { private static final Map EVALUATORS = Map.ofEntries( Map.entry(CARTESIAN_POINT, (fieldEval, source) -> fieldEval), Map.entry(KEYWORD, ToCartesianPointFromStringEvaluator.Factory::new), - Map.entry(TEXT, ToCartesianPointFromStringEvaluator.Factory::new) + Map.entry(TEXT, ToCartesianPointFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToCartesianPointFromStringEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java index 03ac4bdf48243..83e66e9e3190f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java @@ -26,6 +26,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT; import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial; @@ -40,7 +41,8 @@ public class ToCartesianShape extends AbstractConvertFunction { Map.entry(CARTESIAN_POINT, (fieldEval, source) -> fieldEval), Map.entry(CARTESIAN_SHAPE, (fieldEval, source) -> fieldEval), Map.entry(KEYWORD, ToCartesianShapeFromStringEvaluator.Factory::new), - Map.entry(TEXT, ToCartesianShapeFromStringEvaluator.Factory::new) + Map.entry(TEXT, ToCartesianShapeFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToCartesianShapeFromStringEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java index 9a6a91b7ccedd..8c4375b424cdc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java @@ -32,6 +32,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_NANOS_FORMATTER; @@ -49,6 +50,7 @@ public class ToDateNanos extends AbstractConvertFunction { Map.entry(LONG, ToDateNanosFromLongEvaluator.Factory::new), Map.entry(KEYWORD, ToDateNanosFromStringEvaluator.Factory::new), Map.entry(TEXT, ToDateNanosFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToDateNanosFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToDateNanosFromDoubleEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToLongFromUnsignedLongEvaluator.Factory::new) /* diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java index c66ba7f87a1c5..f8fe663b9086c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java @@ -30,6 +30,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToLong; @@ -47,6 +48,7 @@ public class ToDatetime extends AbstractConvertFunction { Map.entry(LONG, (field, source) -> field), Map.entry(KEYWORD, ToDatetimeFromStringEvaluator.Factory::new), Map.entry(TEXT, ToDatetimeFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToDatetimeFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToLongFromDoubleEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToLongFromUnsignedLongEvaluator.Factory::new), Map.entry(INTEGER, ToLongFromIntEvaluator.Factory::new) // CastIntToLongEvaluator would be a candidate, but not MV'd diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java index de88281e7dbd1..67b7af73576eb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java @@ -30,6 +30,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToDouble; @@ -44,6 +45,7 @@ public class ToDouble extends AbstractConvertFunction { Map.entry(DATETIME, ToDoubleFromLongEvaluator.Factory::new), // CastLongToDoubleEvaluator would be a candidate, but not MV'd Map.entry(KEYWORD, ToDoubleFromStringEvaluator.Factory::new), Map.entry(TEXT, ToDoubleFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToDoubleFromStringEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToDoubleFromUnsignedLongEvaluator.Factory::new), Map.entry(LONG, ToDoubleFromLongEvaluator.Factory::new), // CastLongToDoubleEvaluator would be a candidate, but not MV'd Map.entry(INTEGER, ToDoubleFromIntEvaluator.Factory::new), // CastIntToDoubleEvaluator would be a candidate, but not MV'd diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java index 51cb08137a58c..42af06a40553d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java @@ -25,6 +25,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial; @@ -38,7 +39,8 @@ public class ToGeoPoint extends AbstractConvertFunction { private static final Map EVALUATORS = Map.ofEntries( Map.entry(GEO_POINT, (fieldEval, source) -> fieldEval), Map.entry(KEYWORD, ToGeoPointFromStringEvaluator.Factory::new), - Map.entry(TEXT, ToGeoPointFromStringEvaluator.Factory::new) + Map.entry(TEXT, ToGeoPointFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToGeoPointFromStringEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java index 00e9fb3e598f1..b5b6db2752b06 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java @@ -26,6 +26,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial; @@ -40,7 +41,8 @@ public class ToGeoShape extends AbstractConvertFunction { Map.entry(GEO_POINT, (fieldEval, source) -> fieldEval), Map.entry(GEO_SHAPE, (fieldEval, source) -> fieldEval), Map.entry(KEYWORD, ToGeoShapeFromStringEvaluator.Factory::new), - Map.entry(TEXT, ToGeoShapeFromStringEvaluator.Factory::new) + Map.entry(TEXT, ToGeoShapeFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToGeoShapeFromStringEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java index 6df85948d94ef..cd161744bfc86 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java @@ -25,6 +25,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.IP; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToIP; @@ -34,7 +35,8 @@ public class ToIP extends AbstractConvertFunction { private static final Map EVALUATORS = Map.ofEntries( Map.entry(IP, (field, source) -> field), Map.entry(KEYWORD, ToIPFromStringEvaluator.Factory::new), - Map.entry(TEXT, ToIPFromStringEvaluator.Factory::new) + Map.entry(TEXT, ToIPFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToIPFromStringEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java index 1785160594a78..d316b6eb46c38 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java @@ -25,11 +25,13 @@ import java.util.Map; import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; +import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.safeToInt; @@ -49,10 +51,11 @@ public class ToInteger extends AbstractConvertFunction { Map.entry(DATETIME, ToIntegerFromLongEvaluator.Factory::new), Map.entry(KEYWORD, ToIntegerFromStringEvaluator.Factory::new), Map.entry(TEXT, ToIntegerFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToIntegerFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToIntegerFromDoubleEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToIntegerFromUnsignedLongEvaluator.Factory::new), Map.entry(LONG, ToIntegerFromLongEvaluator.Factory::new), - Map.entry(DataType.COUNTER_INTEGER, (fieldEval, source) -> fieldEval) + Map.entry(COUNTER_INTEGER, (fieldEval, source) -> fieldEval) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java index e5f138df159cd..dbfb52b408b44 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java @@ -31,6 +31,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.safeDoubleToLong; @@ -47,6 +48,7 @@ public class ToLong extends AbstractConvertFunction { Map.entry(BOOLEAN, ToLongFromBooleanEvaluator.Factory::new), Map.entry(KEYWORD, ToLongFromStringEvaluator.Factory::new), Map.entry(TEXT, ToLongFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToLongFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToLongFromDoubleEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToLongFromUnsignedLongEvaluator.Factory::new), Map.entry(INTEGER, ToLongFromIntEvaluator.Factory::new), // CastIntToLongEvaluator would be a candidate, but not MV'd diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java index f9bc15c4d6903..2c8ecd794ef0b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java @@ -36,6 +36,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.IP; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; @@ -60,6 +61,7 @@ public class ToString extends AbstractConvertFunction implements EvaluatorMapper Map.entry(LONG, ToStringFromLongEvaluator.Factory::new), Map.entry(INTEGER, ToStringFromIntEvaluator.Factory::new), Map.entry(TEXT, (fieldEval, source) -> fieldEval), + Map.entry(SEMANTIC_TEXT, (fieldEval, source) -> fieldEval), Map.entry(VERSION, ToStringFromVersionEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToStringFromUnsignedLongEvaluator.Factory::new), Map.entry(GEO_POINT, ToStringFromGeoPointEvaluator.Factory::new), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java index bfbfcf44b3945..ea06793f7adb6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java @@ -30,6 +30,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.booleanToUnsignedLong; @@ -51,6 +52,7 @@ public class ToUnsignedLong extends AbstractConvertFunction { Map.entry(BOOLEAN, ToUnsignedLongFromBooleanEvaluator.Factory::new), Map.entry(KEYWORD, ToUnsignedLongFromStringEvaluator.Factory::new), Map.entry(TEXT, ToUnsignedLongFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToUnsignedLongFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToUnsignedLongFromDoubleEvaluator.Factory::new), Map.entry(LONG, ToUnsignedLongFromLongEvaluator.Factory::new), Map.entry(INTEGER, ToUnsignedLongFromIntEvaluator.Factory::new) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java index f6002c3c6bb17..296ddb35c3c41 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java @@ -24,6 +24,7 @@ import java.util.Map; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToVersion; @@ -38,7 +39,8 @@ public class ToVersion extends AbstractConvertFunction { private static final Map EVALUATORS = Map.ofEntries( Map.entry(VERSION, (fieldEval, source) -> fieldEval), Map.entry(KEYWORD, ToVersionFromStringEvaluator.Factory::new), - Map.entry(TEXT, ToVersionFromStringEvaluator.Factory::new) + Map.entry(TEXT, ToVersionFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToVersionFromStringEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java index 614d9aa3ec920..6bb249385affe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java @@ -42,6 +42,7 @@ public class Equals extends EsqlBinaryComparison implements Negatable parameters() { builder.expectFlattenedInt(IntStream::max); builder.expectFlattenedLong(LongStream::max); List suppliers = builder.suppliers(); - suppliers.add( - new TestCaseSupplier( - "(a, b)", - List.of(DataType.KEYWORD, DataType.KEYWORD), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("a"), DataType.KEYWORD, "a"), - new TestCaseSupplier.TypedData(new BytesRef("b"), DataType.KEYWORD, "b") - ), - "GreatestBytesRefEvaluator[values=[MvMax[field=Attribute[channel=0]], MvMax[field=Attribute[channel=1]]]]", - DataType.KEYWORD, - equalTo(new BytesRef("b")) + for (DataType stringType : DataType.stringTypes()) { + suppliers.add( + new TestCaseSupplier( + "(a, b)", + List.of(stringType, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("a"), stringType, "a"), + new TestCaseSupplier.TypedData(new BytesRef("b"), stringType, "b") + ), + "GreatestBytesRefEvaluator[values=[MvMax[field=Attribute[channel=0]], MvMax[field=Attribute[channel=1]]]]", + stringType, + equalTo(new BytesRef("b")) + ) ) - ) - ); + ); + } suppliers.add( new TestCaseSupplier( "(a, b)", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java index d95cc79dd22e0..3b24a4cbdc1eb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java @@ -39,21 +39,23 @@ public static Iterable parameters() { builder.expectFlattenedInt(IntStream::min); builder.expectFlattenedLong(LongStream::min); List suppliers = builder.suppliers(); - suppliers.add( - new TestCaseSupplier( - "(a, b)", - List.of(DataType.KEYWORD, DataType.KEYWORD), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("a"), DataType.KEYWORD, "a"), - new TestCaseSupplier.TypedData(new BytesRef("b"), DataType.KEYWORD, "b") - ), - "LeastBytesRefEvaluator[values=[MvMin[field=Attribute[channel=0]], MvMin[field=Attribute[channel=1]]]]", - DataType.KEYWORD, - equalTo(new BytesRef("a")) + for (DataType stringType : DataType.stringTypes()) { + suppliers.add( + new TestCaseSupplier( + "(a, b)", + List.of(stringType, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("a"), stringType, "a"), + new TestCaseSupplier.TypedData(new BytesRef("b"), stringType, "b") + ), + "LeastBytesRefEvaluator[values=[MvMin[field=Attribute[channel=0]], MvMin[field=Attribute[channel=1]]]]", + stringType, + equalTo(new BytesRef("a")) + ) ) - ) - ); + ); + } suppliers.add( new TestCaseSupplier( "(a, b)", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java index f472e5ef5efd9..60901e2a8214f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java @@ -35,25 +35,17 @@ public FromBase64Tests(@Name("TestCase") Supplier tes @ParametersFactory public static Iterable parameters() { List suppliers = new ArrayList<>(); - suppliers.add(new TestCaseSupplier(List.of(DataType.KEYWORD), () -> { - BytesRef input = new BytesRef(randomAlphaOfLength(6)); - return new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(input, DataType.KEYWORD, "string")), - "FromBase64Evaluator[field=Attribute[channel=0]]", - DataType.KEYWORD, - equalTo(new BytesRef(Base64.getDecoder().decode(input.utf8ToString().getBytes(StandardCharsets.UTF_8)))) - ); - })); - - suppliers.add(new TestCaseSupplier(List.of(DataType.TEXT), () -> { - BytesRef input = new BytesRef(randomAlphaOfLength(54)); - return new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(input, DataType.TEXT, "string")), - "FromBase64Evaluator[field=Attribute[channel=0]]", - DataType.KEYWORD, - equalTo(new BytesRef(Base64.getDecoder().decode(input.utf8ToString().getBytes(StandardCharsets.UTF_8)))) - ); - })); + for (DataType dataType : DataType.stringTypes()) { + suppliers.add(new TestCaseSupplier(List.of(dataType), () -> { + BytesRef input = new BytesRef(randomAlphaOfLength(54)); + return new TestCaseSupplier.TestCase( + List.of(new TestCaseSupplier.TypedData(input, dataType, "string")), + "FromBase64Evaluator[field=Attribute[channel=0]]", + DataType.KEYWORD, + equalTo(new BytesRef(Base64.getDecoder().decode(input.utf8ToString().getBytes(StandardCharsets.UTF_8)))) + ); + })); + } return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java index 06b5af8d7067b..6e6ff7bf52fce 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java @@ -36,25 +36,17 @@ public ToBase64Tests(@Name("TestCase") Supplier testC @ParametersFactory public static Iterable parameters() { List suppliers = new ArrayList<>(); - suppliers.add(new TestCaseSupplier(List.of(DataType.KEYWORD), () -> { - BytesRef input = (BytesRef) randomLiteral(DataType.KEYWORD).value(); - return new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(input, DataType.KEYWORD, "string")), - "ToBase64Evaluator[field=Attribute[channel=0]]", - DataType.KEYWORD, - equalTo(new BytesRef(Base64.getEncoder().encode(input.utf8ToString().getBytes(StandardCharsets.UTF_8)))) - ); - })); - - suppliers.add(new TestCaseSupplier(List.of(DataType.TEXT), () -> { - BytesRef input = (BytesRef) randomLiteral(DataType.TEXT).value(); - return new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(input, DataType.TEXT, "string")), - "ToBase64Evaluator[field=Attribute[channel=0]]", - DataType.KEYWORD, - equalTo(new BytesRef(Base64.getEncoder().encode(input.utf8ToString().getBytes(StandardCharsets.UTF_8)))) - ); - })); + for (DataType dataType : DataType.stringTypes()) { + suppliers.add(new TestCaseSupplier(List.of(dataType), () -> { + BytesRef input = (BytesRef) randomLiteral(dataType).value(); + return new TestCaseSupplier.TestCase( + List.of(new TestCaseSupplier.TypedData(input, dataType, "string")), + "ToBase64Evaluator[field=Attribute[channel=0]]", + DataType.KEYWORD, + equalTo(new BytesRef(Base64.getEncoder().encode(input.utf8ToString().getBytes(StandardCharsets.UTF_8)))) + ); + })); + } return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java index 13920fac26f5b..65f5653f27e1a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java @@ -123,7 +123,7 @@ protected static void bytesRefs( Function expectedDataType, BiFunction, Matcher> matcher ) { - for (DataType type : new DataType[] { DataType.KEYWORD, DataType.TEXT, DataType.IP, DataType.VERSION }) { + for (DataType type : new DataType[] { DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT, DataType.IP, DataType.VERSION }) { if (type != DataType.IP) { cases.add( new TestCaseSupplier( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java index 37f4464c8b3ca..33733d5e70c61 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java @@ -171,6 +171,22 @@ private static void bytesRefs(List suppliers) { ); })); + suppliers.add(new TestCaseSupplier(List.of(DataType.SEMANTIC_TEXT, DataType.SEMANTIC_TEXT), () -> { + List field1 = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); + List field2 = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.SEMANTIC_TEXT, "field1"), + new TestCaseSupplier.TypedData(field2, DataType.SEMANTIC_TEXT, "field2") + ), + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataType.SEMANTIC_TEXT, + equalTo(result) + ); + })); + suppliers.add(new TestCaseSupplier(List.of(DataType.IP, DataType.IP), () -> { List field1 = randomList(1, 10, () -> randomLiteral(DataType.IP).value()); List field2 = randomList(1, 10, () -> randomLiteral(DataType.IP).value()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java index 859c79090d62f..d5284602bf40c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java @@ -255,6 +255,23 @@ private static void bytesRefs(List suppliers) { ); })); + suppliers.add(new TestCaseSupplier(List.of(DataType.SEMANTIC_TEXT, DataType.INTEGER, DataType.INTEGER), () -> { + List field = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); + int length = field.size(); + int start = randomIntBetween(0, length - 1); + int end = randomIntBetween(start, length - 1); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field, DataType.SEMANTIC_TEXT, "field"), + new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"), + new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end") + ), + "MvSliceBytesRefEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]", + DataType.SEMANTIC_TEXT, + equalTo(start == end ? field.get(start) : field.subList(start, end + 1)) + ); + })); + suppliers.add(new TestCaseSupplier(List.of(DataType.IP, DataType.INTEGER, DataType.INTEGER), () -> { List field = randomList(1, 10, () -> randomLiteral(DataType.IP).value()); int length = field.size(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java index 63f538059dddf..e5f240c811bd0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java @@ -171,6 +171,20 @@ private static void bytesRefs(List suppliers) { ); })); + suppliers.add(new TestCaseSupplier(List.of(DataType.SEMANTIC_TEXT, DataType.KEYWORD), () -> { + List field = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); + BytesRef order = new BytesRef("ASC"); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field, DataType.SEMANTIC_TEXT, "field"), + new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral() + ), + "MvSortBytesRef[field=Attribute[channel=0], order=true]", + DataType.SEMANTIC_TEXT, + equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList()) + ); + })); + suppliers.add(new TestCaseSupplier(List.of(DataType.IP, DataType.KEYWORD), () -> { List field = randomList(1, 10, () -> randomLiteral(DataType.IP).value()); BytesRef order = new BytesRef("DESC"); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java index 2ad953c9296b7..42c6284a3c25a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java @@ -48,7 +48,7 @@ public static Iterable parameters() { for (int length = 4; length < 100; length++) { suppliers(suppliers, length); } - Set supported = Set.of(DataType.NULL, DataType.KEYWORD, DataType.TEXT); + Set supported = Set.of(DataType.NULL, DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT); List> supportedPerPosition = List.of(supported, supported); for (DataType lhs : DataType.types()) { if (lhs == DataType.NULL || DataType.isRepresentable(lhs) == false) { @@ -72,6 +72,7 @@ private static void suppliers(List suppliers, int length) { if (length > 3) { suppliers.add(supplier("ascii", DataType.KEYWORD, length, () -> randomAlphaOfLengthBetween(1, 10))); suppliers.add(supplier("unicode", DataType.TEXT, length, () -> randomRealisticUnicodeOfLengthBetween(1, 10))); + suppliers.add(supplier("unicode", DataType.SEMANTIC_TEXT, length, () -> randomRealisticUnicodeOfLengthBetween(1, 10))); } else { add(suppliers, "ascii", length, () -> randomAlphaOfLengthBetween(1, 10)); add(suppliers, "unicode", length, () -> randomRealisticUnicodeOfLengthBetween(1, 10)); @@ -99,7 +100,7 @@ private static TestCaseSupplier supplier(String name, DataType type, int length, private static void add(List suppliers, String name, int length, Supplier valueSupplier) { Map>> permutations = new HashMap>>(); - List supportedDataTypes = List.of(DataType.KEYWORD, DataType.TEXT); + List supportedDataTypes = DataType.stringTypes().stream().toList(); permutations.put(0, List.of(List.of(DataType.KEYWORD), List.of(DataType.TEXT))); for (int v = 0; v < length - 1; v++) { List> current = permutations.get(v); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java index 627c46da025ea..e4e54a9e0935f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java @@ -167,6 +167,20 @@ public static Iterable parameters() { ); })); + suppliers.add(new TestCaseSupplier("semantic_text as input", List.of(DataType.SEMANTIC_TEXT, DataType.INTEGER), () -> { + String text = randomUnicodeOfLengthBetween(1, 64); + int length = between(1, text.length()); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataType.SEMANTIC_TEXT, "str"), + new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length") + ), + "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataType.KEYWORD, + equalTo(new BytesRef(unicodeLeftSubstring(text, length))) + ); + })); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { case 0 -> "string"; case 1 -> "integer"; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java index 6ae5a9d961398..ba4c8c8ce1ea4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java @@ -73,6 +73,16 @@ private static List makeTestCases(String title, Supplier new TestCaseSupplier.TestCase( + List.of(new TestCaseSupplier.TypedData(new BytesRef(text.get()), DataType.SEMANTIC_TEXT, "f")), + "LengthEvaluator[val=Attribute[channel=0]]", + DataType.INTEGER, + equalTo(expectedLength) + ) ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java index dab2fca212ff4..4f8adf3abaae6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java @@ -125,7 +125,7 @@ private static void casesForString( } private static void cases(List cases, String title, Supplier textAndPattern, boolean expected) { - for (DataType type : new DataType[] { DataType.KEYWORD, DataType.TEXT }) { + for (DataType type : DataType.stringTypes()) { cases.add(new TestCaseSupplier(title + " with " + type.esType(), List.of(type, type, DataType.BOOLEAN), () -> { TextAndPattern v = textAndPattern.get(); return new TestCaseSupplier.TestCase( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java index 4d97a2f629c23..8b4ea066fdccb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java @@ -62,6 +62,22 @@ public static Iterable parameters() { ); })); + cases.add( + new TestCaseSupplier("Repeat basic test with semantic_text input", List.of(DataType.SEMANTIC_TEXT, DataType.INTEGER), () -> { + String text = randomAlphaOfLength(10); + int number = between(0, 10); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataType.SEMANTIC_TEXT, "str"), + new TestCaseSupplier.TypedData(number, DataType.INTEGER, "number") + ), + "RepeatEvaluator[str=Attribute[channel=0], number=Attribute[channel=1]]", + DataType.KEYWORD, + equalTo(new BytesRef(text.repeat(number))) + ); + }) + ); + cases.add(new TestCaseSupplier("Repeat with number zero", List.of(DataType.KEYWORD, DataType.INTEGER), () -> { String text = randomAlphaOfLength(10); int number = 0; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java index 2873f18d53957..58d52cc02b548 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java @@ -33,7 +33,7 @@ public ReverseTests(@Name("TestCase") Supplier testCa public static Iterable parameters() { List suppliers = new ArrayList<>(); - for (DataType stringType : new DataType[] { DataType.KEYWORD, DataType.TEXT }) { + for (DataType stringType : DataType.stringTypes()) { for (var supplier : TestCaseSupplier.stringCases(stringType)) { suppliers.add(makeSupplier(supplier)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java index a1ef77a62b67c..bf93ef42ed6ad 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java @@ -166,6 +166,19 @@ public static Iterable parameters() { equalTo(new BytesRef(unicodeRightSubstring(text, length))) ); })); + suppliers.add(new TestCaseSupplier("ascii as semantic_text", List.of(DataType.SEMANTIC_TEXT, DataType.INTEGER), () -> { + String text = randomAlphaOfLengthBetween(1, 64); + int length = between(1, text.length()); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataType.SEMANTIC_TEXT, "str"), + new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length") + ), + "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataType.KEYWORD, + equalTo(new BytesRef(unicodeRightSubstring(text, length))) + ); + })); return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { case 0 -> "string"; case 1 -> "integer"; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java index b5560f37914a9..098be8e1fda37 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java @@ -42,9 +42,8 @@ public SplitTests(@Name("TestCase") Supplier testCase @ParametersFactory public static Iterable parameters() { List suppliers = new ArrayList<>(); - List supportedDataTyes = List.of(DataType.KEYWORD, DataType.TEXT); - for (DataType sType : supportedDataTyes) { - for (DataType dType : supportedDataTyes) { + for (DataType sType : DataType.stringTypes()) { + for (DataType dType : DataType.stringTypes()) { suppliers.add(new TestCaseSupplier("split test " + sType.toString() + " " + dType.toString(), List.of(sType, dType), () -> { String delimiter = randomAlphaOfLength(1); List strings = IntStream.range(0, between(1, 5)) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java index 6b934aae775df..ae8a2a1840dfb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java @@ -72,6 +72,25 @@ public static Iterable parameters() { ); } ), + new TestCaseSupplier( + "Substring basic test with semantic_text input", + List.of(DataType.SEMANTIC_TEXT, DataType.INTEGER, DataType.INTEGER), + () -> { + int start = between(1, 8); + int length = between(1, 10 - start); + String text = randomAlphaOfLength(10); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataType.SEMANTIC_TEXT, "str"), + new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"), + new TestCaseSupplier.TypedData(length, DataType.INTEGER, "end") + ), + "SubstringEvaluator[str=Attribute[channel=0], start=Attribute[channel=1], length=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(text.substring(start - 1, start + length - 1))) + ); + } + ), new TestCaseSupplier("Substring empty string", List.of(DataType.TEXT, DataType.INTEGER, DataType.INTEGER), () -> { int start = between(1, 8); int length = between(1, 10 - start); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java index 1f564ecb87f1e..69dbe023bde66 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java @@ -45,6 +45,8 @@ public static Iterable parameters() { suppliers.add(supplier("keyword unicode", DataType.KEYWORD, () -> randomUnicodeOfLengthBetween(1, 10))); suppliers.add(supplier("text ascii", DataType.TEXT, () -> randomAlphaOfLengthBetween(1, 10))); suppliers.add(supplier("text unicode", DataType.TEXT, () -> randomUnicodeOfLengthBetween(1, 10))); + suppliers.add(supplier("semantic_text ascii", DataType.SEMANTIC_TEXT, () -> randomAlphaOfLengthBetween(1, 10))); + suppliers.add(supplier("semantic_text unicode", DataType.SEMANTIC_TEXT, () -> randomUnicodeOfLengthBetween(1, 10))); // add null as parameter return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java index 7c136c3bb83c2..33d6f929503b3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java @@ -45,6 +45,8 @@ public static Iterable parameters() { suppliers.add(supplier("keyword unicode", DataType.KEYWORD, () -> randomUnicodeOfLengthBetween(1, 10))); suppliers.add(supplier("text ascii", DataType.TEXT, () -> randomAlphaOfLengthBetween(1, 10))); suppliers.add(supplier("text unicode", DataType.TEXT, () -> randomUnicodeOfLengthBetween(1, 10))); + suppliers.add(supplier("semantic_text ascii", DataType.SEMANTIC_TEXT, () -> randomAlphaOfLengthBetween(1, 10))); + suppliers.add(supplier("semantic_text unicode", DataType.SEMANTIC_TEXT, () -> randomUnicodeOfLengthBetween(1, 10))); // add null as parameter return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java index 53b3a99f97b9c..eed2c7379e9e1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java @@ -53,7 +53,7 @@ public static Iterable parameters() { } private static void addCases(List suppliers) { - for (DataType type : new DataType[] { DataType.KEYWORD, DataType.TEXT }) { + for (DataType type : new DataType[] { DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT }) { suppliers.add(new TestCaseSupplier(" with " + type.esType(), List.of(type, type), () -> { BytesRef str = new BytesRef(randomAlphaOfLength(5)); String patternString = randomAlphaOfLength(2); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java index 66edd56e32f20..0fb416584b472 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java @@ -217,8 +217,8 @@ public static Iterable parameters() { } private static String typeErrorString = - "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long, text, " - + "unsigned_long or version"; + "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long," + + " semantic_text, text, unsigned_long or version"; @Override protected Expression build(Source source, List args) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java index a4d1bf69796e0..a4f1a19e135ef 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java @@ -150,7 +150,7 @@ public static Iterable parameters() { o, v, t, - (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, text, unsigned_long or version" + (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" ) ) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java index d3fede5c2e2ce..86a4676e35009 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java @@ -150,7 +150,7 @@ public static Iterable parameters() { o, v, t, - (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, text, unsigned_long or version" + (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" ) ) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java index 2a1dfb098a3a4..b004adca351ab 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java @@ -187,8 +187,24 @@ private static void bytesRefs(List suppliers, int items) { ); })); - for (DataType type1 : new DataType[] { DataType.KEYWORD, DataType.TEXT }) { - for (DataType type2 : new DataType[] { DataType.KEYWORD, DataType.TEXT }) { + suppliers.add(new TestCaseSupplier("semantic_text", List.of(DataType.SEMANTIC_TEXT, DataType.SEMANTIC_TEXT), () -> { + List inlist = randomList(items, items, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); + Object field = inlist.get(0); + List args = new ArrayList<>(inlist.size() + 1); + for (Object i : inlist) { + args.add(new TestCaseSupplier.TypedData(i, DataType.SEMANTIC_TEXT, "inlist" + i)); + } + args.add(new TestCaseSupplier.TypedData(field, DataType.SEMANTIC_TEXT, "field")); + return new TestCaseSupplier.TestCase( + args, + matchesPattern("InBytesRefEvaluator.*"), + DataType.BOOLEAN, + equalTo(inlist.contains(field)) + ); + })); + + for (DataType type1 : DataType.stringTypes()) { + for (DataType type2 : DataType.stringTypes()) { if (type1 == type2 || items > 1) continue; suppliers.add(new TestCaseSupplier(type1 + " " + type2, List.of(type1, type2), () -> { List inlist = randomList(items, items, () -> randomLiteral(type1).value()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java index 3b8270c1576fd..5793f26ecd447 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java @@ -150,7 +150,7 @@ public static Iterable parameters() { o, v, t, - (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, text, unsigned_long or version" + (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" ) ) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java index 647988fe35326..e8f9f26a76f43 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java @@ -150,7 +150,7 @@ public static Iterable parameters() { o, v, t, - (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, text, unsigned_long or version" + (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" ) ) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java index babb9fc8c0bd1..b2228b5543ef2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java @@ -30,11 +30,13 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.HALF_FLOAT; import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.IP; +import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT; import static org.elasticsearch.xpack.esql.core.type.DataType.PARTIAL_AGG; import static org.elasticsearch.xpack.esql.core.type.DataType.SCALED_FLOAT; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.SHORT; import static org.elasticsearch.xpack.esql.core.type.DataType.SOURCE; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; @@ -69,6 +71,8 @@ public void testCommonTypeStrings() { } else if ((isString(dataType1) && isString(dataType2))) { if (dataType1 == dataType2) { assertEqualsCommonType(dataType1, dataType2, dataType1); + } else if (dataType1 == SEMANTIC_TEXT || dataType2 == SEMANTIC_TEXT) { + assertEqualsCommonType(dataType1, dataType2, KEYWORD); } else { assertEqualsCommonType(dataType1, dataType2, TEXT); } diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java index fed63477701e3..f9a1318cd9740 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java @@ -13,8 +13,10 @@ import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceFeature; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -132,7 +134,11 @@ public void testApisWithoutTaskType() throws IOException { @SuppressWarnings("unchecked") public void testGetServicesWithoutTaskType() throws IOException { List services = getAllServices(); - assertThat(services.size(), equalTo(19)); + if (ElasticInferenceServiceFeature.ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled()) { + assertThat(services.size(), equalTo(19)); + } else { + assertThat(services.size(), equalTo(18)); + } String[] providers = new String[services.size()]; for (int i = 0; i < services.size(); i++) { @@ -141,16 +147,15 @@ public void testGetServicesWithoutTaskType() throws IOException { } Arrays.sort(providers); - assertArrayEquals( - providers, - List.of( + + var providerList = new ArrayList<>( + Arrays.asList( "alibabacloud-ai-search", "amazonbedrock", "anthropic", "azureaistudio", "azureopenai", "cohere", - "elastic", "elasticsearch", "googleaistudio", "googlevertexai", @@ -163,8 +168,12 @@ public void testGetServicesWithoutTaskType() throws IOException { "test_service", "text_embedding_test_service", "watsonxai" - ).toArray() + ) ); + if (ElasticInferenceServiceFeature.ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled()) { + providerList.add(6, "elastic"); + } + assertArrayEquals(providers, providerList.toArray()); } @SuppressWarnings("unchecked") @@ -248,7 +257,12 @@ public void testGetServicesWithCompletionTaskType() throws IOException { @SuppressWarnings("unchecked") public void testGetServicesWithSparseEmbeddingTaskType() throws IOException { List services = getServices(TaskType.SPARSE_EMBEDDING); - assertThat(services.size(), equalTo(6)); + + if (ElasticInferenceServiceFeature.ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled()) { + assertThat(services.size(), equalTo(6)); + } else { + assertThat(services.size(), equalTo(5)); + } String[] providers = new String[services.size()]; for (int i = 0; i < services.size(); i++) { @@ -257,10 +271,14 @@ public void testGetServicesWithSparseEmbeddingTaskType() throws IOException { } Arrays.sort(providers); - assertArrayEquals( - providers, - List.of("alibabacloud-ai-search", "elastic", "elasticsearch", "hugging_face", "hugging_face_elser", "test_service").toArray() + + var providerList = new ArrayList<>( + Arrays.asList("alibabacloud-ai-search", "elasticsearch", "hugging_face", "hugging_face_elser", "test_service") ); + if (ElasticInferenceServiceFeature.ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled()) { + providerList.add(1, "elastic"); + } + assertArrayEquals(providers, providerList.toArray()); } public void testSkipValidationAndStart() throws IOException { diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/20_ignored_source.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/20_ignored_source.yml index 2f111d579ebb1..ff7f01fccaa3c 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/20_ignored_source.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/20_ignored_source.yml @@ -6,9 +6,10 @@ setup: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: object: enabled: false diff --git a/x-pack/plugin/mapper-constant-keyword/src/yamlRestTest/resources/rest-api-spec/test/20_synthetic_source.yml b/x-pack/plugin/mapper-constant-keyword/src/yamlRestTest/resources/rest-api-spec/test/20_synthetic_source.yml index b64fb7b822713..6cd30f42c52e9 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/yamlRestTest/resources/rest-api-spec/test/20_synthetic_source.yml +++ b/x-pack/plugin/mapper-constant-keyword/src/yamlRestTest/resources/rest-api-spec/test/20_synthetic_source.yml @@ -7,9 +7,10 @@ constant_keyword: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: const_kwd: type: constant_keyword diff --git a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/80_synthetic_source.yml b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/80_synthetic_source.yml index b88fca3c478a9..e8b86231b7196 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/80_synthetic_source.yml +++ b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/80_synthetic_source.yml @@ -7,9 +7,10 @@ synthetic source: indices.create: index: synthetic_source_test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -52,9 +53,10 @@ synthetic source with copy_to: indices.create: index: synthetic_source_test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -111,9 +113,10 @@ synthetic source with disabled doc_values: indices.create: index: synthetic_source_test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword diff --git a/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/40_synthetic_source.yml b/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/40_synthetic_source.yml index 1ec91f5fde8d1..c96eeeb943831 100644 --- a/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/40_synthetic_source.yml +++ b/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/40_synthetic_source.yml @@ -7,9 +7,10 @@ setup: indices.create: index: test1 body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: ver: type: version @@ -76,9 +77,10 @@ synthetic source with copy_to: indices.create: index: synthetic_source_test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: ver: type: version diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/TrainedModelProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/TrainedModelProvider.java index ff5f37427b18f..5cf349b96a4f7 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/TrainedModelProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/TrainedModelProvider.java @@ -14,6 +14,8 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.bulk.BulkItemResponse; @@ -28,6 +30,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -43,6 +46,7 @@ import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.engine.VersionConflictEngineException; @@ -105,6 +109,7 @@ import java.util.Objects; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; @@ -1053,63 +1058,98 @@ public void expandIds( } public void getInferenceStats(String[] modelIds, @Nullable TaskId parentTaskId, ActionListener> listener) { - MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); - Arrays.stream(modelIds).map(TrainedModelProvider::buildStatsSearchRequest).forEach(multiSearchRequest::add); - if (multiSearchRequest.requests().isEmpty()) { - listener.onResponse(Collections.emptyList()); - return; - } - if (parentTaskId != null) { - multiSearchRequest.setParentTask(parentTaskId); - } - executeAsyncWithOrigin( - client.threadPool().getThreadContext(), - ML_ORIGIN, - multiSearchRequest, - ActionListener.wrap(responses -> { - List allStats = new ArrayList<>(modelIds.length); - int modelIndex = 0; - assert responses.getResponses().length == modelIds.length : "mismatch between search response size and models requested"; - for (MultiSearchResponse.Item response : responses.getResponses()) { - if (response.isFailure()) { - if (ExceptionsHelper.unwrapCause(response.getFailure()) instanceof ResourceNotFoundException) { - modelIndex++; - continue; - } + + SubscribableListener.newForked((delegate) -> { + // first wait for the index to be available + executeAsyncWithOrigin( + client.threadPool().getThreadContext(), + ML_ORIGIN, + new ClusterHealthRequest(new TimeValue(2, TimeUnit.SECONDS), MlStatsIndex.indexPattern()).waitForYellowStatus(), + delegate, + client.admin().cluster()::health + ); + }) + .>andThen( + client.threadPool().executor(MachineLearning.UTILITY_THREAD_POOL_NAME), + client.threadPool().getThreadContext(), + (delegate, clusterHealthResponse) -> { + if (clusterHealthResponse.isTimedOut()) { logger.error( - () -> "[" + Strings.arrayToCommaDelimitedString(modelIds) + "] search failed for models", - response.getFailure() - ); - listener.onFailure( - ExceptionsHelper.serverError( - "Searching for stats for models [{}] failed", - response.getFailure(), - Strings.arrayToCommaDelimitedString(modelIds) - ) + "getInferenceStats Timed out waiting for index [{}] to be available, " + + "this will probably cause the request to fail", + MlStatsIndex.indexPattern() ); - return; } - try { - InferenceStats inferenceStats = handleMultiNodeStatsResponse(response.getResponse(), modelIds[modelIndex++]); - if (inferenceStats != null) { - allStats.add(inferenceStats); - } - } catch (Exception e) { - listener.onFailure(e); + + MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); + Arrays.stream(modelIds).map(TrainedModelProvider::buildStatsSearchRequest).forEach(multiSearchRequest::add); + if (multiSearchRequest.requests().isEmpty()) { + delegate.onResponse(Collections.emptyList()); return; } + if (parentTaskId != null) { + multiSearchRequest.setParentTask(parentTaskId); + } + executeAsyncWithOrigin( + client.threadPool().getThreadContext(), + ML_ORIGIN, + multiSearchRequest, + ActionListener.wrap(responses -> { + List allStats = new ArrayList<>(modelIds.length); + int modelIndex = 0; + assert responses.getResponses().length == modelIds.length + : "mismatch between search response size and models requested"; + for (MultiSearchResponse.Item response : responses.getResponses()) { + if (response.isFailure()) { + if (ExceptionsHelper.unwrapCause(response.getFailure()) instanceof ResourceNotFoundException) { + modelIndex++; + continue; + } + logger.error( + () -> "[" + Strings.arrayToCommaDelimitedString(modelIds) + "] search failed for models", + response.getFailure() + ); + delegate.onFailure( + ExceptionsHelper.serverError( + "Searching for stats for models [{}] failed", + response.getFailure(), + Strings.arrayToCommaDelimitedString(modelIds) + ) + ); + return; + } + try { + InferenceStats inferenceStats = handleMultiNodeStatsResponse( + response.getResponse(), + modelIds[modelIndex++] + ); + if (inferenceStats != null) { + allStats.add(inferenceStats); + } + } catch (Exception e) { + delegate.onFailure(e); + return; + } + } + delegate.onResponse(allStats); + }, e -> { + Throwable unwrapped = ExceptionsHelper.unwrapCause(e); + if (unwrapped instanceof ResourceNotFoundException) { + delegate.onResponse(Collections.emptyList()); + return; + } + delegate.onFailure((Exception) unwrapped); + }), + client::multiSearch + ); + } - listener.onResponse(allStats); - }, e -> { - Throwable unwrapped = ExceptionsHelper.unwrapCause(e); - if (unwrapped instanceof ResourceNotFoundException) { - listener.onResponse(Collections.emptyList()); - return; - } - listener.onFailure((Exception) unwrapped); - }), - client::multiSearch - ); + ) + .addListener( + listener, + client.threadPool().executor(MachineLearning.UTILITY_THREAD_POOL_NAME), + client.threadPool().getThreadContext() + ); } private static SearchRequest buildStatsSearchRequest(String modelId) { diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/async/AsyncTaskManagementServiceTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/async/AsyncTaskManagementServiceTests.java index 65e5efbccb1af..018504785b5eb 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/async/AsyncTaskManagementServiceTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/async/AsyncTaskManagementServiceTests.java @@ -313,7 +313,7 @@ public void execute(TestRequest request, TestTask task, ActionListener response = getResponse(responseHolder.get().id, TimeValue.ZERO); if (success) { assertThat(response.getException(), nullValue()); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java index 8b63b76cdf248..7d120e62e0260 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java @@ -396,7 +396,7 @@ public void onFailure(Exception e) { }); assertUnblockIn10s(latch2); - // the the client answer + // the client answer unblock.countDown(); } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java index 0cbd7f1389188..207f92759fa34 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java @@ -70,7 +70,7 @@ public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryB // set page size if (size != null) { int sz = container.limit() > 0 ? Math.min(container.limit(), size) : size; - // now take into account the the minimum page (if set) + // now take into account the minimum page (if set) // that is, return the multiple of the minimum page size closer to the set size int minSize = container.minPageSize(); sz = minSize > 0 ? (Math.max(sz / minSize, 1) * minSize) : sz; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/MetricAggExtractor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/MetricAggExtractor.java index f76702f5ffe5d..c12a172453941 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/MetricAggExtractor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/MetricAggExtractor.java @@ -139,7 +139,7 @@ private Object handleTargetType(Object object) { return DateUtils.asDateTimeWithMillis(((Number) object).longValue(), zoneId); } else if (dataType.isInteger()) { // MIN and MAX need to return the same type as field's and SUM a long for integral types, but ES returns them always as - // floating points -> convert them in the the SELECT pipeline, if needed + // floating points -> convert them in the SELECT pipeline, if needed return convert(object, dataType); } } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/100_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/100_synthetic_source.yml index cc0e8aff9b239..0937d24217e31 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/100_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/100_synthetic_source.yml @@ -7,9 +7,10 @@ aggregate_metric_double: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: metric: type: aggregate_metric_double @@ -62,9 +63,10 @@ aggregate_metric_double with ignore_malformed: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: metric: type: aggregate_metric_double diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/histogram.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/histogram.yml index 726b9d153025e..88445cbad1dc8 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/histogram.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/histogram.yml @@ -182,9 +182,10 @@ histogram with synthetic source: indices.create: index: histo_synthetic body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: latency: type: histogram @@ -228,9 +229,10 @@ histogram with synthetic source and zero counts: indices.create: index: histo_synthetic body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: latency: type: histogram @@ -317,9 +319,10 @@ histogram with synthetic source and ignore_malformed: indices.create: index: histo_synthetic body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: latency: type: histogram diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/enrich/40_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/enrich/40_synthetic_source.yml index 1c2e1cd922a65..1aaa39a0f13b7 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/enrich/40_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/enrich/40_synthetic_source.yml @@ -4,9 +4,10 @@ setup: indices.create: index: source body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: baz: type: keyword diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/30_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/30_types.yml index cfc7f2e4036fb..9658412f150fd 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/30_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/30_types.yml @@ -809,9 +809,10 @@ synthetic _source text stored: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: card: type: text @@ -840,9 +841,10 @@ synthetic _source text with parent keyword: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: card: type: keyword diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml index 55bd39bdd73cc..ded66b9453452 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml @@ -481,9 +481,10 @@ setup: indices.create: index: test2 body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: "emp_no": type: long @@ -526,9 +527,10 @@ setup: indices.create: index: test2 body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: "emp_no": type: long diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/searchable_snapshots/20_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/searchable_snapshots/20_synthetic_source.yml index b1ab120fff441..df3c905408a87 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/searchable_snapshots/20_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/searchable_snapshots/20_synthetic_source.yml @@ -11,9 +11,9 @@ setup: settings: number_of_shards: 1 number_of_replicas: 0 + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: obj: properties: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml index b971c246ac50a..5e636aebc0271 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml @@ -13,9 +13,10 @@ Filter single field: indices.create: index: index_fls body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -75,9 +76,10 @@ Filter fields in object: indices.create: index: index_fls body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -142,9 +144,10 @@ Fields under a disabled object - uses _ignored_source: indices.create: index: index_fls body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -236,12 +239,11 @@ Dynamic fields beyond limit - uses _ignored_source: settings: index: mapping: + source.mode: synthetic total_fields: ignore_dynamic_beyond_limit: true limit: 2 mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -301,9 +303,10 @@ Field with ignored_malformed: indices.create: index: index_fls body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/40_document_level_security_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/40_document_level_security_synthetic_source.yml index 52abe0a3d83d7..37e78d86f6667 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/40_document_level_security_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/40_document_level_security_synthetic_source.yml @@ -13,9 +13,10 @@ Filter on single field: indices.create: index: index_dls body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -95,9 +96,10 @@ Filter on nested field: indices.create: index: index_dls body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -178,9 +180,10 @@ Filter on object with stored source: indices.create: index: index_dls body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -258,9 +261,10 @@ Filter on field within a disabled object: indices.create: index: index_dls body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -335,9 +339,10 @@ Filter on field with ignored_malformed: indices.create: index: index_dls body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml index e1b297f1b5d78..6ccd24ae84af9 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml @@ -100,12 +100,11 @@ setup: indices.create: index: test_synthetic body: - mappings: - _source: - mode: synthetic settings: number_of_shards: 1 number_of_replicas: 0 + index: + mapping.source.mode: synthetic - do: snapshot.create: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml index 700142cec9987..17517640f2aa5 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml @@ -8,9 +8,10 @@ indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: shape: type: geo_shape @@ -74,9 +75,10 @@ indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: shape: type: geo_shape @@ -157,9 +159,10 @@ indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: shape: type: shape @@ -223,9 +226,10 @@ indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: shape: type: shape @@ -306,9 +310,10 @@ indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: point: type: geo_point @@ -422,9 +427,10 @@ indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: geo_point: type: geo_point @@ -501,9 +507,10 @@ indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: point: type: point @@ -597,9 +604,10 @@ indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: point: type: point diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/preview_transforms_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/preview_transforms_synthetic_source.yml index 08055946a7831..ee5ec824fd212 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/preview_transforms_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/preview_transforms_synthetic_source.yml @@ -6,9 +6,10 @@ simple: indices.create: index: airline-data body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: time: type: date diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/30_ignore_above_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/30_ignore_above_synthetic_source.yml index 2e3ba773fb0f2..d844bf9de9129 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/30_ignore_above_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/30_ignore_above_synthetic_source.yml @@ -10,10 +10,9 @@ wildcard field type ignore_above: settings: index: mapping: + source.mode: synthetic ignore_above: 10 mappings: - _source: - mode: synthetic properties: a_wildcard: type: wildcard diff --git a/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/30_synthetic_source.yml b/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/30_synthetic_source.yml index ffa76f7433985..20472669d4d77 100644 --- a/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/30_synthetic_source.yml +++ b/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/30_synthetic_source.yml @@ -7,9 +7,10 @@ synthetic source: indices.create: index: synthetic_source_test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -48,9 +49,10 @@ synthetic source with copy_to: indices.create: index: synthetic_source_test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword