From 022a678df686a1bf335a1776e5389a78fd72ce48 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 26 Sep 2024 06:18:22 +1000 Subject: [PATCH 01/36] Mute org.elasticsearch.xpack.inference.TextEmbeddingCrudIT testPutE5WithTrainedModelAndInference #113565 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 831556825693a..07a31dce9047c 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -301,6 +301,9 @@ tests: - class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT method: test {p0=esql/70_locale/Date format with Italian locale} issue: https://github.com/elastic/elasticsearch/issues/113540 +- class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT + method: testPutE5WithTrainedModelAndInference + issue: https://github.com/elastic/elasticsearch/issues/113565 # Examples: # From cd950bb2fa060978c1f08da456685b26c16b8d2e Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 25 Sep 2024 15:30:22 -0500 Subject: [PATCH 02/36] Adding component template substitutions to the simulate ingest API (#113276) --- docs/changelog/113276.yaml | 5 + .../indices/put-component-template.asciidoc | 7 +- .../ingest/apis/simulate-ingest.asciidoc | 117 ++++++- .../test/ingest/80_ingest_simulate.yml | 303 ++++++++++++++++++ .../action/bulk/BulkFeatures.java | 3 +- .../bulk/TransportSimulateBulkAction.java | 3 + 6 files changed, 434 insertions(+), 4 deletions(-) create mode 100644 docs/changelog/113276.yaml diff --git a/docs/changelog/113276.yaml b/docs/changelog/113276.yaml new file mode 100644 index 0000000000000..87241878b3ec4 --- /dev/null +++ b/docs/changelog/113276.yaml @@ -0,0 +1,5 @@ +pr: 113276 +summary: Adding component template substitutions to the simulate ingest API +area: Ingest Node +type: enhancement +issues: [] diff --git a/docs/reference/indices/put-component-template.asciidoc b/docs/reference/indices/put-component-template.asciidoc index 0a0e36b63e6cd..d880edfe42b8c 100644 --- a/docs/reference/indices/put-component-template.asciidoc +++ b/docs/reference/indices/put-component-template.asciidoc @@ -129,6 +129,8 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[put-component-template-api-request-body]] ==== {api-request-body-title} +// tag::template[] + `template`:: (Required, object) This is the template to be applied, may optionally include a `mappings`, @@ -136,7 +138,7 @@ This is the template to be applied, may optionally include a `mappings`, + .Properties of `template` [%collapsible%open] -==== +===== `aliases`:: (Optional, object of objects) Aliases to add. + @@ -147,7 +149,7 @@ include::{es-ref-dir}/indices/create-index.asciidoc[tag=aliases-props] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=mappings] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=settings] -==== +===== `version`:: (Optional, integer) @@ -174,6 +176,7 @@ This map is not automatically generated by {es}. Marks this component template as deprecated. When a deprecated component template is referenced when creating or updating a non-deprecated index template, {es} will emit a deprecation warning. +end::template[] [[put-component-template-api-example]] ==== {api-examples-title} diff --git a/docs/reference/ingest/apis/simulate-ingest.asciidoc b/docs/reference/ingest/apis/simulate-ingest.asciidoc index ee84a39ee6f65..ac6da515402bb 100644 --- a/docs/reference/ingest/apis/simulate-ingest.asciidoc +++ b/docs/reference/ingest/apis/simulate-ingest.asciidoc @@ -83,11 +83,32 @@ POST /_ingest/_simulate } ] } + }, + "component_template_substitutions": { <2> + "my-component-template": { + "template": { + "mappings": { + "dynamic": "true", + "properties": { + "field3": { + "type": "keyword" + } + } + }, + "settings": { + "index": { + "default_pipeline": "my-pipeline" + } + } + } + } } } ---- <1> This replaces the existing `my-pipeline` pipeline with the contents given here for the duration of this request. +<2> This replaces the existing `my-component-template` component template with the contents given here for the duration of this request. +These templates can be used to change the pipeline(s) used, or to modify the mapping that will be used to validate the result. [[simulate-ingest-api-request]] ==== {api-request-title} @@ -191,6 +212,19 @@ Map of pipeline IDs to substitute pipeline definition objects. include::put-pipeline.asciidoc[tag=pipeline-object] ==== +`component_template_substitutions`:: +(Optional, map of strings to objects) +Map of component template names to substitute component template definition objects. ++ +.Properties of component template definition objects +[%collapsible%open] + +==== + +include::{es-ref-dir}/indices/put-component-template.asciidoc[tag=template] + +==== + [[simulate-ingest-api-example]] ==== {api-examples-title} @@ -268,7 +302,7 @@ The API returns the following response: [[simulate-ingest-api-request-body-ex]] ===== Specify a pipeline substitution in the request body -In this example the index `index` has a default pipeline called `my-pipeline` and a final +In this example the index `my-index` has a default pipeline called `my-pipeline` and a final pipeline called `my-final-pipeline`. But a substitute definition of `my-pipeline` is provided in `pipeline_substitutions`. The substitute `my-pipeline` will be used in place of the `my-pipeline` that is in the system, and then the `my-final-pipeline` that is already @@ -348,6 +382,87 @@ The API returns the following response: } ---- +[[simulate-ingest-api-substitute-component-templates-ex]] +===== Specify a component template substitution in the request body +In this example, imagine that the index `my-index` has a strict mapping with only the `foo` +keyword field defined. Say that field mapping came from a component template named +`my-mappings-template`. We want to test adding a new field, `bar`. So a substitute definition of +`my-mappings-template` is provided in `component_template_substitutions`. The substitute +`my-mappings-template` will be used in place of the existing mapping for `my-index` and in place +of the `my-mappings-template` that is in the system. + +[source,console] +---- +POST /_ingest/_simulate +{ + "docs": [ + { + "_index": "my-index", + "_id": "123", + "_source": { + "foo": "foo" + } + }, + { + "_index": "my-index", + "_id": "456", + "_source": { + "bar": "rab" + } + } + ], + "component_template_substitutions": { + "my-mappings_template": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + }, + "bar": { + "type": "keyword" + } + } + } + } + } + } +} +---- + +The API returns the following response: + +[source,console-result] +---- +{ + "docs": [ + { + "doc": { + "_id": "123", + "_index": "my-index", + "_version": -3, + "_source": { + "foo": "foo" + }, + "executed_pipelines": [] + } + }, + { + "doc": { + "_id": "456", + "_index": "my-index", + "_version": -3, + "_source": { + "bar": "rab" + }, + "executed_pipelines": [] + } + } + ] +} +---- + //// [source,console] ---- diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index 35ec9979c3250..f3a977cd96f62 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -183,6 +183,7 @@ setup: body: settings: default_pipeline: "my-pipeline" + - match: { acknowledged: true } - do: headers: @@ -303,3 +304,305 @@ setup: - match: { docs.1.doc._index: "second-index" } - match: { docs.1.doc._source.bar: "foo" } - not_exists: docs.1.doc.error + +--- +"Test ingest simulate with template substitutions for component templates": + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.component.template.substitutions"] + reason: "ingest simulate component template substitutions added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "bar-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "bar", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: mappings_template + body: + template: + mappings: + dynamic: strict + properties: + foo: + type: keyword + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo_pipeline" + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [tsdb_templated_*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - mappings_template + - settings_template + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO", + "other": "other" + } + } + ], + "component_template_substitutions": { + "mappings_template": { + "template": { + "mappings": { + "dynamic": "true", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + }, + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": "bar-pipeline" + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.other: "other" } + - match: { docs.0.doc._source.bar: true } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["bar-pipeline"] } + - not_exists: docs.0.doc.error + + - do: + indices.create: + index: foo-1 + - match: { acknowledged: true } + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO", + "other": "other" + } + } + ], + "component_template_substitutions": { + "mappings_template": { + "template": { + "mappings": { + "dynamic": "true", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + }, + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": "bar-pipeline" + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.other: "other" } + - match: { docs.0.doc._source.bar: true } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["bar-pipeline"] } + - not_exists: docs.0.doc.error + +--- +"Test ingest simulate with template substitutions for component templates removing pipelines": + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.component.template.substitutions"] + reason: "ingest simulate component template substitutions added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo_pipeline" + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [tsdb_templated_*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - settings_template + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO" + } + } + ], + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": null + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: [] } + - not_exists: docs.0.doc.error + + - do: + indices.create: + index: foo-1 + - match: { acknowledged: true } + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO" + } + } + ], + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": null + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: [] } + - not_exists: docs.0.doc.error diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java index 8299d53da17aa..af1782ac1ade3 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java @@ -14,11 +14,12 @@ import java.util.Set; +import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS; import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION; import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION_TEMPLATES; public class BulkFeatures implements FeatureSpecification { public Set getFeatures() { - return Set.of(SIMULATE_MAPPING_VALIDATION, SIMULATE_MAPPING_VALIDATION_TEMPLATES); + return Set.of(SIMULATE_MAPPING_VALIDATION, SIMULATE_MAPPING_VALIDATION_TEMPLATES, SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 0ea763c215959..c860c49809cb5 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -69,6 +69,9 @@ public class TransportSimulateBulkAction extends TransportAbstractBulkAction { public static final NodeFeature SIMULATE_MAPPING_VALIDATION = new NodeFeature("simulate.mapping.validation"); public static final NodeFeature SIMULATE_MAPPING_VALIDATION_TEMPLATES = new NodeFeature("simulate.mapping.validation.templates"); + public static final NodeFeature SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS = new NodeFeature( + "simulate.component.template.substitutions" + ); private final IndicesService indicesService; private final NamedXContentRegistry xContentRegistry; private final Set indexSettingProviders; From 35fbbec46abb2c6e9d84ffce063a2775b02bdfcc Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Wed, 25 Sep 2024 14:38:37 -0700 Subject: [PATCH 03/36] Fix synthetic source for flattened field when used with ignore_above (#113499) --- docs/changelog/113499.yaml | 6 + rest-api-spec/build.gradle | 1 + .../test/get/100_synthetic_source.yml | 57 +++++++ .../540_ignore_above_synthetic_source.yml | 8 +- .../index/mapper/MapperFeatures.java | 3 +- .../flattened/FlattenedFieldMapper.java | 10 +- .../flattened/FlattenedFieldParser.java | 79 +++------- .../FlattenedFieldSyntheticWriterHelper.java | 17 ++- ...ortedSetDocValuesSyntheticFieldLoader.java | 142 ++++++++++++++++-- .../flattened/FlattenedFieldMapperTests.java | 131 +++++++++++++++- .../flattened/FlattenedFieldParserTests.java | 14 +- ...ttenedFieldSyntheticWriterHelperTests.java | 31 +++- 12 files changed, 397 insertions(+), 102 deletions(-) create mode 100644 docs/changelog/113499.yaml diff --git a/docs/changelog/113499.yaml b/docs/changelog/113499.yaml new file mode 100644 index 0000000000000..a4d7f28eb0de4 --- /dev/null +++ b/docs/changelog/113499.yaml @@ -0,0 +1,6 @@ +pr: 113499 +summary: Fix synthetic source for flattened field when used with `ignore_above` +area: Logs +type: bug +issues: + - 112044 diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 5ea0b202d76de..45c1b65d19600 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -58,4 +58,5 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("tsdb/140_routing_path/multi-value routing path field", "Multi-value routing paths are allowed now. See #112645") task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") + task.skipTest("search/540_ignore_above_synthetic_source/ignore_above mapping level setting on arrays", "Temporary mute while backporting to 8.x") }) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml index f1e296ed8e304..a7600da575cd3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml @@ -1050,6 +1050,63 @@ flattened field with ignore_above: - is_false: fields + +--- +flattened field with ignore_above and arrays: + - requires: + cluster_features: ["mapper.flattened.ignore_above_with_arrays_support"] + reason: requires support of ignore_above synthetic source with arrays + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + field: + type: flattened + ignore_above: 10 + + - do: + index: + index: test + id: 1 + body: | + { + "field": [ + { "key1": { "key2": "key2", "key3": "key3_ignored" }, "key4": "key4_ignored", "key5": { "key6": "key6_ignored" }, "key7": "key7" }, + { "key1": { "key2": "key12", "key13": "key13_ignored" }, "key4": "key14_ignored", "key15": { "key16": "key16_ignored" }, "key17": [ "key17", "key18" ] } + ] + } + + - do: + get: + index: test + id: 1 + + - match: { _index: "test" } + - match: { _id: "1" } + - match: { _version: 1 } + - match: { found: true } + - match: + _source: + field: + key1: + key2: [ "key12", "key2" ] + key3: "key3_ignored" + key13: "key13_ignored" + key4: [ "key14_ignored", "key4_ignored" ] + key5: + key6: "key6_ignored" + key7: "key7" + key15: + key16: "key16_ignored" + key17: [ "key17", "key18" ] + + - is_false: fields + --- completion: - requires: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml index defdc8467bf8d..11259d3e1bfd1 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml @@ -44,8 +44,8 @@ ignore_above mapping level setting: --- ignore_above mapping level setting on arrays: - requires: - cluster_features: [ "mapper.ignore_above_index_level_setting" ] - reason: introduce ignore_above index level setting + cluster_features: [ "mapper.flattened.ignore_above_with_arrays_support" ] + reason: requires support of ignore_above with arrays for flattened fields - do: indices.create: index: test @@ -80,9 +80,9 @@ ignore_above mapping level setting on arrays: match_all: {} - length: { hits.hits: 1 } - #TODO: synthetic source field reconstruction bug (TBD: add link to the issue here) + #TODO: synthetic source field reconstruction bug (TBD: add link to the issue here) #- match: { hits.hits.0._source.keyword: ["foo bar", "the quick brown fox"] } - - match: { hits.hits.0._source.flattened.value: ["the quick brown fox", "jumps over"] } + - match: { hits.hits.0._source.flattened.value: [ "jumps over", "the quick brown fox" ] } - match: { hits.hits.0.fields.keyword.0: "foo bar" } - match: { hits.hits.0.fields.flattened.0.value: "jumps over" } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index ac7d10abc7121..2f665fd5d1e6a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -44,7 +44,8 @@ public Set getFeatures() { FlattenedFieldMapper.IGNORE_ABOVE_SUPPORT, IndexSettings.IGNORE_ABOVE_INDEX_LEVEL_SETTING, SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX, - TimeSeriesRoutingHashFieldMapper.TS_ROUTING_HASH_FIELD_PARSES_BYTES_REF + TimeSeriesRoutingHashFieldMapper.TS_ROUTING_HASH_FIELD_PARSES_BYTES_REF, + FlattenedFieldMapper.IGNORE_ABOVE_WITH_ARRAYS_SUPPORT ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 2c504262c35ad..ac1de94ea7a73 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -112,9 +112,11 @@ public final class FlattenedFieldMapper extends FieldMapper { public static final NodeFeature IGNORE_ABOVE_SUPPORT = new NodeFeature("flattened.ignore_above_support"); + public static final NodeFeature IGNORE_ABOVE_WITH_ARRAYS_SUPPORT = new NodeFeature("mapper.flattened.ignore_above_with_arrays_support"); public static final String CONTENT_TYPE = "flattened"; public static final String KEYED_FIELD_SUFFIX = "._keyed"; + public static final String KEYED_IGNORED_VALUES_FIELD_SUFFIX = "._keyed._ignored"; public static final String TIME_SERIES_DIMENSIONS_ARRAY_PARAM = "time_series_dimensions"; private static class Defaults { @@ -835,6 +837,7 @@ private FlattenedFieldMapper( this.fieldParser = new FlattenedFieldParser( mappedFieldType.name(), mappedFieldType.name() + KEYED_FIELD_SUFFIX, + mappedFieldType.name() + KEYED_IGNORED_VALUES_FIELD_SUFFIX, mappedFieldType, builder.depthLimit.get(), builder.ignoreAbove.get(), @@ -903,7 +906,12 @@ public FieldMapper.Builder getMergeBuilder() { @Override protected SyntheticSourceSupport syntheticSourceSupport() { if (fieldType().hasDocValues()) { - var loader = new FlattenedSortedSetDocValuesSyntheticFieldLoader(fullPath(), fullPath() + "._keyed", leafName()); + var loader = new FlattenedSortedSetDocValuesSyntheticFieldLoader( + fullPath(), + fullPath() + KEYED_FIELD_SUFFIX, + ignoreAbove() < Integer.MAX_VALUE ? fullPath() + KEYED_IGNORED_VALUES_FIELD_SUFFIX : null, + leafName() + ); return new SyntheticSourceSupport.Native(loader); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java index 2291d8266ed8b..351e3149da3df 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java @@ -11,6 +11,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexableField; @@ -18,11 +19,7 @@ import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocumentParserContext; -import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.XContentDataHelper; -import org.elasticsearch.xcontent.CopyingXContentParser; -import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; @@ -39,6 +36,7 @@ class FlattenedFieldParser { private final String rootFieldFullPath; private final String keyedFieldFullPath; + private final String keyedIgnoredValuesFieldFullPath; private final MappedFieldType fieldType; private final int depthLimit; @@ -48,6 +46,7 @@ class FlattenedFieldParser { FlattenedFieldParser( String rootFieldFullPath, String keyedFieldFullPath, + String keyedIgnoredValuesFieldFullPath, MappedFieldType fieldType, int depthLimit, int ignoreAbove, @@ -55,6 +54,7 @@ class FlattenedFieldParser { ) { this.rootFieldFullPath = rootFieldFullPath; this.keyedFieldFullPath = keyedFieldFullPath; + this.keyedIgnoredValuesFieldFullPath = keyedIgnoredValuesFieldFullPath; this.fieldType = fieldType; this.depthLimit = depthLimit; this.ignoreAbove = ignoreAbove; @@ -65,36 +65,18 @@ public List parse(final DocumentParserContext documentParserCont XContentParser parser = documentParserContext.parser(); XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); - XContentBuilder rawDataForSyntheticSource = null; - if (documentParserContext.canAddIgnoredField() && ignoreAbove < Integer.MAX_VALUE) { - var copyingParser = new CopyingXContentParser(parser); - rawDataForSyntheticSource = copyingParser.getBuilder(); - parser = copyingParser; - } - ContentPath path = new ContentPath(); List fields = new ArrayList<>(); var context = new Context(parser, documentParserContext); parseObject(context, path, fields); - if (rawDataForSyntheticSource != null && context.isIgnoredValueEncountered()) { - // One or more inner fields are ignored due to `ignore_above`. - // Because of that we will store whole object as is in order to generate synthetic source. - documentParserContext.addIgnoredField( - IgnoredSourceFieldMapper.NameValue.fromContext( - documentParserContext, - rootFieldFullPath, - XContentDataHelper.encodeXContentBuilder(rawDataForSyntheticSource) - ) - ); - } return fields; } private void parseObject(Context context, ContentPath path, List fields) throws IOException { String currentName = null; - XContentParser parser = context.getParser(); + XContentParser parser = context.parser(); while (true) { XContentParser.Token token = parser.nextToken(); if (token == XContentParser.Token.END_OBJECT) { @@ -111,7 +93,7 @@ private void parseObject(Context context, ContentPath path, List } private void parseArray(Context context, ContentPath path, String currentName, List fields) throws IOException { - XContentParser parser = context.getParser(); + XContentParser parser = context.parser(); while (true) { XContentParser.Token token = parser.nextToken(); if (token == XContentParser.Token.END_ARRAY) { @@ -128,7 +110,7 @@ private void parseFieldValue( String currentName, List fields ) throws IOException { - XContentParser parser = context.getParser(); + XContentParser parser = context.parser(); if (token == XContentParser.Token.START_OBJECT) { path.add(currentName); validateDepthLimit(path); @@ -151,19 +133,23 @@ private void parseFieldValue( } private void addField(Context context, ContentPath path, String currentName, String value, List fields) { - if (value.length() > ignoreAbove) { - context.onIgnoredValue(); - return; - } - String key = path.pathAsText(currentName); if (key.contains(SEPARATOR)) { throw new IllegalArgumentException( "Keys in [flattened] fields cannot contain the reserved character \\0. Offending key: [" + key + "]." ); } + String keyedValue = createKeyedValue(key, value); BytesRef bytesKeyedValue = new BytesRef(keyedValue); + + if (value.length() > ignoreAbove) { + if (context.documentParserContext().mappingLookup().isSourceSynthetic()) { + fields.add(new StoredField(keyedIgnoredValuesFieldFullPath, bytesKeyedValue)); + } + return; + } + // check the keyed value doesn't exceed the IndexWriter.MAX_TERM_LENGTH limit enforced by Lucene at index time // in that case we can already throw a more user friendly exception here which includes the offending fields key and value lengths if (bytesKeyedValue.length > IndexWriter.MAX_TERM_LENGTH) { @@ -198,10 +184,10 @@ private void addField(Context context, ContentPath path, String currentName, Str final String keyedFieldName = FlattenedFieldParser.extractKey(bytesKeyedValue).utf8ToString(); if (fieldType.isDimension() && fieldType.dimensions().contains(keyedFieldName)) { final BytesRef keyedFieldValue = FlattenedFieldParser.extractValue(bytesKeyedValue); - context.getDocumentParserContext() + context.documentParserContext() .getDimensions() .addString(rootFieldFullPath + "." + keyedFieldName, keyedFieldValue) - .validate(context.getDocumentParserContext().indexSettings()); + .validate(context.documentParserContext().indexSettings()); } } } @@ -239,32 +225,5 @@ static BytesRef extractValue(BytesRef keyedValue) { return new BytesRef(keyedValue.bytes, valueStart, keyedValue.length - valueStart); } - private static class Context { - private final XContentParser parser; - private final DocumentParserContext documentParserContext; - - private boolean ignoredValueEncountered; - - private Context(XContentParser parser, DocumentParserContext documentParserContext) { - this.parser = parser; - this.documentParserContext = documentParserContext; - this.ignoredValueEncountered = false; - } - - public XContentParser getParser() { - return parser; - } - - public DocumentParserContext getDocumentParserContext() { - return documentParserContext; - } - - public void onIgnoredValue() { - this.ignoredValueEncountered = true; - } - - public boolean isIgnoredValueEncountered() { - return ignoredValueEncountered; - } - } + private record Context(XContentParser parser, DocumentParserContext documentParserContext) {} } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelper.java index de578d724d98c..950fef95772fb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelper.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.mapper.flattened; -import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.xcontent.XContentBuilder; @@ -226,19 +225,23 @@ public boolean equals(Object obj) { } } - private final SortedSetDocValues dv; + interface SortedKeyedValues { + BytesRef next() throws IOException; + } + + private final SortedKeyedValues sortedKeyedValues; - FlattenedFieldSyntheticWriterHelper(final SortedSetDocValues dv) { - this.dv = dv; + FlattenedFieldSyntheticWriterHelper(final SortedKeyedValues sortedKeyedValues) { + this.sortedKeyedValues = sortedKeyedValues; } void write(final XContentBuilder b) throws IOException { - KeyValue curr = new KeyValue(dv.lookupOrd(dv.nextOrd())); + KeyValue curr = new KeyValue(sortedKeyedValues.next()); KeyValue prev = KeyValue.EMPTY; final List values = new ArrayList<>(); values.add(curr.value()); - for (int i = 1; i < dv.docValueCount(); i++) { - KeyValue next = new KeyValue(dv.lookupOrd(dv.nextOrd())); + for (BytesRef nextValue = sortedKeyedValues.next(); nextValue != null; nextValue = sortedKeyedValues.next()) { + KeyValue next = new KeyValue(nextValue); writeObject(b, curr, next, curr.start(prev), curr.end(next), values); values.add(next.value()); prev = curr; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedSortedSetDocValuesSyntheticFieldLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedSortedSetDocValuesSyntheticFieldLoader.java index 482273d137621..f957d7ce01902 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedSortedSetDocValuesSyntheticFieldLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedSortedSetDocValuesSyntheticFieldLoader.java @@ -13,27 +13,44 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; +import java.util.stream.Stream; -public class FlattenedSortedSetDocValuesSyntheticFieldLoader extends SourceLoader.DocValuesBasedSyntheticFieldLoader { - private DocValuesFieldValues docValues = NO_VALUES; +class FlattenedSortedSetDocValuesSyntheticFieldLoader implements SourceLoader.SyntheticFieldLoader { private final String fieldFullPath; private final String keyedFieldFullPath; + private final String keyedIgnoredValuesFieldFullPath; private final String leafName; + private DocValuesFieldValues docValues = NO_VALUES; + private List ignoredValues = List.of(); + /** * Build a loader for flattened fields from doc values. * - * @param fieldFullPath full path to the original field - * @param keyedFieldFullPath full path to the keyed field to load doc values from - * @param leafName the name of the leaf field to use in the rendered {@code _source} + * @param fieldFullPath full path to the original field + * @param keyedFieldFullPath full path to the keyed field to load doc values from + * @param keyedIgnoredValuesFieldFullPath full path to the keyed field that stores values that are not present in doc values + * due to ignore_above + * @param leafName the name of the leaf field to use in the rendered {@code _source} */ - public FlattenedSortedSetDocValuesSyntheticFieldLoader(String fieldFullPath, String keyedFieldFullPath, String leafName) { + FlattenedSortedSetDocValuesSyntheticFieldLoader( + String fieldFullPath, + String keyedFieldFullPath, + @Nullable String keyedIgnoredValuesFieldFullPath, + String leafName + ) { this.fieldFullPath = fieldFullPath; this.keyedFieldFullPath = keyedFieldFullPath; + this.keyedIgnoredValuesFieldFullPath = keyedIgnoredValuesFieldFullPath; this.leafName = leafName; } @@ -42,6 +59,18 @@ public String fieldName() { return fieldFullPath; } + @Override + public Stream> storedFieldLoaders() { + if (keyedIgnoredValuesFieldFullPath == null) { + return Stream.empty(); + } + + return Stream.of(Map.entry(keyedIgnoredValuesFieldFullPath, (values) -> { + ignoredValues = new ArrayList<>(); + ignoredValues.addAll(values); + })); + } + @Override public DocValuesLoader docValuesLoader(LeafReader reader, int[] docIdsInLeaf) throws IOException { final SortedSetDocValues dv = DocValues.getSortedSet(reader, keyedFieldFullPath); @@ -56,23 +85,40 @@ public DocValuesLoader docValuesLoader(LeafReader reader, int[] docIdsInLeaf) th @Override public boolean hasValue() { - return docValues.count() > 0; + return docValues.count() > 0 || ignoredValues.isEmpty() == false; } @Override public void write(XContentBuilder b) throws IOException { - if (docValues.count() == 0) { + if (docValues.count() == 0 && ignoredValues.isEmpty()) { return; } + + FlattenedFieldSyntheticWriterHelper.SortedKeyedValues sortedKeyedValues = new DocValuesSortedKeyedValues(docValues); + if (ignoredValues.isEmpty() == false) { + var ignoredValuesSet = new TreeSet(); + for (Object value : ignoredValues) { + ignoredValuesSet.add((BytesRef) value); + } + ignoredValues = List.of(); + sortedKeyedValues = new DocValuesWithIgnoredSortedKeyedValues(sortedKeyedValues, ignoredValuesSet); + } + var writer = new FlattenedFieldSyntheticWriterHelper(sortedKeyedValues); + b.startObject(leafName); - docValues.write(b); + writer.write(b); b.endObject(); } + @Override + public void reset() { + ignoredValues = List.of(); + } + private interface DocValuesFieldValues { int count(); - void write(XContentBuilder b) throws IOException; + SortedSetDocValues getValues(); } private static final DocValuesFieldValues NO_VALUES = new DocValuesFieldValues() { @@ -82,7 +128,9 @@ public int count() { } @Override - public void write(XContentBuilder b) {} + public SortedSetDocValues getValues() { + return null; + } }; /** @@ -92,11 +140,9 @@ public void write(XContentBuilder b) {} private static class FlattenedFieldDocValuesLoader implements DocValuesLoader, DocValuesFieldValues { private final SortedSetDocValues dv; private boolean hasValue; - private final FlattenedFieldSyntheticWriterHelper writer; FlattenedFieldDocValuesLoader(final SortedSetDocValues dv) { this.dv = dv; - this.writer = new FlattenedFieldSyntheticWriterHelper(dv); } @Override @@ -110,8 +156,74 @@ public int count() { } @Override - public void write(XContentBuilder b) throws IOException { - this.writer.write(b); + public SortedSetDocValues getValues() { + return dv; + } + } + + private static class DocValuesWithIgnoredSortedKeyedValues implements FlattenedFieldSyntheticWriterHelper.SortedKeyedValues { + private final FlattenedFieldSyntheticWriterHelper.SortedKeyedValues docValues; + private final TreeSet ignoredValues; + + private BytesRef currentFromDocValues; + + private DocValuesWithIgnoredSortedKeyedValues( + FlattenedFieldSyntheticWriterHelper.SortedKeyedValues docValues, + TreeSet ignoredValues + ) { + this.docValues = docValues; + this.ignoredValues = ignoredValues; + } + + /** + * Returns next keyed field value to be included in synthetic source. + * This function merges keyed values from doc values and ignored values (due to ignore_above) + * that are loaded from stored fields and provided as input. + * Sort order of keyed values is preserved during merge so the output is the same as if + * it was using only doc values. + * @return + * @throws IOException + */ + @Override + public BytesRef next() throws IOException { + if (currentFromDocValues == null) { + currentFromDocValues = docValues.next(); + } + + if (ignoredValues.isEmpty() == false) { + BytesRef ignoredCandidate = ignoredValues.first(); + if (currentFromDocValues == null || ignoredCandidate.compareTo(currentFromDocValues) <= 0) { + ignoredValues.pollFirst(); + return ignoredCandidate; + } + } + if (currentFromDocValues == null) { + return null; + } + + var toReturn = currentFromDocValues; + currentFromDocValues = null; + return toReturn; + } + } + + private static class DocValuesSortedKeyedValues implements FlattenedFieldSyntheticWriterHelper.SortedKeyedValues { + private final DocValuesFieldValues docValues; + private int seen = 0; + + private DocValuesSortedKeyedValues(DocValuesFieldValues docValues) { + this.docValues = docValues; + } + + @Override + public BytesRef next() throws IOException { + if (seen < docValues.count()) { + seen += 1; + var sortedSetDocValues = docValues.getValues(); + return sortedSetDocValues.lookupOrd(sortedSetDocValues.nextOrd()); + } + + return null; } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index 285431b881add..5aca2357092e4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -9,8 +9,11 @@ package org.elasticsearch.index.mapper.flattened; +import org.apache.lucene.document.Document; import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.util.BytesRef; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.bytes.BytesArray; @@ -41,7 +44,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -712,7 +718,7 @@ protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } - private static void randomMapExample(final TreeMap example, int depth, int maxDepth) { + private static void randomMapExample(final Map example, int depth, int maxDepth) { for (int i = 0; i < randomIntBetween(2, 5); i++) { int j = depth >= maxDepth ? randomIntBetween(1, 2) : randomIntBetween(1, 3); switch (j) { @@ -728,7 +734,7 @@ private static void randomMapExample(final TreeMap example, int example.put(randomAlphaOfLength(6), randomList); } case 3 -> { - final TreeMap nested = new TreeMap<>(); + final Map nested = new HashMap<>(); randomMapExample(nested, depth + 1, maxDepth); example.put(randomAlphaOfLength(10), nested); } @@ -742,11 +748,73 @@ private static class FlattenedFieldSyntheticSourceSupport implements SyntheticSo @Override public SyntheticSourceExample example(int maxValues) throws IOException { - // NOTE: values must be keywords and we use a TreeMap to preserve order (doc values are sorted and the result - // is created with keys and nested keys in sorted order). - final TreeMap map = new TreeMap<>(); - randomMapExample(map, 0, maxValues); - return new SyntheticSourceExample(map, map, this::mapping); + if (randomBoolean()) { + // Create a singleton value + var value = randomObject(); + return new SyntheticSourceExample(value, mergeIntoExpectedMap(List.of(value)), this::mapping); + } + + // Create an array of flattened field values + var values = new ArrayList>(); + for (int i = 0; i < maxValues; i++) { + values.add(randomObject()); + } + var merged = mergeIntoExpectedMap(values); + + return new SyntheticSourceExample(values, merged, this::mapping); + } + + private Map randomObject() { + var maxDepth = randomIntBetween(1, 3); + + final Map map = new HashMap<>(); + randomMapExample(map, 0, maxDepth); + + return map; + } + + // Since arrays are moved to leafs in synthetic source, the result is not an array of objects + // but one big object containing merged values from all input objects. + // This function performs that transformation. + private Map mergeIntoExpectedMap(List> inputValues) { + // Fields are sorted since they come (mostly) from doc_values. + var result = new TreeMap(); + doMerge(inputValues, result); + return result; + } + + @SuppressWarnings("unchecked") + private void doMerge(List> inputValues, TreeMap result) { + for (var iv : inputValues) { + for (var field : iv.entrySet()) { + if (field.getValue() instanceof Map inputNestedMap) { + var intermediateResultMap = result.get(field.getKey()); + if (intermediateResultMap == null) { + var map = new TreeMap(); + + result.put(field.getKey(), map); + doMerge(List.of((Map) inputNestedMap), map); + } else if (intermediateResultMap instanceof Map m) { + doMerge(List.of((Map) inputNestedMap), (TreeMap) m); + } else { + throw new IllegalStateException("Conflicting entries in merged map"); + } + } else { + var valueAtCurrentLevel = result.get(field.getKey()); + if (valueAtCurrentLevel == null) { + result.put(field.getKey(), field.getValue()); + } else if (valueAtCurrentLevel instanceof List) { + ((List) valueAtCurrentLevel).add(field.getValue()); + } else { + var list = new ArrayList<>(); + list.add(valueAtCurrentLevel); + list.add(field.getValue()); + + result.put(field.getKey(), list); + } + } + } + } } @Override @@ -762,8 +830,57 @@ private void mapping(XContentBuilder b) throws IOException { } } + public void testSyntheticSourceWithOnlyIgnoredValues() throws IOException { + DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { + b.startObject("field").field("type", "flattened").field("ignore_above", 1).endObject(); + })); + + var syntheticSource = syntheticSource(mapper, b -> { + b.startObject("field"); + { + b.field("key1", "val1"); + b.startObject("obj1"); + { + b.field("key2", "val2"); + b.field("key3", List.of("val3", "val4")); + } + b.endObject(); + } + b.endObject(); + }); + assertThat(syntheticSource, equalTo("{\"field\":{\"key1\":\"val1\",\"obj1\":{\"key2\":\"val2\",\"key3\":[\"val3\",\"val4\"]}}}")); + } + @Override protected boolean supportsCopyTo() { return false; } + + @Override + public void assertStoredFieldsEquals(String info, IndexReader leftReader, IndexReader rightReader) throws IOException { + assert leftReader.maxDoc() == rightReader.maxDoc(); + StoredFields leftStoredFields = leftReader.storedFields(); + StoredFields rightStoredFields = rightReader.storedFields(); + for (int i = 0; i < leftReader.maxDoc(); i++) { + Document leftDoc = leftStoredFields.document(i); + Document rightDoc = rightStoredFields.document(i); + + // Everything is from LuceneTestCase except this part. + // LuceneTestCase sorts by name of the field only which results in a difference + // between keyed ignored field values that have the same name. + Comparator comp = Comparator.comparing(IndexableField::name).thenComparing(IndexableField::binaryValue); + List leftFields = new ArrayList<>(leftDoc.getFields()); + List rightFields = new ArrayList<>(rightDoc.getFields()); + Collections.sort(leftFields, comp); + Collections.sort(rightFields, comp); + + Iterator leftIterator = leftFields.iterator(); + Iterator rightIterator = rightFields.iterator(); + while (leftIterator.hasNext()) { + assertTrue(info, rightIterator.hasNext()); + assertStoredFieldEquals(info, leftIterator.next(), rightIterator.next()); + } + assertFalse(info, rightIterator.hasNext()); + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParserTests.java index 736c877eff6c5..68be241ca1885 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParserTests.java @@ -33,7 +33,15 @@ public class FlattenedFieldParserTests extends ESTestCase { @Before public void setUp() throws Exception { super.setUp(); - parser = new FlattenedFieldParser("field", "field._keyed", new FakeFieldType("field"), Integer.MAX_VALUE, Integer.MAX_VALUE, null); + parser = new FlattenedFieldParser( + "field", + "field._keyed", + "field._keyed._ignored", + new FakeFieldType("field"), + Integer.MAX_VALUE, + Integer.MAX_VALUE, + null + ); } public void testTextValues() throws Exception { @@ -283,6 +291,7 @@ public void testDepthLimit() throws Exception { FlattenedFieldParser configuredParser = new FlattenedFieldParser( "field", "field._keyed", + "field._keyed._ignored", new FakeFieldType("field"), 2, Integer.MAX_VALUE, @@ -306,6 +315,7 @@ public void testDepthLimitBoundary() throws Exception { FlattenedFieldParser configuredParser = new FlattenedFieldParser( "field", "field._keyed", + "field._keyed._ignored", new FakeFieldType("field"), 3, Integer.MAX_VALUE, @@ -323,6 +333,7 @@ public void testIgnoreAbove() throws Exception { FlattenedFieldParser configuredParser = new FlattenedFieldParser( "field", "field._keyed", + "field._keyed._ignored", new FakeFieldType("field"), Integer.MAX_VALUE, 10, @@ -345,6 +356,7 @@ public void testNullValues() throws Exception { FlattenedFieldParser configuredParser = new FlattenedFieldParser( "field", "field._keyed", + "field._keyed._ignored", fieldType, Integer.MAX_VALUE, Integer.MAX_VALUE, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelperTests.java index c04766ad57112..71e31d1ff371d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelperTests.java @@ -35,7 +35,7 @@ public void testSingleField() throws IOException { byte[] bytes = ("test" + '\0' + "one").getBytes(StandardCharsets.UTF_8); when(dv.nextOrd()).thenReturn(0L); when(dv.lookupOrd(0L)).thenReturn(new BytesRef(bytes, 0, bytes.length)); - FlattenedFieldSyntheticWriterHelper writer = new FlattenedFieldSyntheticWriterHelper(dv); + FlattenedFieldSyntheticWriterHelper writer = new FlattenedFieldSyntheticWriterHelper(new SortedSetSortedKeyedValues(dv)); ByteArrayOutputStream baos = new ByteArrayOutputStream(); XContentBuilder b = new XContentBuilder(XContentType.JSON.xContent(), baos); @@ -52,7 +52,7 @@ public void testSingleField() throws IOException { public void testFlatObject() throws IOException { // GIVEN final SortedSetDocValues dv = mock(SortedSetDocValues.class); - final FlattenedFieldSyntheticWriterHelper writer = new FlattenedFieldSyntheticWriterHelper(dv); + final FlattenedFieldSyntheticWriterHelper writer = new FlattenedFieldSyntheticWriterHelper(new SortedSetSortedKeyedValues(dv)); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); final XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), baos); final List bytes = List.of("a" + '\0' + "value_a", "b" + '\0' + "value_b", "c" + '\0' + "value_c", "d" + '\0' + "value_d") @@ -79,7 +79,7 @@ public void testFlatObject() throws IOException { public void testSingleObject() throws IOException { // GIVEN final SortedSetDocValues dv = mock(SortedSetDocValues.class); - final FlattenedFieldSyntheticWriterHelper writer = new FlattenedFieldSyntheticWriterHelper(dv); + final FlattenedFieldSyntheticWriterHelper writer = new FlattenedFieldSyntheticWriterHelper(new SortedSetSortedKeyedValues(dv)); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); final XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), baos); final List bytes = List.of( @@ -111,7 +111,7 @@ public void testSingleObject() throws IOException { public void testMultipleObjects() throws IOException { // GIVEN final SortedSetDocValues dv = mock(SortedSetDocValues.class); - final FlattenedFieldSyntheticWriterHelper writer = new FlattenedFieldSyntheticWriterHelper(dv); + final FlattenedFieldSyntheticWriterHelper writer = new FlattenedFieldSyntheticWriterHelper(new SortedSetSortedKeyedValues(dv)); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); final XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), baos); final List bytes = List.of("a.x" + '\0' + "10", "a.y" + '\0' + "20", "b.a" + '\0' + "30", "b.c" + '\0' + "40") @@ -138,7 +138,7 @@ public void testMultipleObjects() throws IOException { public void testSingleArray() throws IOException { // GIVEN final SortedSetDocValues dv = mock(SortedSetDocValues.class); - final FlattenedFieldSyntheticWriterHelper writer = new FlattenedFieldSyntheticWriterHelper(dv); + final FlattenedFieldSyntheticWriterHelper writer = new FlattenedFieldSyntheticWriterHelper(new SortedSetSortedKeyedValues(dv)); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); final XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), baos); final List bytes = List.of("a.x" + '\0' + "10", "a.x" + '\0' + "20", "a.x" + '\0' + "30", "a.x" + '\0' + "40") @@ -165,7 +165,7 @@ public void testSingleArray() throws IOException { public void testMultipleArrays() throws IOException { // GIVEN final SortedSetDocValues dv = mock(SortedSetDocValues.class); - final FlattenedFieldSyntheticWriterHelper writer = new FlattenedFieldSyntheticWriterHelper(dv); + final FlattenedFieldSyntheticWriterHelper writer = new FlattenedFieldSyntheticWriterHelper(new SortedSetSortedKeyedValues(dv)); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); final XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), baos); final List bytes = List.of( @@ -191,4 +191,23 @@ public void testMultipleArrays() throws IOException { // THEN assertEquals("{\"a\":{\"x\":[\"10\",\"20\"]},\"b\":{\"y\":[\"30\",\"40\",\"50\"]}}", baos.toString(StandardCharsets.UTF_8)); } + + private class SortedSetSortedKeyedValues implements FlattenedFieldSyntheticWriterHelper.SortedKeyedValues { + private final SortedSetDocValues dv; + private int seen = 0; + + private SortedSetSortedKeyedValues(SortedSetDocValues dv) { + this.dv = dv; + } + + @Override + public BytesRef next() throws IOException { + if (seen < dv.docValueCount()) { + seen += 1; + return dv.lookupOrd(dv.nextOrd()); + } + + return null; + } + } } From 5e06092d5eae6550a030449f2a1f04e67fafb494 Mon Sep 17 00:00:00 2001 From: Stanislav Malyshev Date: Wed, 25 Sep 2024 15:55:30 -0600 Subject: [PATCH 04/36] Improve DateTime error handling and add some bad date tests (#112723) * Improve DateTime error handling and add some bad date tests --- docs/changelog/112723.yaml | 6 +++ .../common/time/JavaDateMathParser.java | 4 +- .../index/mapper/DateFieldTypeTests.java | 41 +++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/112723.yaml diff --git a/docs/changelog/112723.yaml b/docs/changelog/112723.yaml new file mode 100644 index 0000000000000..dbee3232d1c75 --- /dev/null +++ b/docs/changelog/112723.yaml @@ -0,0 +1,6 @@ +pr: 112723 +summary: Improve DateTime error handling and add some bad date tests +area: Search +type: bug +issues: + - 112190 diff --git a/server/src/main/java/org/elasticsearch/common/time/JavaDateMathParser.java b/server/src/main/java/org/elasticsearch/common/time/JavaDateMathParser.java index b5eb2efe2e06c..0ee0b34da3a5c 100644 --- a/server/src/main/java/org/elasticsearch/common/time/JavaDateMathParser.java +++ b/server/src/main/java/org/elasticsearch/common/time/JavaDateMathParser.java @@ -12,13 +12,13 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Strings; +import java.time.DateTimeException; import java.time.DayOfWeek; import java.time.Instant; import java.time.LocalTime; import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; -import java.time.format.DateTimeParseException; import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; import java.time.temporal.TemporalAdjusters; @@ -220,7 +220,7 @@ private Instant parseDateTime(String value, ZoneId timeZone, boolean roundUpIfNo return DateFormatters.from(accessor).withZoneSameLocal(timeZone).toInstant(); } - } catch (IllegalArgumentException | DateTimeParseException e) { + } catch (IllegalArgumentException | DateTimeException e) { throw new ElasticsearchParseException( "failed to parse date field [{}] with format [{}]: [{}]", e, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index f22681138378f..d925a9dd1d691 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -22,6 +22,7 @@ import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; @@ -143,6 +144,46 @@ private void doTestIsFieldWithinQuery(DateFieldType ft, DirectoryReader reader, assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-12", "2016-04-03", false, false, zone, null, context)); assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-12", "2016-04-03", false, true, zone, null, context)); assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-12", "2016-04-03", true, false, zone, null, context)); + // Bad dates + assertThrows( + ElasticsearchParseException.class, + () -> ft.isFieldWithinQuery(reader, "2015-00-01", "2016-04-03", randomBoolean(), randomBoolean(), zone, null, context) + ); + assertThrows( + ElasticsearchParseException.class, + () -> ft.isFieldWithinQuery(reader, "2015-01-01", "2016-04-00", randomBoolean(), randomBoolean(), zone, null, context) + ); + assertThrows( + ElasticsearchParseException.class, + () -> ft.isFieldWithinQuery(reader, "2015-22-01", "2016-04-00", randomBoolean(), randomBoolean(), zone, null, context) + ); + assertThrows( + ElasticsearchParseException.class, + () -> ft.isFieldWithinQuery(reader, "2015-01-01", "2016-04-45", randomBoolean(), randomBoolean(), zone, null, context) + ); + assertThrows( + ElasticsearchParseException.class, + () -> ft.isFieldWithinQuery(reader, "2015-01-01", "2016-04-01T25:00:00", randomBoolean(), randomBoolean(), zone, null, context) + ); + if (ft.resolution().equals(Resolution.NANOSECONDS)) { + assertThrows( + IllegalArgumentException.class, + () -> ft.isFieldWithinQuery(reader, "-2016-04-01", "2016-04-01", randomBoolean(), randomBoolean(), zone, null, context) + ); + assertThrows( + IllegalArgumentException.class, + () -> ft.isFieldWithinQuery( + reader, + "9223372036854775807", + "2016-04-01", + randomBoolean(), + randomBoolean(), + zone, + null, + context + ) + ); + } } public void testValueFormat() { From d19a9cc6f1f003eb6eb40ffd3b66756a3b9ae6b0 Mon Sep 17 00:00:00 2001 From: Carlos Delgado <6339205+carlosdelest@users.noreply.github.com> Date: Thu, 26 Sep 2024 08:12:34 +0200 Subject: [PATCH 05/36] Fix release tests by checking QSTR function is registered (#113527) --- muted-tests.yml | 2 -- .../function/fulltext/QueryStringFunctionTests.java | 7 +++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 07a31dce9047c..7c31529b5b716 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -290,8 +290,6 @@ tests: - class: org.elasticsearch.validation.DotPrefixClientYamlTestSuiteIT method: test {p0=dot_prefix/10_basic/Deprecated index template with a dot prefix index pattern} issue: https://github.com/elastic/elasticsearch/issues/113529 -- class: org.elasticsearch.xpack.esql.expression.function.fulltext.QueryStringFunctionTests - issue: https://github.com/elastic/elasticsearch/issues/113496 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates} issue: https://github.com/elastic/elasticsearch/issues/113537 diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java index e622ff5ba2579..37e16a2499cd9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java @@ -18,17 +18,24 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionName; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.hamcrest.Matcher; +import org.junit.BeforeClass; import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.function.Supplier; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.QSTR_FUNCTION; import static org.hamcrest.Matchers.equalTo; @FunctionName("qstr") public class QueryStringFunctionTests extends AbstractFunctionTestCase { + @BeforeClass + public static void checkFunctionEnabled() { + assumeTrue("QSTR capability should be enabled ", QSTR_FUNCTION.isEnabled()); + } + public QueryStringFunctionTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } From fb4ebfb03b89cbb9b4e4fa0e631bca5ac497cd44 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:34:03 +1000 Subject: [PATCH 06/36] Mute org.elasticsearch.xpack.inference.TextEmbeddingCrudIT testPutE5Small_withPlatformAgnosticVariant #113577 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 7c31529b5b716..9589746c7db47 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -302,6 +302,9 @@ tests: - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT method: testPutE5WithTrainedModelAndInference issue: https://github.com/elastic/elasticsearch/issues/113565 +- class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT + method: testPutE5Small_withPlatformAgnosticVariant + issue: https://github.com/elastic/elasticsearch/issues/113577 # Examples: # From 55ef5e6ba13c95dd6eef8811181e2391ea9d14d0 Mon Sep 17 00:00:00 2001 From: Gergely Kalapos Date: Thu, 26 Sep 2024 08:36:58 +0200 Subject: [PATCH 07/36] Remove dynamic templates from otel-plugin that set `index:false` (#113409) * Remove dynamic templates from otel-plugin that set `index:false` * Update x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml Co-authored-by: Felix Barnsteiner * Remove unused dynamic templates * Update metrics-otel@mappings.yaml --------- Co-authored-by: Felix Barnsteiner Co-authored-by: Elastic Machine --- .../src/main/resources/metrics@mappings.json | 44 ------------------- .../metrics-otel@mappings.yaml | 30 +++++++++++++ .../metrics-otel@template.yaml | 2 - .../rest-api-spec/test/20_metrics_tests.yml | 18 ++++++++ 4 files changed, 48 insertions(+), 46 deletions(-) diff --git a/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json index 9c58322f12d03..b4aa999697632 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json @@ -43,50 +43,6 @@ "default_metric": "value_count" } } - }, - { - "histogram": { - "mapping": { - "type": "histogram", - "ignore_malformed": true - } - } - }, - { - "counter_long": { - "mapping": { - "type": "long", - "time_series_metric": "counter", - "ignore_malformed": true - } - } - }, - { - "gauge_long": { - "mapping": { - "type": "long", - "time_series_metric": "gauge", - "ignore_malformed": true - } - } - }, - { - "counter_double": { - "mapping": { - "type": "double", - "time_series_metric": "counter", - "ignore_malformed": true - } - } - }, - { - "gauge_double": { - "mapping": { - "type": "double", - "time_series_metric": "gauge", - "ignore_malformed": true - } - } } ], "properties": { diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml index b7a17dba973f8..8594c4962d96d 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml @@ -15,3 +15,33 @@ template: type: keyword time_series_dimension: true ignore_above: 1024 + dynamic_templates: + - histogram: + mapping: + type: histogram + ignore_malformed: true + - counter_long: + mapping: + type: long + time_series_metric: counter + ignore_malformed: true + - gauge_long: + mapping: + type: long + time_series_metric: gauge + ignore_malformed: true + - counter_double: + mapping: + type: double + time_series_metric: counter + ignore_malformed: true + - gauge_double: + mapping: + type: double + time_series_metric: gauge + ignore_malformed: true + - summary: + mapping: + type: aggregate_metric_double + metrics: sum, value_count + default_metric: value_count diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml index a4413d266181d..c2a318f809b7d 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml @@ -8,7 +8,6 @@ _meta: description: default OpenTelemetry metrics template installed by x-pack managed: true composed_of: - - metrics@mappings - metrics@tsdb-settings - otel@mappings - metrics-otel@mappings @@ -38,4 +37,3 @@ template: ignore_above: 1024 type: keyword match_mapping_type: string - diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml index 1823dfab7e716..46af03ee85467 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml @@ -178,3 +178,21 @@ IP dimensions: expand_wildcards: hidden - match: { .$idx0name.mappings.properties.resource.properties.attributes.properties.host\.ip.type: 'ip' } - match: { .$idx0name.mappings.properties.attributes.properties.philip.type: "keyword" } +--- +"Long data type in attributes must be accepted": + - do: + indices.get_index_template: + name: metrics-otel@template + - length: {index_templates: 1} + - do: + bulk: + index: metrics-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","attributes":{"processor.pid": 17}}' + - is_false: errors + - do: + search: + index: metrics-generic.otel-default + - length: { hits.hits: 1 } From 44a5ce379808d4dcf235bb779e3dfdb0947092fb Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 26 Sep 2024 08:17:04 +0100 Subject: [PATCH 08/36] Expose `Connection` to remote clusters (#113453) Sometimes we might need to invoke different requests on a remote cluster depending on the version of the transport protocol it understands, but today we cannot make that distinction (without starting to execute an action on the remote cluster and failing while serializing the request at least). This commit allows callers access to the underlying `Transport.Connection` instance so that we can implement better BwC logic. --- .../internal/ParentTaskAssigningClient.java | 9 +- .../client/internal/RemoteClusterClient.java | 29 +++-- .../transport/RemoteClusterAwareClient.java | 66 +++++++----- .../ParentTaskAssigningClientTests.java | 35 ++++++ .../transport/RemoteClusterClientTests.java | 101 ++++++++++++------ ...rectToLocalClusterRemoteClusterClient.java | 25 ++++- .../xpack/ccr/CcrLicenseChecker.java | 17 ++- .../DefaultCheckpointProviderTests.java | 37 ++++++- 8 files changed, 240 insertions(+), 79 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/client/internal/ParentTaskAssigningClient.java b/server/src/main/java/org/elasticsearch/client/internal/ParentTaskAssigningClient.java index 8e90a459bcafd..014fef8bdfe56 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/ParentTaskAssigningClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/ParentTaskAssigningClient.java @@ -18,6 +18,7 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.transport.RemoteClusterService; +import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportResponse; import java.util.concurrent.Executor; @@ -75,12 +76,18 @@ public RemoteClusterClient getRemoteClusterClient( return new RemoteClusterClient() { @Override public void execute( + Transport.Connection connection, RemoteClusterActionType action, Request request, ActionListener listener ) { request.setParentTask(parentTask); - delegate.execute(action, request, listener); + delegate.execute(connection, action, request, listener); + } + + @Override + public void getConnection(Request request, ActionListener listener) { + delegate.getConnection(request, listener); } }; } diff --git a/server/src/main/java/org/elasticsearch/client/internal/RemoteClusterClient.java b/server/src/main/java/org/elasticsearch/client/internal/RemoteClusterClient.java index 56ab07d26b4c7..9e3497601fb57 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/RemoteClusterClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/RemoteClusterClient.java @@ -13,6 +13,8 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.RemoteClusterActionType; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportResponse; /** @@ -21,16 +23,31 @@ public interface RemoteClusterClient { /** * Executes an action, denoted by an {@link ActionType}, on the remote cluster. - * - * @param action The action type to execute. - * @param request The action request. - * @param listener A listener for the response - * @param The request type. - * @param the response type. + */ + default void execute( + RemoteClusterActionType action, + Request request, + ActionListener listener + ) { + getConnection( + request, + listener.delegateFailureAndWrap((responseListener, connection) -> execute(connection, action, request, responseListener)) + ); + } + + /** + * Executes an action, denoted by an {@link ActionType}, using a connection to the remote cluster obtained using {@link #getConnection}. */ void execute( + Transport.Connection connection, RemoteClusterActionType action, Request request, ActionListener listener ); + + /** + * Obtain a connection to the remote cluster for use with the {@link #execute} override that allows to specify the connection. Useful + * for cases where you need to inspect {@link Transport.Connection#getVersion} before deciding the exact remote action to invoke. + */ + void getConnection(@Nullable Request request, ActionListener listener); } diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAwareClient.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAwareClient.java index 126393b688d5d..c21cd4dd2f714 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAwareClient.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAwareClient.java @@ -12,8 +12,9 @@ import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.RemoteClusterActionType; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.client.internal.RemoteClusterClient; -import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.core.Nullable; import java.util.concurrent.Executor; @@ -35,41 +36,48 @@ final class RemoteClusterAwareClient implements RemoteClusterClient { @Override public void execute( + Transport.Connection connection, RemoteClusterActionType action, Request request, ActionListener listener ) { - maybeEnsureConnected(listener.delegateFailureAndWrap((delegateListener, v) -> { - final Transport.Connection connection; - try { - if (request instanceof RemoteClusterAwareRequest) { - DiscoveryNode preferredTargetNode = ((RemoteClusterAwareRequest) request).getPreferredTargetNode(); - connection = remoteClusterService.getConnection(preferredTargetNode, clusterAlias); + service.sendRequest( + connection, + action.name(), + request, + TransportRequestOptions.EMPTY, + new ActionListenerResponseHandler<>(listener, action.getResponseReader(), responseExecutor) + ); + } + + @Override + public void getConnection(@Nullable Request request, ActionListener listener) { + SubscribableListener + + .newForked(ensureConnectedListener -> { + if (ensureConnected) { + remoteClusterService.ensureConnected(clusterAlias, ensureConnectedListener); } else { - connection = remoteClusterService.getConnection(clusterAlias); + ensureConnectedListener.onResponse(null); } - } catch (ConnectTransportException e) { - if (ensureConnected == false) { - // trigger another connection attempt, but don't wait for it to complete - remoteClusterService.ensureConnected(clusterAlias, ActionListener.noop()); + }) + + .andThenApply(ignored -> { + try { + if (request instanceof RemoteClusterAwareRequest remoteClusterAwareRequest) { + return remoteClusterService.getConnection(remoteClusterAwareRequest.getPreferredTargetNode(), clusterAlias); + } else { + return remoteClusterService.getConnection(clusterAlias); + } + } catch (ConnectTransportException e) { + if (ensureConnected == false) { + // trigger another connection attempt, but don't wait for it to complete + remoteClusterService.ensureConnected(clusterAlias, ActionListener.noop()); + } + throw e; } - throw e; - } - service.sendRequest( - connection, - action.name(), - request, - TransportRequestOptions.EMPTY, - new ActionListenerResponseHandler<>(delegateListener, action.getResponseReader(), responseExecutor) - ); - })); - } + }) - private void maybeEnsureConnected(ActionListener ensureConnectedListener) { - if (ensureConnected) { - ActionListener.run(ensureConnectedListener, l -> remoteClusterService.ensureConnected(clusterAlias, l)); - } else { - ensureConnectedListener.onResponse(null); - } + .addListener(listener); } } diff --git a/server/src/test/java/org/elasticsearch/client/internal/ParentTaskAssigningClientTests.java b/server/src/test/java/org/elasticsearch/client/internal/ParentTaskAssigningClientTests.java index 143858625ca76..b067509f4668c 100644 --- a/server/src/test/java/org/elasticsearch/client/internal/ParentTaskAssigningClientTests.java +++ b/server/src/test/java/org/elasticsearch/client/internal/ParentTaskAssigningClientTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.transport.RemoteClusterService; +import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportResponse; import java.util.concurrent.Executor; @@ -83,6 +84,24 @@ public void assertSame(parentTaskId, request.getParentTask()); listener.onFailure(new UnsupportedOperationException("fake remote-cluster client")); } + + @Override + public void execute( + Transport.Connection connection, + RemoteClusterActionType action, + Request request, + ActionListener listener + ) { + execute(action, request, listener); + } + + @Override + public void getConnection( + Request request, + ActionListener listener + ) { + listener.onResponse(null); + } }; } }; @@ -107,6 +126,22 @@ public void ) ).getMessage() ); + + assertEquals( + "fake remote-cluster client", + asInstanceOf( + UnsupportedOperationException.class, + safeAwaitFailure( + ClusterStateResponse.class, + listener -> remoteClusterClient.execute( + null, + ClusterStateAction.REMOTE_TYPE, + new ClusterStateRequest(TEST_REQUEST_TIMEOUT), + listener + ) + ) + ).getMessage() + ); } } } diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java index 9ea75f060a30d..985fd6e10445d 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java @@ -17,7 +17,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.search.TransportSearchScrollAction; -import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; @@ -103,13 +103,25 @@ public void testConnectAndExecuteRequest() throws Exception { randomFrom(RemoteClusterService.DisconnectedStrategy.values()) ); ClusterStateResponse clusterStateResponse = safeAwait( - listener -> client.execute( - ClusterStateAction.REMOTE_TYPE, - new ClusterStateRequest(TEST_REQUEST_TIMEOUT), + listener -> ActionListener.run( ActionListener.runBefore( listener, () -> assertTrue(Thread.currentThread().getName().contains('[' + TEST_THREAD_POOL_NAME + ']')) - ) + ), + clusterStateResponseListener -> { + final var request = new ClusterStateRequest(TEST_REQUEST_TIMEOUT); + if (randomBoolean()) { + client.execute(ClusterStateAction.REMOTE_TYPE, request, clusterStateResponseListener); + } else { + SubscribableListener.newForked( + l -> client.getConnection(randomFrom(request, null), l) + ) + .andThen( + (l, connection) -> client.execute(connection, ClusterStateAction.REMOTE_TYPE, request, l) + ) + .addListener(clusterStateResponseListener); + } + } ) ); assertNotNull(clusterStateResponse); @@ -169,12 +181,13 @@ public void testEnsureWeReconnect() throws Exception { for (int i = 0; i < 10; i++) { RemoteClusterConnection remoteClusterConnection = remoteClusterService.getRemoteClusterConnection("test"); assertBusy(remoteClusterConnection::assertNoRunningConnections); - ConnectionManager connectionManager = remoteClusterConnection.getConnectionManager(); - Transport.Connection connection = connectionManager.getConnection(remoteNode); - PlainActionFuture closeFuture = new PlainActionFuture<>(); - connection.addCloseListener(closeFuture); - connectionManager.disconnectFromNode(remoteNode); - closeFuture.get(); + + safeAwait(connectionClosedListener -> { + ConnectionManager connectionManager = remoteClusterConnection.getConnectionManager(); + Transport.Connection connection = connectionManager.getConnection(remoteNode); + connection.addCloseListener(connectionClosedListener.map(v -> v)); + connectionManager.disconnectFromNode(remoteNode); + }); var client = remoteClusterService.getRemoteClusterClient( "test", @@ -184,11 +197,21 @@ public void testEnsureWeReconnect() throws Exception { RemoteClusterService.DisconnectedStrategy.RECONNECT_UNLESS_SKIP_UNAVAILABLE ) ); - ClusterStateResponse clusterStateResponse = safeAwait( - listener -> client.execute(ClusterStateAction.REMOTE_TYPE, new ClusterStateRequest(TEST_REQUEST_TIMEOUT), listener) - ); - assertNotNull(clusterStateResponse); - assertEquals("foo_bar_cluster", clusterStateResponse.getState().getClusterName().value()); + + if (randomBoolean()) { + final ClusterStateResponse clusterStateResponse = safeAwait( + listener -> client.execute( + ClusterStateAction.REMOTE_TYPE, + new ClusterStateRequest(TEST_REQUEST_TIMEOUT), + listener + ) + ); + assertNotNull(clusterStateResponse); + assertEquals("foo_bar_cluster", clusterStateResponse.getState().getClusterName().value()); + } else { + final Transport.Connection connection = safeAwait(listener -> client.getConnection(null, listener)); + assertFalse(connection.isClosed()); + } assertTrue(remoteClusterConnection.isNodeConnected(remoteNode)); } } @@ -271,28 +294,42 @@ public void testQuicklySkipUnavailableClusters() throws Exception { assertFalse(remoteClusterService.isRemoteNodeConnected("test", remoteNode)); // check that we quickly fail - ESTestCase.assertThat( - safeAwaitFailure( - ClusterStateResponse.class, - listener -> client.execute( - ClusterStateAction.REMOTE_TYPE, - new ClusterStateRequest(TEST_REQUEST_TIMEOUT), - listener - ) - ), - instanceOf(ConnectTransportException.class) - ); + if (randomBoolean()) { + ESTestCase.assertThat( + safeAwaitFailure( + ClusterStateResponse.class, + listener -> client.execute( + ClusterStateAction.REMOTE_TYPE, + new ClusterStateRequest(TEST_REQUEST_TIMEOUT), + listener + ) + ), + instanceOf(ConnectTransportException.class) + ); + } else { + ESTestCase.assertThat( + safeAwaitFailure(Transport.Connection.class, listener -> client.getConnection(null, listener)), + instanceOf(ConnectTransportException.class) + ); + } } finally { service.clearAllRules(); latch.countDown(); } - assertBusy(() -> { - ClusterStateResponse ignored = safeAwait( - listener -> client.execute(ClusterStateAction.REMOTE_TYPE, new ClusterStateRequest(TEST_REQUEST_TIMEOUT), listener) - ); + assertBusy( // keep retrying on an exception, the goal is to check that we eventually reconnect - }); + randomFrom( + () -> safeAwait( + listener -> client.execute( + ClusterStateAction.REMOTE_TYPE, + new ClusterStateRequest(TEST_REQUEST_TIMEOUT), + listener.map(v -> v) + ) + ), + () -> safeAwait(listener -> client.getConnection(null, listener.map(v -> v))) + ) + ); assertTrue(remoteClusterService.isRemoteNodeConnected("test", remoteNode)); } } diff --git a/test/framework/src/main/java/org/elasticsearch/client/internal/RedirectToLocalClusterRemoteClusterClient.java b/test/framework/src/main/java/org/elasticsearch/client/internal/RedirectToLocalClusterRemoteClusterClient.java index cb2b2a78c1bd6..ff3910b2debfb 100644 --- a/test/framework/src/main/java/org/elasticsearch/client/internal/RedirectToLocalClusterRemoteClusterClient.java +++ b/test/framework/src/main/java/org/elasticsearch/client/internal/RedirectToLocalClusterRemoteClusterClient.java @@ -11,10 +11,10 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.RemoteClusterActionType; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportResponse; /** @@ -22,10 +22,10 @@ */ public class RedirectToLocalClusterRemoteClusterClient implements RemoteClusterClient { - private final ElasticsearchClient delegate; + private final ElasticsearchClient localNodeClient; - public RedirectToLocalClusterRemoteClusterClient(ElasticsearchClient delegate) { - this.delegate = delegate; + public RedirectToLocalClusterRemoteClusterClient(ElasticsearchClient localNodeClient) { + this.localNodeClient = localNodeClient; } @SuppressWarnings("unchecked") @@ -35,6 +35,21 @@ public void Request request, ActionListener listener ) { - delegate.execute(new ActionType(action.name()), request, listener.map(r -> (Response) r)); + localNodeClient.execute(new ActionType<>(action.name()), request, listener.map(r -> (Response) r)); + } + + @Override + public void execute( + Transport.Connection connection, + RemoteClusterActionType action, + Request request, + ActionListener listener + ) { + throw new AssertionError("not implemented on RedirectToLocalClusterRemoteClusterClient"); + } + + @Override + public void getConnection(Request request, ActionListener listener) { + throw new AssertionError("not implemented on RedirectToLocalClusterRemoteClusterClient"); } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java index b4607e002f27e..2c633a43264f6 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java @@ -43,6 +43,7 @@ import org.elasticsearch.license.RemoteClusterLicenseChecker; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.transport.RemoteClusterService; +import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.xpack.ccr.action.CcrRequests; import org.elasticsearch.xpack.ccr.action.ShardChangesAction; @@ -424,6 +425,7 @@ public static RemoteClusterClient wrapRemoteClusterClient( return new RemoteClusterClient() { @Override public void execute( + Transport.Connection connection, RemoteClusterActionType action, Request request, ActionListener listener @@ -434,9 +436,14 @@ public void null, request, listener, - (r, l) -> client.execute(action, r, l) + (r, l) -> client.execute(connection, action, r, l) ); } + + @Override + public void getConnection(Request request, ActionListener listener) { + client.getConnection(request, listener); + } }; } } @@ -466,6 +473,7 @@ private static RemoteClusterClient systemClient(ThreadContext threadContext, Rem return new RemoteClusterClient() { @Override public void execute( + Transport.Connection connection, RemoteClusterActionType action, Request request, ActionListener listener @@ -473,9 +481,14 @@ public void final Supplier supplier = threadContext.newRestorableContext(false); try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { threadContext.markAsSystemContext(); - delegate.execute(action, request, new ContextPreservingActionListener<>(supplier, listener)); + delegate.execute(connection, action, request, new ContextPreservingActionListener<>(supplier, listener)); } } + + @Override + public void getConnection(Request request, ActionListener listener) { + delegate.getConnection(request, listener); + } }; } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/DefaultCheckpointProviderTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/DefaultCheckpointProviderTests.java index 1c38ed50ede39..3df47fb3bc066 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/DefaultCheckpointProviderTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/DefaultCheckpointProviderTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.test.MockLog.LoggingExpectation; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.ActionNotFoundTransportException; +import org.elasticsearch.transport.Transport; import org.elasticsearch.xpack.core.transform.action.GetCheckpointAction; import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint; @@ -358,13 +359,18 @@ public void testCreateNextCheckpointWithRemoteClient() throws InterruptedExcepti String transformId = getTestName(); TransformConfig transformConfig = TransformConfigTests.randomTransformConfig(transformId); + doAnswer(withMockConnection()).when(remoteClient1).getConnection(any(), any()); + doAnswer(withMockConnection()).when(remoteClient2).getConnection(any(), any()); + doAnswer(withMockConnection()).when(remoteClient3).getConnection(any(), any()); + GetCheckpointAction.Response checkpointResponse = new GetCheckpointAction.Response(Map.of("index-1", new long[] { 1L, 2L, 3L })); doAnswer(withResponse(checkpointResponse)).when(client).execute(eq(GetCheckpointAction.INSTANCE), any(), any()); GetCheckpointAction.Response remoteCheckpointResponse = new GetCheckpointAction.Response( Map.of("index-1", new long[] { 4L, 5L, 6L, 7L, 8L }) ); - doAnswer(withResponse(remoteCheckpointResponse)).when(remoteClient1).execute(eq(GetCheckpointAction.REMOTE_TYPE), any(), any()); + doAnswer(withRemoteResponse(remoteCheckpointResponse)).when(remoteClient1) + .execute(any(), eq(GetCheckpointAction.REMOTE_TYPE), any(), any()); RemoteClusterResolver remoteClusterResolver = mock(RemoteClusterResolver.class); @@ -401,18 +407,25 @@ public void testCreateNextCheckpointWithRemoteClients() throws InterruptedExcept String transformId = getTestName(); TransformConfig transformConfig = TransformConfigTests.randomTransformConfig(transformId); + doAnswer(withMockConnection()).when(remoteClient1).getConnection(any(), any()); + doAnswer(withMockConnection()).when(remoteClient2).getConnection(any(), any()); + doAnswer(withMockConnection()).when(remoteClient3).getConnection(any(), any()); + GetCheckpointAction.Response remoteCheckpointResponse1 = new GetCheckpointAction.Response( Map.of("index-1", new long[] { 1L, 2L, 3L }) ); - doAnswer(withResponse(remoteCheckpointResponse1)).when(remoteClient1).execute(eq(GetCheckpointAction.REMOTE_TYPE), any(), any()); + doAnswer(withRemoteResponse(remoteCheckpointResponse1)).when(remoteClient1) + .execute(any(), eq(GetCheckpointAction.REMOTE_TYPE), any(), any()); GetCheckpointAction.Response remoteCheckpointResponse2 = new GetCheckpointAction.Response( Map.of("index-1", new long[] { 4L, 5L, 6L, 7L, 8L }) ); - doAnswer(withResponse(remoteCheckpointResponse2)).when(remoteClient2).execute(eq(GetCheckpointAction.REMOTE_TYPE), any(), any()); + doAnswer(withRemoteResponse(remoteCheckpointResponse2)).when(remoteClient2) + .execute(any(), eq(GetCheckpointAction.REMOTE_TYPE), any(), any()); GetCheckpointAction.Response remoteCheckpointResponse3 = new GetCheckpointAction.Response(Map.of("index-1", new long[] { 9L })); - doAnswer(withResponse(remoteCheckpointResponse3)).when(remoteClient3).execute(eq(GetCheckpointAction.REMOTE_TYPE), any(), any()); + doAnswer(withRemoteResponse(remoteCheckpointResponse3)).when(remoteClient3) + .execute(any(), eq(GetCheckpointAction.REMOTE_TYPE), any(), any()); RemoteClusterResolver remoteClusterResolver = mock(RemoteClusterResolver.class); @@ -483,4 +496,20 @@ private static Answer withResponse(Response response) { return null; }; } + + private static Answer withRemoteResponse(Response response) { + return invocationOnMock -> { + ActionListener listener = invocationOnMock.getArgument(3); + listener.onResponse(response); + return null; + }; + } + + private static Answer withMockConnection() { + return invocationOnMock -> { + ActionListener listener = invocationOnMock.getArgument(1); + listener.onResponse(mock(Transport.Connection.class)); + return null; + }; + } } From 7bfb090c4ce4b443a330707dd4c18474feb8fb1d Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 26 Sep 2024 08:18:52 +0100 Subject: [PATCH 09/36] Clean up persistent task APIs (#113523) Removes unused mutability from request types and unnecessary subclasses of `ActionType<>`. --- .../CompletionPersistentTaskAction.java | 26 ++++--------- .../RemovePersistentTaskAction.java | 21 +++------- .../persistent/StartPersistentTaskAction.java | 39 ++++--------------- .../UpdatePersistentTaskStatusAction.java | 34 ++++------------ .../authz/privilege/SystemPrivilege.java | 2 +- 5 files changed, 28 insertions(+), 94 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java index bfb2d3cc9aac8..7fd3459e010c7 100644 --- a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java @@ -35,28 +35,18 @@ * ActionType that is used by executor node to indicate that the persistent action finished or failed on the node and needs to be * removed from the cluster state in case of successful completion or restarted on some other node in case of failure. */ -public class CompletionPersistentTaskAction extends ActionType { +public class CompletionPersistentTaskAction { - public static final CompletionPersistentTaskAction INSTANCE = new CompletionPersistentTaskAction(); - public static final String NAME = "cluster:admin/persistent/completion"; + public static final ActionType INSTANCE = new ActionType<>("cluster:admin/persistent/completion"); - private CompletionPersistentTaskAction() { - super(NAME); - } + private CompletionPersistentTaskAction() {/* no instances */} public static class Request extends MasterNodeRequest { - private String taskId; - - private Exception exception; - - private long allocationId = -1; - - private String localAbortReason; - - public Request() { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); - } + private final String taskId; + private final Exception exception; + private final long allocationId; + private final String localAbortReason; public Request(StreamInput in) throws IOException { super(in); @@ -129,7 +119,7 @@ public TransportAction( IndexNameExpressionResolver indexNameExpressionResolver ) { super( - CompletionPersistentTaskAction.NAME, + INSTANCE.name(), transportService, clusterService, threadPool, diff --git a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java index 5453f07c0ca00..86f9f981b2c7e 100644 --- a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java @@ -29,22 +29,15 @@ import java.io.IOException; import java.util.Objects; -public class RemovePersistentTaskAction extends ActionType { +public class RemovePersistentTaskAction { - public static final RemovePersistentTaskAction INSTANCE = new RemovePersistentTaskAction(); - public static final String NAME = "cluster:admin/persistent/remove"; + public static final ActionType INSTANCE = new ActionType<>("cluster:admin/persistent/remove"); - private RemovePersistentTaskAction() { - super(NAME); - } + private RemovePersistentTaskAction() {/* no instances */} public static class Request extends MasterNodeRequest { - private String taskId; - - public Request() { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); - } + private final String taskId; public Request(StreamInput in) throws IOException { super(in); @@ -56,10 +49,6 @@ public Request(String taskId) { this.taskId = taskId; } - public void setTaskId(String taskId) { - this.taskId = taskId; - } - @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -99,7 +88,7 @@ public TransportAction( IndexNameExpressionResolver indexNameExpressionResolver ) { super( - RemovePersistentTaskAction.NAME, + INSTANCE.name(), transportService, clusterService, threadPool, diff --git a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java index 155bee21a310e..fd3d87c21ab43 100644 --- a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java @@ -21,7 +21,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.Nullable; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -35,26 +34,16 @@ /** * This action can be used to add the record for the persistent action to the cluster state. */ -public class StartPersistentTaskAction extends ActionType { +public class StartPersistentTaskAction { - public static final StartPersistentTaskAction INSTANCE = new StartPersistentTaskAction(); - public static final String NAME = "cluster:admin/persistent/start"; + public static final ActionType INSTANCE = new ActionType<>("cluster:admin/persistent/start"); - private StartPersistentTaskAction() { - super(NAME); - } + private StartPersistentTaskAction() {/* no instances */} public static class Request extends MasterNodeRequest { - - private String taskId; - - private String taskName; - - private PersistentTaskParams params; - - public Request() { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); - } + private final String taskId; + private final String taskName; + private final PersistentTaskParams params; public Request(StreamInput in) throws IOException { super(in); @@ -117,27 +106,13 @@ public String getTaskName() { return taskName; } - public void setTaskName(String taskName) { - this.taskName = taskName; - } - public String getTaskId() { return taskId; } - public void setTaskId(String taskId) { - this.taskId = taskId; - } - public PersistentTaskParams getParams() { return params; } - - @Nullable - public void setParams(PersistentTaskParams params) { - this.params = params; - } - } public static class TransportAction extends TransportMasterNodeAction { @@ -156,7 +131,7 @@ public TransportAction( IndexNameExpressionResolver indexNameExpressionResolver ) { super( - StartPersistentTaskAction.NAME, + INSTANCE.name(), transportService, clusterService, threadPool, diff --git a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java index a8a076c642d8c..268522441f520 100644 --- a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java @@ -31,24 +31,16 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; -public class UpdatePersistentTaskStatusAction extends ActionType { +public class UpdatePersistentTaskStatusAction { - public static final UpdatePersistentTaskStatusAction INSTANCE = new UpdatePersistentTaskStatusAction(); - public static final String NAME = "cluster:admin/persistent/update_status"; + public static final ActionType INSTANCE = new ActionType<>("cluster:admin/persistent/update_status"); - private UpdatePersistentTaskStatusAction() { - super(NAME); - } + private UpdatePersistentTaskStatusAction() {/* no instances */} public static class Request extends MasterNodeRequest { - - private String taskId; - private long allocationId = -1L; - private PersistentTaskState state; - - public Request() { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); - } + private final String taskId; + private final long allocationId; + private final PersistentTaskState state; public Request(StreamInput in) throws IOException { super(in); @@ -64,26 +56,14 @@ public Request(String taskId, long allocationId, PersistentTaskState state) { this.state = state; } - public void setTaskId(String taskId) { - this.taskId = taskId; - } - public String getTaskId() { return taskId; } - public void setAllocationId(long allocationId) { - this.allocationId = allocationId; - } - public long getAllocationId() { return allocationId; } - public void setState(PersistentTaskState state) { - this.state = state; - } - public PersistentTaskState getState() { return state; } @@ -138,7 +118,7 @@ public TransportAction( IndexNameExpressionResolver indexNameExpressionResolver ) { super( - UpdatePersistentTaskStatusAction.NAME, + INSTANCE.name(), transportService, clusterService, threadPool, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java index 013d7cc21a54a..9233841891461 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java @@ -39,7 +39,7 @@ public final class SystemPrivilege extends Privilege { RetentionLeaseActions.REMOVE.name() + "*", // needed for CCR to remove retention leases RetentionLeaseActions.RENEW.name() + "*", // needed for CCR to renew retention leases "indices:admin/settings/update", // needed for DiskThresholdMonitor.markIndicesReadOnly - CompletionPersistentTaskAction.NAME, // needed for ShardFollowTaskCleaner + CompletionPersistentTaskAction.INSTANCE.name(), // needed for ShardFollowTaskCleaner "indices:data/write/*", // needed for SystemIndexMigrator "indices:data/read/*", // needed for SystemIndexMigrator "indices:admin/refresh", // needed for SystemIndexMigrator From 99015aa948d3244deb924865b91317ac53ff75bd Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 26 Sep 2024 17:21:55 +1000 Subject: [PATCH 10/36] Mute org.elasticsearch.xpack.ml.integration.MlJobIT testCantCreateJobWithSameID #113581 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 9589746c7db47..602d790246648 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -305,6 +305,9 @@ tests: - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT method: testPutE5Small_withPlatformAgnosticVariant issue: https://github.com/elastic/elasticsearch/issues/113577 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testCantCreateJobWithSameID + issue: https://github.com/elastic/elasticsearch/issues/113581 # Examples: # From fffe8844e926d16c109afc0cea71aee6c65f2774 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:42:40 +0300 Subject: [PATCH 11/36] Apply auto-flattening to `subobjects: auto` (#112092) * Introduce mode `subobjects=auto` for objects * Update docs/changelog/110524.yaml * compilation error * tests and fixes * refactor * spotless * more tests * fix nested objects * fix test * update fetch test * add QA coverage * update tests * update tests * update tests * Apply auto-flattening to `subobjects: auto` * Update docs/changelog/112092.yaml * sync * dont flatten subobjects auto * refine test * fix path for nested flattened objects and dynamic * document `subobjects: auto` * Apply suggestions from code review Co-authored-by: Felix Barnsteiner * comment updates * restore indentation in comment * update comment * update comment * update comment * update comment * rename isFlattenable * add test for dynamic template * fix copy_to and noop dynamic updates * tests * update comment * fix tests * update cluster feature in yaml test * address comments --------- Co-authored-by: Felix Barnsteiner --- docs/changelog/112092.yaml | 5 + .../mapping/params/subobjects.asciidoc | 224 ++++++++++++++++-- .../logsdb/qa/DataGenerationHelper.java | 7 +- modules/dot-prefix-validation/build.gradle | 10 + rest-api-spec/build.gradle | 30 ++- .../test/index/92_metrics_auto_subobjects.yml | 8 +- .../indices.create/20_synthetic_source.yml | 136 ++++++++++- .../15_composition.yml | 4 +- .../test/search/330_fetch_fields.yml | 2 +- .../index/mapper/DocumentParser.java | 29 ++- .../index/mapper/DocumentParserContext.java | 64 ++++- .../mapper/DotExpandingXContentParser.java | 68 +++++- .../index/mapper/DynamicFieldsBuilder.java | 5 +- .../index/mapper/MapperFeatures.java | 1 + .../index/mapper/ObjectMapper.java | 209 ++++++++++------ .../index/mapper/DocumentParserTests.java | 54 +++++ .../DotExpandingXContentParserTests.java | 142 +++++++++-- .../index/mapper/DynamicTemplatesTests.java | 68 +++++- .../mapper/IgnoredSourceFieldMapperTests.java | 60 +++++ .../index/mapper/ObjectMapperTests.java | 181 ++++++++------ 20 files changed, 1083 insertions(+), 224 deletions(-) create mode 100644 docs/changelog/112092.yaml diff --git a/docs/changelog/112092.yaml b/docs/changelog/112092.yaml new file mode 100644 index 0000000000000..35c731074d760 --- /dev/null +++ b/docs/changelog/112092.yaml @@ -0,0 +1,5 @@ +pr: 112092 +summary: "Apply auto-flattening to `subobjects: auto`" +area: Mapping +type: enhancement +issues: [] diff --git a/docs/reference/mapping/params/subobjects.asciidoc b/docs/reference/mapping/params/subobjects.asciidoc index b0a5d3817c332..63e8e3c2db3fe 100644 --- a/docs/reference/mapping/params/subobjects.asciidoc +++ b/docs/reference/mapping/params/subobjects.asciidoc @@ -10,7 +10,7 @@ where for instance a field `metrics.time` holds a value too, which is common whe A document holding a value for both `metrics.time.max` and `metrics.time` gets rejected given that `time` would need to be a leaf field to hold a value as well as an object to hold the `max` sub-field. -The `subobjects` setting, which can be applied only to the top-level mapping definition and +The `subobjects: false` setting, which can be applied only to the top-level mapping definition and to <> fields, disables the ability for an object to hold further subobjects and makes it possible to store documents where field names contain dots and share common prefixes. From the example above, if the object container `metrics` has `subobjects` set to `false`, it can hold values for both `time` and `time.max` directly @@ -109,26 +109,138 @@ PUT my-index-000001/_doc/metric_1 <1> The entire mapping is configured to not support objects. <2> The document does not support objects +Setting `subobjects: false` disallows the definition of <> and <> sub-fields, which +can be too restrictive in cases where it's desirable to have <> objects or sub-objects with specific +behavior (e.g. with `enabled:false`). In this case, it's possible to set `subobjects: auto`, which +<> whenever possible and falls back to creating an object mapper otherwise (instead of +rejecting the mapping as `subobjects: false` does). For instance: + +[source,console] +-------------------------------------------------- +PUT my-index-000002 +{ + "mappings": { + "properties": { + "metrics": { + "type": "object", + "subobjects": "auto", <1> + "properties": { + "inner": { + "type": "object", + "enabled": false + }, + "nested": { + "type": "nested" + } + } + } + } + } +} + +PUT my-index-000002/_doc/metric_1 +{ + "metrics.time" : 100, <2> + "metrics.time.min" : 10, + "metrics.time.max" : 900 +} + +PUT my-index-000002/_doc/metric_2 +{ + "metrics" : { <3> + "time" : 100, + "time.min" : 10, + "time.max" : 900, + "inner": { + "foo": "bar", + "path.to.some.field": "baz" + }, + "nested": [ + { "id": 10 }, + { "id": 1 } + ] + } +} + +GET my-index-000002/_mapping +-------------------------------------------------- + +[source,console-result] +-------------------------------------------------- +{ + "my-index-000002" : { + "mappings" : { + "properties" : { + "metrics" : { + "subobjects" : auto, + "properties" : { + "inner": { <4> + "type": "object", + "enabled": false + }, + "nested": { + "type": "nested", + "properties" : { + "id" : { + "type" : "long" + } + } + }, + "time" : { + "type" : "long" + }, + "time.min" : { + "type" : "long" + }, + "time.max" : { + "type" : "long" + } + } + } + } + } + } +} +-------------------------------------------------- + +<1> The `metrics` field can only hold statically defined objects, namely `inner` and `nested`. +<2> Sample document holding flat paths +<3> Sample document holding an object (configured with sub-objects) and its leaf sub-fields +<4> The resulting mapping where dots in field names (`time.min`, `time_max`), as well as the +statically-defined sub-objects `inner` and `nested`, were preserved + The `subobjects` setting for existing fields and the top-level mapping definition cannot be updated. +[[auto-flattening]] ==== Auto-flattening object mappings -It is generally recommended to define the properties of an object that is configured with `subobjects: false` with dotted field names -(as shown in the first example). -However, it is also possible to define these properties as sub-objects in the mappings. -In that case, the mapping will be automatically flattened before it is stored. -This makes it easier to re-use existing mappings without having to re-write them. +It is generally recommended to define the properties of an object that is configured with `subobjects: false` or +`subobjects: auto` with dotted field names (as shown in the first example). However, it is also possible to define +these properties as sub-objects in the mappings. In that case, the mapping will be automatically flattened before +it is stored. This makes it easier to re-use existing mappings without having to re-write them. + +Note that auto-flattening does not apply if any of the following <> are set +on object mappings that are defined under an object configured with `subobjects: false` or `subobjects: auto`: -Note that auto-flattening will not work when certain <> are set -on object mappings that are defined under an object configured with `subobjects: false`: +* The <> mapping parameter is `false`. +* The <> mapping parameter contradicts the implicit or explicit value of the parent. +For example, when `dynamic` is set to `false` in the root of the mapping, object mappers that set `dynamic` to `true` +can't be auto-flattened. +* The <> mapping parameter is set to `auto` or `true` explicitly. -* The <> mapping parameter must not be `false`. -* The <> mapping parameter must not contradict the implicit or explicit value of the parent. For example, when `dynamic` is set to `false` in the root of the mapping, object mappers that set `dynamic` to `true` can't be auto-flattened. -* The <> mapping parameter must not be set to `true` explicitly. +If such a sub-object is detected, the behavior depends on the `subobjects` value: + +* `subobjects: false` is not compatible, so a mapping error is returned during mapping construction. +* `subobjects: auto` reverts to adding the object to the mapping, bypassing auto-flattening for it. Still, any +intermediate objects will be auto-flattened if applicable (i.e. the object name gets directly attached under the parent +object with `subobjects: auto`). Auto-flattening can be applied within sub-objects, if they are configured with +`subobjects: auto` too. + +Auto-flattening example with `subobjects: false`: [source,console] -------------------------------------------------- -PUT my-index-000002 +PUT my-index-000003 { "mappings": { "properties": { @@ -147,13 +259,13 @@ PUT my-index-000002 } } } -GET my-index-000002/_mapping +GET my-index-000003/_mapping -------------------------------------------------- [source,console-result] -------------------------------------------------- { - "my-index-000002" : { + "my-index-000003" : { "mappings" : { "properties" : { "metrics" : { @@ -175,5 +287,85 @@ GET my-index-000002/_mapping <1> The metrics object can contain further object mappings that will be auto-flattened. Object mappings at this level must not set certain mapping parameters as explained above. -<2> This field will be auto-flattened to `"time.min"` before the mapping is stored. -<3> The auto-flattened `"time.min"` field can be inspected by looking at the index mapping. +<2> This field will be auto-flattened to `time.min` before the mapping is stored. +<3> The auto-flattened `time.min` field can be inspected by looking at the index mapping. + +Auto-flattening example with `subobjects: auto`: + +[source,console] +-------------------------------------------------- +PUT my-index-000004 +{ + "mappings": { + "properties": { + "metrics": { + "subobjects": "auto", + "properties": { + "time": { + "type": "object", <1> + "properties": { + "min": { "type": "long" } <2> + } + }, + "to": { + "type": "object", + "properties": { + "inner_metrics": { <3> + "type": "object", + "subobjects": "auto", + "properties": { + "time": { + "type": "object", + "properties": { + "max": { "type": "long" } <4> + } + } + } + } + } + } + } + } + } + } +} +GET my-index-000004/_mapping +-------------------------------------------------- + +[source,console-result] +-------------------------------------------------- +{ + "my-index-000004" : { + "mappings" : { + "properties" : { + "metrics" : { + "subobjects" : "auto", + "properties" : { + "time.min" : { <5> + "type" : "long" + }, + "to.inner_metrics" : { <6> + "subobjects" : "auto", + "properties" : { + "time.max" : { <7> + "type" : "long" + } + } + } + } + } + } + } + } +} +-------------------------------------------------- + +<1> The metrics object can contain further object mappings that may be auto-flattened, depending on their mapping +parameters as explained above. +<2> This field will be auto-flattened to `time.min` before the mapping is stored. +<3> This object has param `subobjects: auto` so it can't be auto-flattened. Its parent does qualify for auto-flattening, +so it becomes `to.inner_metrics` before the mapping is stored. +<4> This field will be auto-flattened to `time.max` before the mapping is stored. +<5> The auto-flattened `time.min` field can be inspected by looking at the index mapping. +<6> The inner object `to.inner_metrics` can be inspected by looking at the index mapping. +<7> The auto-flattened `time.max` field can be inspected by looking at the index mapping. diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java index 7fd1ccde10053..ce0820b940bf8 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java @@ -35,12 +35,7 @@ class DataGenerationHelper { private final DataGenerator dataGenerator; DataGenerationHelper() { - // TODO enable subobjects: auto - // It is disabled because it currently does not have auto flattening and that results in asserts being triggered when using copy_to. - this.subobjects = ESTestCase.randomValueOtherThan( - ObjectMapper.Subobjects.AUTO, - () -> ESTestCase.randomFrom(ObjectMapper.Subobjects.values()) - ); + this.subobjects = ESTestCase.randomFrom(ObjectMapper.Subobjects.values()); this.keepArraySource = ESTestCase.randomBoolean(); var specificationBuilder = DataGeneratorSpecification.builder().withFullyDynamicMapping(ESTestCase.randomBoolean()); diff --git a/modules/dot-prefix-validation/build.gradle b/modules/dot-prefix-validation/build.gradle index 6e232570b4a22..b300cae20d717 100644 --- a/modules/dot-prefix-validation/build.gradle +++ b/modules/dot-prefix-validation/build.gradle @@ -27,3 +27,13 @@ tasks.named('yamlRestTest') { tasks.named('yamlRestCompatTest') { usesDefaultDistribution() } + +tasks.named("yamlRestCompatTestTransform").configure( + { task -> + task.skipTest("tsdb/140_routing_path/multi-value routing path field", "Multi-value routing paths are allowed now. See #112645") + task.skipTest( + "dot_prefix/10_basic/Deprecated index template with a dot prefix index pattern", + "Tentantively disabled until #112092 gets backported to 8.x" + ) + } +) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 45c1b65d19600..1d69c170d7553 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -54,9 +54,29 @@ tasks.named("precommit").configure { dependsOn 'enforceYamlTestConvention' } -tasks.named("yamlRestCompatTestTransform").configure({ task -> - task.skipTest("tsdb/140_routing_path/multi-value routing path field", "Multi-value routing paths are allowed now. See #112645") - task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") - task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") - task.skipTest("search/540_ignore_above_synthetic_source/ignore_above mapping level setting on arrays", "Temporary mute while backporting to 8.x") +tasks.named("yamlRestCompatTestTransform").configure({task -> + task.skipTest("tsdb/140_routing_path/multi-value routing path field", "Multi-value routing paths are allowed now. See #112645") + task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") + task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") + task.skipTest("search/540_ignore_above_synthetic_source/ignore_above mapping level setting on arrays", "Temporary mute while backporting to 8.x") + task.skipTest("indices.create/20_synthetic_source/subobjects auto", "Tentantively disabled until #112092 gets backported to 8.x") + task.skipTest( + "index/92_metrics_auto_subobjects/Metrics object indexing with synthetic source", + "Tentantively disabled until #112092 gets backported to 8.x" + ) + task.skipTest( + "index/92_metrics_auto_subobjects/Root without subobjects with synthetic source", + "Tentantively disabled until #112092 gets backported to 8.x" + ) + task.skipTest( + "indices.put_index_template/15_composition/Composable index templates that include subobjects: auto at root", + "Tentantively disabled until #112092 gets backported to 8.x" + ) + task.skipTest( + "indices.put_index_template/15_composition/Composable index templates that include subobjects: auto on arbitrary field", + "Tentantively disabled until #112092 gets backported to 8.x" + ) + task.skipTest("index/92_metrics_auto_subobjects/Metrics object indexing", "Tentantively disabled until #112092 gets backported to 8.x") + task.skipTest("index/92_metrics_auto_subobjects/Root with metrics", "Tentantively disabled until #112092 gets backported to 8.x") + task.skipTest("search/330_fetch_fields/Test with subobjects: auto", "Tentantively disabled until #112092 gets backported to 8.x") }) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/92_metrics_auto_subobjects.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/92_metrics_auto_subobjects.yml index 414c24cfffd7d..603cc4fc2e304 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/92_metrics_auto_subobjects.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/92_metrics_auto_subobjects.yml @@ -2,7 +2,7 @@ "Metrics object indexing": - requires: test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ] - cluster_features: ["mapper.subobjects_auto"] + cluster_features: ["mapper.subobjects_auto_fixes"] reason: requires supporting subobjects auto setting - do: @@ -69,7 +69,7 @@ "Root with metrics": - requires: test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ] - cluster_features: ["mapper.subobjects_auto"] + cluster_features: ["mapper.subobjects_auto_fixes"] reason: requires supporting subobjects auto setting - do: @@ -131,7 +131,7 @@ "Metrics object indexing with synthetic source": - requires: test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ] - cluster_features: ["mapper.subobjects_auto"] + cluster_features: ["mapper.subobjects_auto_fixes"] reason: added in 8.4.0 - do: @@ -201,7 +201,7 @@ "Root without subobjects with synthetic source": - requires: test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ] - cluster_features: ["mapper.subobjects_auto"] + cluster_features: ["mapper.subobjects_auto_fixes"] reason: added in 8.4.0 - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index b5a9146bc54a6..41d9fcc30a880 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -887,7 +887,7 @@ doubly nested object: --- subobjects auto: - requires: - cluster_features: ["mapper.subobjects_auto"] + cluster_features: ["mapper.subobjects_auto_fixes"] reason: requires tracking ignored source and supporting subobjects auto setting - do: @@ -924,9 +924,21 @@ subobjects auto: type: keyword nested: type: nested - auto_obj: - type: object - subobjects: auto + path: + properties: + to: + properties: + auto_obj: + type: object + subobjects: auto + properties: + inner: + properties: + id: + type: keyword + id: + type: + integer - do: bulk: @@ -934,13 +946,13 @@ subobjects auto: refresh: true body: - '{ "create": { } }' - - '{ "id": 1, "foo": 10, "foo.bar": 100, "regular": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' + - '{ "id": 1, "foo": 10, "foo.bar": 100, "regular.trace.id": ["b", "a", "b"], "regular.span.id": "1" }' - '{ "create": { } }' - '{ "id": 2, "foo": 20, "foo.bar": 200, "stored": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' - '{ "create": { } }' - '{ "id": 3, "foo": 30, "foo.bar": 300, "nested": [ { "a": 10, "b": 20 }, { "a": 100, "b": 200 } ] }' - '{ "create": { } }' - - '{ "id": 4, "auto_obj": { "foo": 40, "foo.bar": 400 } }' + - '{ "id": 4, "path.to.auto_obj": { "foo": 40, "foo.bar": 400, "inner.id": "baz" }, "path.to.id": 4000 }' - match: { errors: false } @@ -952,8 +964,8 @@ subobjects auto: - match: { hits.hits.0._source.id: 1 } - match: { hits.hits.0._source.foo: 10 } - match: { hits.hits.0._source.foo\.bar: 100 } - - match: { hits.hits.0._source.regular.span.id: "1" } - - match: { hits.hits.0._source.regular.trace.id: [ "a", "b" ] } + - match: { hits.hits.0._source.regular\.span\.id: "1" } + - match: { hits.hits.0._source.regular\.trace\.id: [ "a", "b" ] } - match: { hits.hits.1._source.id: 2 } - match: { hits.hits.1._source.foo: 20 } - match: { hits.hits.1._source.foo\.bar: 200 } @@ -969,8 +981,110 @@ subobjects auto: - match: { hits.hits.2._source.nested.1.a: 100 } - match: { hits.hits.2._source.nested.1.b: 200 } - match: { hits.hits.3._source.id: 4 } - - match: { hits.hits.3._source.auto_obj.foo: 40 } - - match: { hits.hits.3._source.auto_obj.foo\.bar: 400 } + - match: { hits.hits.3._source.path\.to\.auto_obj.foo: 40 } + - match: { hits.hits.3._source.path\.to\.auto_obj.foo\.bar: 400 } + - match: { hits.hits.3._source.path\.to\.auto_obj.inner\.id: baz } + - match: { hits.hits.3._source.path\.to\.id: 4000 } + + +--- +subobjects auto with path flattening: + - requires: + cluster_features: ["mapper.subobjects_auto_fixes"] + reason: requires tracking ignored source and supporting subobjects auto setting + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + subobjects: auto + properties: + id: + type: integer + attributes: + type: object + subobjects: auto + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "id": 1, "attributes": { "foo": { "bar": 10 } } }' + - '{ "create": { } }' + - '{ "id": 2, "attributes": { "foo": { "bar": 20 } } }' + - '{ "create": { } }' + - '{ "id": 3, "attributes": { "foo": { "bar": 30 } } }' + - '{ "create": { } }' + - '{ "id": 4, "attributes": { "foo": { "bar": 40 } } }' + + - match: { errors: false } + + - do: + search: + index: test + sort: id + + - match: { hits.hits.0._source.id: 1 } + - match: { hits.hits.0._source.attributes.foo\.bar: 10 } + - match: { hits.hits.1._source.id: 2 } + - match: { hits.hits.1._source.attributes.foo\.bar: 20 } + - match: { hits.hits.2._source.id: 3 } + - match: { hits.hits.2._source.attributes.foo\.bar: 30 } + - match: { hits.hits.3._source.id: 4 } + - match: { hits.hits.3._source.attributes.foo\.bar: 40 } + + +--- +subobjects auto with dynamic template: + - requires: + cluster_features: ["mapper.subobjects_auto_fixes"] + reason: requires tracking ignored source and supporting subobjects auto setting + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + subobjects: auto + dynamic_templates: + - attributes_tmpl: + match: attributes + mapping: + type: object + enabled: false + subobjects: auto + properties: + id: + type: integer + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "id": 1, "attributes": { "foo": 10, "path.to.bar": "val1" }, "a": 100, "a.b": 1000 }' + + - match: { errors: false } + + - do: + search: + index: test + sort: id + + - match: { hits.hits.0._source.id: 1 } + - match: { hits.hits.0._source.attributes.foo: 10 } + - match: { hits.hits.0._source.attributes.path\.to\.bar: val1 } + - match: { hits.hits.0._source.a: 100 } + - match: { hits.hits.0._source.a\.b: 1000 } + --- synthetic_source with copy_to: @@ -1755,7 +1869,7 @@ synthetic_source with copy_to pointing to ambiguous field and subobjects false: --- synthetic_source with copy_to pointing to ambiguous field and subobjects auto: - requires: - cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + cluster_features: ["mapper.subobjects_auto_fixes"] reason: requires copy_to support in synthetic source - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml index 3d82539944a97..912f4e9f93df9 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml @@ -453,7 +453,7 @@ --- "Composable index templates that include subobjects: auto at root": - requires: - cluster_features: ["mapper.subobjects_auto"] + cluster_features: ["mapper.subobjects_auto_fixes"] reason: "https://github.com/elastic/elasticsearch/issues/96768 fixed at 8.11.0" test_runner_features: "allowed_warnings" @@ -504,7 +504,7 @@ --- "Composable index templates that include subobjects: auto on arbitrary field": - requires: - cluster_features: ["mapper.subobjects_auto"] + cluster_features: ["mapper.subobjects_auto_fixes"] reason: "https://github.com/elastic/elasticsearch/issues/96768 fixed at 8.11.0" test_runner_features: "allowed_warnings" diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml index 8a8dffda69e20..2b77b5558b3d3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml @@ -1129,7 +1129,7 @@ fetch geo_point: --- "Test with subobjects: auto": - requires: - cluster_features: "mapper.subobjects_auto" + cluster_features: "mapper.subobjects_auto_fixes" reason: requires support for subobjects auto setting - do: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index ebe9f27f461cf..7f9b59d427656 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -389,6 +389,14 @@ static Mapping createDynamicUpdate(DocumentParserContext context) { rootBuilder.addRuntimeField(runtimeField); } RootObjectMapper root = rootBuilder.build(MapperBuilderContext.root(context.mappingLookup().isSourceSynthetic(), false)); + + // Repeat the check, in case the dynamic mappers don't produce a mapping update. + // For instance, the parsed source may contain intermediate objects that get flattened, + // leading to an empty dynamic update. + if (root.mappers.isEmpty() && root.runtimeFields().isEmpty()) { + return null; + } + return context.mappingLookup().getMapping().mappingUpdate(root); } @@ -638,7 +646,7 @@ private static void parseObject(final DocumentParserContext context, String curr private static void doParseObject(DocumentParserContext context, String currentFieldName, Mapper objectMapper) throws IOException { context.path().add(currentFieldName); boolean withinLeafObject = context.path().isWithinLeafObject(); - if (objectMapper instanceof ObjectMapper objMapper && objMapper.subobjects() != ObjectMapper.Subobjects.ENABLED) { + if (objectMapper instanceof ObjectMapper objMapper && objMapper.subobjects() == ObjectMapper.Subobjects.DISABLED) { context.path().setWithinLeafObject(true); } parseObjectOrField(context, objectMapper); @@ -1012,11 +1020,15 @@ private static Mapper getLeafMapper(final DocumentParserContext context, String // don't create a dynamic mapping for it and don't index it. String fieldPath = context.path().pathAsText(fieldName); MappedFieldType fieldType = context.mappingLookup().getFieldType(fieldPath); - if (fieldType != null) { - // we haven't found a mapper with this name above, which means if a field type is found it is for sure a runtime field. - assert fieldType.hasDocValues() == false && fieldType.isAggregatable() && fieldType.isSearchable(); + + if (fieldType != null && fieldType.hasDocValues() == false && fieldType.isAggregatable() && fieldType.isSearchable()) { + // We haven't found a mapper with this name above, which means it is a runtime field. return noopFieldMapper(fieldPath); } + // No match or the matching field type corresponds to a mapper with flattened name (containing dots), + // e.g. for field 'foo.bar' under root there is no 'bar' mapper in object 'bar'. + // Returning null leads to creating a dynamic mapper. In the case of a mapper with flattened name, + // the dynamic mapper later gets deduplicated when building the dynamic update for the doc at hand. return null; } @@ -1160,11 +1172,10 @@ private static class RootDocumentParserContext extends DocumentParserContext { mappingLookup.getMapping().getRoot(), ObjectMapper.Dynamic.getRootDynamic(mappingLookup) ); - if (mappingLookup.getMapping().getRoot().subobjects() == ObjectMapper.Subobjects.ENABLED) { - this.parser = DotExpandingXContentParser.expandDots(parser, this.path); - } else { - this.parser = parser; - } + // If root supports no subobjects, there's no point in expanding dots in names to subobjects. + this.parser = (mappingLookup.getMapping().getRoot().subobjects() == ObjectMapper.Subobjects.DISABLED) + ? parser + : DotExpandingXContentParser.expandDots(parser, this.path, this); this.document = new LuceneDocument(); this.documents.add(document); this.maxAllowedNumNestedDocs = indexSettings().getMappingNestedDocsLimit(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index c2970d8716147..b8acdb716b467 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -123,6 +123,7 @@ public int get() { private Field version; private final SeqNoFieldMapper.SequenceIDFields seqID; private final Set fieldsAppliedFromTemplates; + private final boolean supportsObjectAutoFlattening; /** * Fields that are copied from values of other fields via copy_to. @@ -177,6 +178,7 @@ private DocumentParserContext( this.copyToFields = copyToFields; this.dynamicMappersSize = dynamicMapperSize; this.recordedSource = recordedSource; + this.supportsObjectAutoFlattening = checkForAutoFlatteningSupport(); } private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, DocumentParserContext in) { @@ -204,6 +206,43 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, ); } + private boolean checkForAutoFlatteningSupport() { + if (root().subobjects() != ObjectMapper.Subobjects.ENABLED) { + return true; + } + for (ObjectMapper objectMapper : mappingLookup.objectMappers().values()) { + if (objectMapper.subobjects() != ObjectMapper.Subobjects.ENABLED) { + return true; + } + } + if (root().dynamicTemplates() != null) { + for (DynamicTemplate dynamicTemplate : root().dynamicTemplates()) { + if (findSubobjects(dynamicTemplate.getMapping())) { + return true; + } + } + } + for (ObjectMapper objectMapper : dynamicObjectMappers.values()) { + if (objectMapper.subobjects() != ObjectMapper.Subobjects.ENABLED) { + return true; + } + } + return false; + } + + @SuppressWarnings("unchecked") + private static boolean findSubobjects(Map mapping) { + for (var entry : mapping.entrySet()) { + if (entry.getKey().equals("subobjects") && (entry.getValue() instanceof Boolean || entry.getValue() instanceof String)) { + return true; + } + if (entry.getValue() instanceof Map && findSubobjects((Map) entry.getValue())) { + return true; + } + } + return false; + } + protected DocumentParserContext( MappingLookup mappingLookup, MappingParserContext mappingParserContext, @@ -464,6 +503,10 @@ public Set getCopyToFields() { return copyToFields; } + boolean supportsObjectAutoFlattening() { + return supportsObjectAutoFlattening; + } + /** * Add a new mapper dynamically created while parsing. * @@ -599,6 +642,25 @@ final ObjectMapper getDynamicObjectMapper(String name) { return dynamicObjectMappers.get(name); } + ObjectMapper findObject(String fullName) { + // does the object mapper already exist? if so, use that + ObjectMapper objectMapper = mappingLookup().objectMappers().get(fullName); + if (objectMapper != null) { + return objectMapper; + } + // has the object mapper been added as a dynamic update already? + return getDynamicObjectMapper(fullName); + } + + ObjectMapper.Builder findObjectBuilder(String fullName) { + // does the object mapper already exist? if so, use that + ObjectMapper objectMapper = findObject(fullName); + if (objectMapper != null) { + return objectMapper.newBuilder(indexSettings().getIndexVersionCreated()); + } + return null; + } + /** * Add a new runtime field dynamically created while parsing. * We use the same set for both new indexed and new runtime fields, @@ -698,7 +760,7 @@ public LuceneDocument doc() { */ public final DocumentParserContext createCopyToContext(String copyToField, LuceneDocument doc) throws IOException { ContentPath path = new ContentPath(); - XContentParser parser = DotExpandingXContentParser.expandDots(new CopyToParser(copyToField, parser()), path); + XContentParser parser = DotExpandingXContentParser.expandDots(new CopyToParser(copyToField, parser()), path, this); return new Wrapper(root(), this) { @Override public ContentPath path() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java index fc003e709cbca..728c7ac6f25ac 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java @@ -18,6 +18,8 @@ import java.io.IOException; import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Deque; import java.util.List; import java.util.Map; @@ -38,9 +40,13 @@ private static final class WrappingParser extends FilterXContentParser { private final ContentPath contentPath; final Deque parsers = new ArrayDeque<>(); + final DocumentParserContext context; + boolean supportsObjectAutoFlattening; - WrappingParser(XContentParser in, ContentPath contentPath) throws IOException { + WrappingParser(XContentParser in, ContentPath contentPath, DocumentParserContext context) throws IOException { this.contentPath = contentPath; + this.context = context; + this.supportsObjectAutoFlattening = (context != null && context.supportsObjectAutoFlattening()); parsers.push(in); if (in.currentToken() == Token.FIELD_NAME) { expandDots(in); @@ -107,7 +113,7 @@ private void doExpandDots(XContentParser delegate, String field, int dotCount) t if (resultSize == 0) { throw new IllegalArgumentException("field name cannot contain only dots"); } - final String[] subpaths; + String[] subpaths; if (resultSize == list.length) { for (String part : list) { // check if the field name contains only whitespace @@ -126,6 +132,9 @@ private void doExpandDots(XContentParser delegate, String field, int dotCount) t } subpaths = extractAndValidateResults(field, list, resultSize); } + if (supportsObjectAutoFlattening && subpaths.length > 1) { + subpaths = maybeFlattenPaths(Arrays.asList(subpaths), context, contentPath).toArray(String[]::new); + } pushSubParser(delegate, subpaths); } @@ -235,11 +244,13 @@ public List listOrderedMap() throws IOException { /** * Wraps an XContentParser such that it re-interprets dots in field names as an object structure - * @param in the parser to wrap - * @return the wrapped XContentParser + * @param in the parser to wrap + * @param contentPath the starting path to expand, can be empty + * @param context provides mapping context to check for objects supporting sub-object auto-flattening + * @return the wrapped XContentParser */ - static XContentParser expandDots(XContentParser in, ContentPath contentPath) throws IOException { - return new WrappingParser(in, contentPath); + static XContentParser expandDots(XContentParser in, ContentPath contentPath, DocumentParserContext context) throws IOException { + return new WrappingParser(in, contentPath, context); } private enum State { @@ -410,4 +421,49 @@ public Token nextToken() throws IOException { return null; } } + + static List maybeFlattenPaths(List subpaths, DocumentParserContext context, ContentPath contentPath) { + String prefixWithDots = contentPath.pathAsText(""); + ObjectMapper parent = contentPath.length() == 0 + ? context.root() + : context.findObject(prefixWithDots.substring(0, prefixWithDots.length() - 1)); + List result = new ArrayList<>(subpaths.size()); + for (int i = 0; i < subpaths.size(); i++) { + String fullPath = prefixWithDots + String.join(".", subpaths.subList(0, i)); + if (i > 0) { + parent = context.findObject(fullPath); + } + boolean match = false; + StringBuilder path = new StringBuilder(subpaths.get(i)); + if (parent == null) { + // We get here for dynamic objects, which always get parsed with subobjects and may get flattened later. + match = true; + } else if (parent.subobjects() == ObjectMapper.Subobjects.ENABLED) { + match = true; + } else if (parent.subobjects() == ObjectMapper.Subobjects.AUTO) { + // Check if there's any subobject in the remaining path. + for (int j = i; j < subpaths.size() - 1; j++) { + if (j > i) { + path.append(".").append(subpaths.get(j)); + } + Mapper mapper = parent.mappers.get(path.toString()); + if (mapper instanceof ObjectMapper objectMapper + && (ObjectMapper.isFlatteningCandidate(objectMapper.subobjects, objectMapper) + || objectMapper.checkFlattenable(null).isPresent())) { + i = j; + match = true; + break; + } + } + } + if (match) { + result.add(path.toString()); + } else { + // We only get here if parent has subobjects set to false, or set to auto with no non-flattenable object in the sub-path. + result.add(String.join(".", subpaths.subList(i, subpaths.size()))); + return result; + } + } + return result; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java index 4b6419b85e155..cf810e278782a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.time.DateTimeException; import java.util.Map; +import java.util.Optional; /** * Encapsulates the logic for dynamically creating fields as part of document parsing. @@ -162,7 +163,9 @@ static Mapper createDynamicObjectMapper(DocumentParserContext context, String na Mapper mapper = createObjectMapperFromTemplate(context, name); return mapper != null ? mapper - : new ObjectMapper.Builder(name, context.parent().subobjects).enabled(ObjectMapper.Defaults.ENABLED) + // Dynamic objects are configured with subobject support, otherwise they can't get auto-flattened + // even if they otherwise qualify. + : new ObjectMapper.Builder(name, Optional.empty()).enabled(ObjectMapper.Defaults.ENABLED) .build(context.createDynamicMapperBuilderContext()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 2f665fd5d1e6a..31df558492b35 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -36,6 +36,7 @@ public Set getFeatures() { NodeMappingStats.SEGMENT_LEVEL_FIELDS_STATS, BooleanFieldMapper.BOOLEAN_DIMENSION, ObjectMapper.SUBOBJECTS_AUTO, + ObjectMapper.SUBOBJECTS_AUTO_FIXES, KeywordFieldMapper.KEYWORD_NORMALIZER_SYNTHETIC_SOURCE, SourceFieldMapper.SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX, Mapper.SYNTHETIC_SOURCE_KEEP_FEATURE, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index f9c854749e885..b9b611d8c62f9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -45,6 +45,7 @@ public class ObjectMapper extends Mapper { public static final String CONTENT_TYPE = "object"; static final String STORE_ARRAY_SOURCE_PARAM = "store_array_source"; static final NodeFeature SUBOBJECTS_AUTO = new NodeFeature("mapper.subobjects_auto"); + static final NodeFeature SUBOBJECTS_AUTO_FIXES = new NodeFeature("mapper.subobjects_auto_fixes"); /** * Enhances the previously boolean option for subobjects support with an intermediate mode `auto` that uses @@ -176,42 +177,84 @@ public final void addDynamic(String name, String prefix, Mapper mapper, Document // If the mapper to add has no dots, or the current object mapper has subobjects set to false, // we just add it as it is for sure a leaf mapper if (name.contains(".") == false || (subobjects.isPresent() && (subobjects.get() == Subobjects.DISABLED))) { - add(name, mapper); - } else { - // We strip off the first object path of the mapper name, load or create - // the relevant object mapper, and then recurse down into it, passing the remainder - // of the mapper name. So for a mapper 'foo.bar.baz', we locate 'foo' and then - // call addDynamic on it with the name 'bar.baz', and next call addDynamic on 'bar' with the name 'baz'. - int firstDotIndex = name.indexOf('.'); - String immediateChild = name.substring(0, firstDotIndex); - String immediateChildFullName = prefix == null ? immediateChild : prefix + "." + immediateChild; - Builder parentBuilder = findObjectBuilder(immediateChildFullName, context); - if (parentBuilder != null) { - parentBuilder.addDynamic(name.substring(firstDotIndex + 1), immediateChildFullName, mapper, context); - add(parentBuilder); - } else if (subobjects.isPresent() && subobjects.get() == Subobjects.AUTO) { - // No matching parent object was found, the mapper is added as a leaf - similar to subobjects false. - add(name, mapper); - } else { - // Expected to find a matching parent object but got null. - throw new IllegalStateException("Missing intermediate object " + immediateChildFullName); + if (mapper instanceof ObjectMapper objectMapper + && isFlatteningCandidate(subobjects, objectMapper) + && objectMapper.checkFlattenable(null).isEmpty()) { + // Subobjects auto and false don't allow adding subobjects dynamically. + return; } + add(name, mapper); + return; } - } + if (subobjects.isPresent() && subobjects.get() == Subobjects.AUTO) { + // Check if there's an existing field with the sanme, to avoid no-op dynamic updates. + ObjectMapper objectMapper = (prefix == null) ? context.root() : context.mappingLookup().objectMappers().get(prefix); + if (objectMapper != null && objectMapper.mappers.containsKey(name)) { + return; + } + + // Check for parent objects. Due to auto-flattening, names with dots are allowed so we need to check for all possible + // object names. For instance, for mapper 'foo.bar.baz.bad', we have the following options: + // -> object 'foo' found => call addDynamic on 'bar.baz.bad' + // ---> object 'bar' found => call addDynamic on 'baz.bad' + // -----> object 'baz' found => add field 'bad' to it + // -----> no match found => add field 'baz.bad' to 'bar' + // ---> object 'bar.baz' found => add field 'bad' to it + // ---> no match found => add field 'bar.baz.bad' to 'foo' + // -> object 'foo.bar' found => call addDynamic on 'baz.bad' + // ---> object 'baz' found => add field 'bad' to it + // ---> no match found=> add field 'baz.bad' to 'foo.bar' + // -> object 'foo.bar.baz' found => add field 'bad' to it + // -> no match found => add field 'foo.bar.baz.bad' to parent + String fullPathToMapper = name.substring(0, name.lastIndexOf(mapper.leafName())); + String[] fullPathTokens = fullPathToMapper.split("\\."); + StringBuilder candidateObject = new StringBuilder(); + String candidateObjectPrefix = prefix == null ? "" : prefix + "."; + for (int i = 0; i < fullPathTokens.length; i++) { + if (candidateObject.isEmpty() == false) { + candidateObject.append("."); + } + candidateObject.append(fullPathTokens[i]); + String candidateFullObject = candidateObjectPrefix.isEmpty() + ? candidateObject.toString() + : candidateObjectPrefix + candidateObject.toString(); + ObjectMapper parent = context.findObject(candidateFullObject); + if (parent != null) { + var parentBuilder = parent.newBuilder(context.indexSettings().getIndexVersionCreated()); + parentBuilder.addDynamic(name.substring(candidateObject.length() + 1), candidateFullObject, mapper, context); + if (parentBuilder.mappersBuilders.isEmpty() == false) { + add(parentBuilder); + } + return; + } + } - private static Builder findObjectBuilder(String fullName, DocumentParserContext context) { - // does the object mapper already exist? if so, use that - ObjectMapper objectMapper = context.mappingLookup().objectMappers().get(fullName); - if (objectMapper != null) { - return objectMapper.newBuilder(context.indexSettings().getIndexVersionCreated()); + // No matching parent object was found, the mapper is added as a leaf - similar to subobjects false. + // This only applies to field mappers, as subobjects get auto-flattened. + if (mapper instanceof FieldMapper fieldMapper) { + FieldMapper.Builder fieldBuilder = fieldMapper.getMergeBuilder(); + fieldBuilder.setLeafName(name); // Update to reflect the current, possibly flattened name. + add(fieldBuilder); + } + return; } - // has the object mapper been added as a dynamic update already? - objectMapper = context.getDynamicObjectMapper(fullName); - if (objectMapper != null) { - return objectMapper.newBuilder(context.indexSettings().getIndexVersionCreated()); + + // We strip off the first object path of the mapper name, load or create + // the relevant object mapper, and then recurse down into it, passing the remainder + // of the mapper name. So for a mapper 'foo.bar.baz', we locate 'foo' and then + // call addDynamic on it with the name 'bar.baz', and next call addDynamic on 'bar' with the name 'baz'. + int firstDotIndex = name.indexOf('.'); + String immediateChild = name.substring(0, firstDotIndex); + String immediateChildFullName = prefix == null ? immediateChild : prefix + "." + immediateChild; + Builder parentBuilder = context.findObjectBuilder(immediateChildFullName); + if (parentBuilder != null) { + parentBuilder.addDynamic(name.substring(firstDotIndex + 1), immediateChildFullName, mapper, context); + add(parentBuilder); + } else { + // Expected to find a matching parent object but got null. + throw new IllegalStateException("Missing intermediate object " + immediateChildFullName); } - // no object mapper found - return null; + } protected final Map buildMappers(MapperBuilderContext mapperBuilderContext) { @@ -227,9 +270,10 @@ protected final Map buildMappers(MapperBuilderContext mapperBuil // mix of object notation and dot notation. mapper = existing.merge(mapper, MapperMergeContext.from(mapperBuilderContext, Long.MAX_VALUE)); } - if (subobjects.isPresent() && subobjects.get() == Subobjects.DISABLED && mapper instanceof ObjectMapper objectMapper) { - // We're parsing a mapping that has set `subobjects: false` but has defined sub-objects - objectMapper.asFlattenedFieldMappers(mapperBuilderContext).forEach(m -> mappers.put(m.leafName(), m)); + if (mapper instanceof ObjectMapper objectMapper && isFlatteningCandidate(subobjects, objectMapper)) { + // We're parsing a mapping that has defined sub-objects, may need to flatten them. + objectMapper.asFlattenedFieldMappers(mapperBuilderContext, throwOnFlattenableError(subobjects)) + .forEach(m -> mappers.put(m.leafName(), m)); } else { mappers.put(mapper.leafName(), mapper); } @@ -624,12 +668,11 @@ private static Map buildMergedMappers( Optional subobjects ) { Map mergedMappers = new HashMap<>(); + var context = objectMergeContext.getMapperBuilderContext(); for (Mapper childOfExistingMapper : existing.mappers.values()) { - if (subobjects.isPresent() - && subobjects.get() == Subobjects.DISABLED - && childOfExistingMapper instanceof ObjectMapper objectMapper) { - // An existing mapping with sub-objects is merged with a mapping that has set `subobjects: false` - objectMapper.asFlattenedFieldMappers(objectMergeContext.getMapperBuilderContext()) + if (childOfExistingMapper instanceof ObjectMapper objectMapper && isFlatteningCandidate(subobjects, objectMapper)) { + // An existing mapping with sub-objects is merged with a mapping that has `subobjects` set to false or auto. + objectMapper.asFlattenedFieldMappers(context, throwOnFlattenableError(subobjects)) .forEach(m -> mergedMappers.put(m.leafName(), m)); } else { putMergedMapper(mergedMappers, childOfExistingMapper); @@ -638,11 +681,9 @@ private static Map buildMergedMappers( for (Mapper mergeWithMapper : mergeWithObject) { Mapper mergeIntoMapper = mergedMappers.get(mergeWithMapper.leafName()); if (mergeIntoMapper == null) { - if (subobjects.isPresent() - && subobjects.get() == Subobjects.DISABLED - && mergeWithMapper instanceof ObjectMapper objectMapper) { - // An existing mapping that has set `subobjects: false` is merged with a mapping with sub-objects - objectMapper.asFlattenedFieldMappers(objectMergeContext.getMapperBuilderContext()) + if (mergeWithMapper instanceof ObjectMapper objectMapper && isFlatteningCandidate(subobjects, objectMapper)) { + // An existing mapping with `subobjects` set to false or auto is merged with a mapping with sub-objects + objectMapper.asFlattenedFieldMappers(context, throwOnFlattenableError(subobjects)) .stream() .filter(m -> objectMergeContext.decrementFieldBudgetIfPossible(m.getTotalFieldsCount())) .forEach(m -> putMergedMapper(mergedMappers, m)); @@ -699,57 +740,83 @@ private static ObjectMapper truncateObjectMapper(MapperMergeContext context, Obj * * @throws IllegalArgumentException if the mapper cannot be flattened */ - List asFlattenedFieldMappers(MapperBuilderContext context) { - List flattenedMappers = new ArrayList<>(); + List asFlattenedFieldMappers(MapperBuilderContext context, boolean throwOnFlattenableError) { + List flattenedMappers = new ArrayList<>(); ContentPath path = new ContentPath(); - asFlattenedFieldMappers(context, flattenedMappers, path); + asFlattenedFieldMappers(context, flattenedMappers, path, throwOnFlattenableError); return flattenedMappers; } - private void asFlattenedFieldMappers(MapperBuilderContext context, List flattenedMappers, ContentPath path) { - ensureFlattenable(context, path); + static boolean isFlatteningCandidate(Optional subobjects, ObjectMapper mapper) { + return subobjects.isPresent() && subobjects.get() != Subobjects.ENABLED && mapper instanceof NestedObjectMapper == false; + } + + private static boolean throwOnFlattenableError(Optional subobjects) { + return subobjects.isPresent() && subobjects.get() == Subobjects.DISABLED; + } + + private void asFlattenedFieldMappers( + MapperBuilderContext context, + List flattenedMappers, + ContentPath path, + boolean throwOnFlattenableError + ) { + var error = checkFlattenable(context); + if (error.isPresent()) { + if (throwOnFlattenableError) { + throw new IllegalArgumentException( + "Object mapper [" + + path.pathAsText(leafName()) + + "] was found in a context where subobjects is set to false. " + + "Auto-flattening [" + + path.pathAsText(leafName()) + + "] failed because " + + error.get() + ); + } + // The object can't be auto-flattened under the parent object, so it gets added at the current level. + // [subobjects=auto] applies auto-flattening to names, so the leaf name may need to change. + // Since mapper objects are immutable, we create a clone of the current one with the updated leaf name. + flattenedMappers.add( + path.pathAsText("").isEmpty() + ? this + : new ObjectMapper(path.pathAsText(leafName()), fullPath, enabled, subobjects, storeArraySource, dynamic, mappers) + ); + return; + } path.add(leafName()); for (Mapper mapper : mappers.values()) { if (mapper instanceof FieldMapper fieldMapper) { FieldMapper.Builder fieldBuilder = fieldMapper.getMergeBuilder(); fieldBuilder.setLeafName(path.pathAsText(mapper.leafName())); flattenedMappers.add(fieldBuilder.build(context)); - } else if (mapper instanceof ObjectMapper objectMapper) { - objectMapper.asFlattenedFieldMappers(context, flattenedMappers, path); + } else if (mapper instanceof ObjectMapper objectMapper && mapper instanceof NestedObjectMapper == false) { + objectMapper.asFlattenedFieldMappers(context, flattenedMappers, path, throwOnFlattenableError); } } path.remove(); } - private void ensureFlattenable(MapperBuilderContext context, ContentPath path) { - if (dynamic != null && context.getDynamic() != dynamic) { - throwAutoFlatteningException( - path, + Optional checkFlattenable(MapperBuilderContext context) { + if (dynamic != null && (context == null || context.getDynamic() != dynamic)) { + return Optional.of( "the value of [dynamic] (" + dynamic + ") is not compatible with the value from its parent context (" - + context.getDynamic() + + (context != null ? context.getDynamic() : "") + ")" ); } + if (storeArraySource()) { + return Optional.of("the value of [store_array_source] is [true]"); + } if (isEnabled() == false) { - throwAutoFlatteningException(path, "the value of [enabled] is [false]"); + return Optional.of("the value of [enabled] is [false]"); } - if (subobjects.isPresent() && subobjects.get() == Subobjects.ENABLED) { - throwAutoFlatteningException(path, "the value of [subobjects] is [true]"); + if (subobjects.isPresent() && subobjects.get() != Subobjects.DISABLED) { + return Optional.of("the value of [subobjects] is [" + subobjects().printedValue + "]"); } - } - - private void throwAutoFlatteningException(ContentPath path, String reason) { - throw new IllegalArgumentException( - "Object mapper [" - + path.pathAsText(leafName()) - + "] was found in a context where subobjects is set to false. " - + "Auto-flattening [" - + path.pathAsText(leafName()) - + "] failed because " - + reason - ); + return Optional.empty(); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index 71b52dc41705b..93f546eb288b9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -2307,6 +2307,60 @@ public void testSubobjectsFalseFlattened() throws Exception { assertNotNull(doc.rootDoc().getField("attributes.simple.attribute")); } + public void testSubobjectsAutoFlattened() throws Exception { + DocumentMapper mapper = createDocumentMapper(mapping(b -> { + b.startObject("attributes"); + { + b.field("dynamic", false); + b.field("subobjects", "auto"); + b.startObject("properties"); + { + b.startObject("simple.attribute").field("type", "keyword").endObject(); + b.startObject("complex.attribute").field("type", "flattened").endObject(); + b.startObject("path").field("type", "object"); + { + b.field("store_array_source", "true").field("subobjects", "auto"); + b.startObject("properties"); + { + b.startObject("nested.attribute").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + b.startObject("flattened_object").field("type", "object"); + { + b.startObject("properties"); + { + b.startObject("nested.attribute").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + })); + ParsedDocument doc = mapper.parse(source(""" + { + "attributes": { + "complex.attribute": { + "foo" : "bar" + }, + "simple.attribute": "sa", + "path": { + "nested.attribute": "na" + }, + "flattened_object.nested.attribute": "fna" + } + } + """)); + assertNotNull(doc.rootDoc().getField("attributes.complex.attribute")); + assertNotNull(doc.rootDoc().getField("attributes.simple.attribute")); + assertNotNull(doc.rootDoc().getField("attributes.path.nested.attribute")); + assertNotNull(doc.rootDoc().getField("attributes.flattened_object.nested.attribute")); + } + public void testWriteToFieldAlias() throws Exception { DocumentMapper mapper = createDocumentMapper(mapping(b -> { b.startObject("alias-field"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DotExpandingXContentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DotExpandingXContentParserTests.java index b38c65c1710d6..c4e223a4d1b77 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DotExpandingXContentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DotExpandingXContentParserTests.java @@ -13,9 +13,12 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; +import org.hamcrest.Matchers; import java.io.IOException; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -26,7 +29,7 @@ private void assertXContentMatches(String dotsExpanded, String withDots) throws final ContentPath contentPath = new ContentPath(); try ( XContentParser inputParser = createParser(JsonXContent.jsonXContent, withDots); - XContentParser expandedParser = DotExpandingXContentParser.expandDots(inputParser, contentPath) + XContentParser expandedParser = DotExpandingXContentParser.expandDots(inputParser, contentPath, null) ) { expandedParser.allowDuplicateKeys(true); @@ -37,7 +40,7 @@ private void assertXContentMatches(String dotsExpanded, String withDots) throws expectedParser.allowDuplicateKeys(true); try ( var p = createParser(JsonXContent.jsonXContent, withDots); - XContentParser actualParser = DotExpandingXContentParser.expandDots(p, contentPath) + XContentParser actualParser = DotExpandingXContentParser.expandDots(p, contentPath, null) ) { XContentParser.Token currentToken; while ((currentToken = actualParser.nextToken()) != null) { @@ -127,7 +130,7 @@ public void testDuplicateKeys() throws IOException { public void testDotsCollapsingFlatPaths() throws IOException { ContentPath contentPath = new ContentPath(); XContentParser parser = DotExpandingXContentParser.expandDots(createParser(JsonXContent.jsonXContent, """ - {"metrics.service.time": 10, "metrics.service.time.max": 500, "metrics.foo": "value"}"""), contentPath); + {"metrics.service.time": 10, "metrics.service.time.max": 500, "metrics.foo": "value"}"""), contentPath, null); parser.nextToken(); assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken()); assertEquals("metrics", parser.currentName()); @@ -197,7 +200,7 @@ public void testDotsCollapsingStructuredPath() throws IOException { }, "foo" : "value" } - }"""), contentPath); + }"""), contentPath, null); parser.nextToken(); assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken()); assertEquals("metrics", parser.currentName()); @@ -235,7 +238,7 @@ public void testDotsCollapsingStructuredPath() throws IOException { public void testSkipChildren() throws IOException { XContentParser parser = DotExpandingXContentParser.expandDots(createParser(JsonXContent.jsonXContent, """ - { "test.with.dots" : "value", "nodots" : "value2" }"""), new ContentPath()); + { "test.with.dots" : "value", "nodots" : "value2" }"""), new ContentPath(), null); parser.nextToken(); // start object assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken()); assertEquals("test", parser.currentName()); @@ -258,7 +261,7 @@ public void testSkipChildren() throws IOException { public void testSkipChildrenWithinInnerObject() throws IOException { XContentParser parser = DotExpandingXContentParser.expandDots(createParser(JsonXContent.jsonXContent, """ - { "test.with.dots" : {"obj" : {"field":"value"}}, "nodots" : "value2" }"""), new ContentPath()); + { "test.with.dots" : {"obj" : {"field":"value"}}, "nodots" : "value2" }"""), new ContentPath(), null); parser.nextToken(); // start object assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken()); @@ -306,7 +309,8 @@ public void testGetTokenLocation() throws IOException { XContentParser expectedParser = createParser(JsonXContent.jsonXContent, jsonInput); XContentParser dotExpandedParser = DotExpandingXContentParser.expandDots( createParser(JsonXContent.jsonXContent, jsonInput), - new ContentPath() + new ContentPath(), + null ); assertEquals(expectedParser.getTokenLocation(), dotExpandedParser.getTokenLocation()); @@ -364,7 +368,8 @@ public void testGetTokenLocation() throws IOException { public void testParseMapUOE() throws Exception { XContentParser dotExpandedParser = DotExpandingXContentParser.expandDots( createParser(JsonXContent.jsonXContent, ""), - new ContentPath() + new ContentPath(), + null ); expectThrows(UnsupportedOperationException.class, dotExpandedParser::map); } @@ -372,7 +377,8 @@ public void testParseMapUOE() throws Exception { public void testParseMapOrderedUOE() throws Exception { XContentParser dotExpandedParser = DotExpandingXContentParser.expandDots( createParser(JsonXContent.jsonXContent, ""), - new ContentPath() + new ContentPath(), + null ); expectThrows(UnsupportedOperationException.class, dotExpandedParser::mapOrdered); } @@ -380,7 +386,8 @@ public void testParseMapOrderedUOE() throws Exception { public void testParseMapStringsUOE() throws Exception { XContentParser dotExpandedParser = DotExpandingXContentParser.expandDots( createParser(JsonXContent.jsonXContent, ""), - new ContentPath() + new ContentPath(), + null ); expectThrows(UnsupportedOperationException.class, dotExpandedParser::mapStrings); } @@ -388,7 +395,8 @@ public void testParseMapStringsUOE() throws Exception { public void testParseMapSupplierUOE() throws Exception { XContentParser dotExpandedParser = DotExpandingXContentParser.expandDots( createParser(JsonXContent.jsonXContent, ""), - new ContentPath() + new ContentPath(), + null ); expectThrows(UnsupportedOperationException.class, () -> dotExpandedParser.map(HashMap::new, XContentParser::text)); } @@ -403,7 +411,8 @@ public void testParseMap() throws Exception { contentPath.setWithinLeafObject(true); XContentParser dotExpandedParser = DotExpandingXContentParser.expandDots( createParser(JsonXContent.jsonXContent, jsonInput), - contentPath + contentPath, + null ); assertEquals(XContentParser.Token.START_OBJECT, dotExpandedParser.nextToken()); assertEquals(XContentParser.Token.FIELD_NAME, dotExpandedParser.nextToken()); @@ -418,7 +427,8 @@ public void testParseMap() throws Exception { public void testParseListUOE() throws Exception { XContentParser dotExpandedParser = DotExpandingXContentParser.expandDots( createParser(JsonXContent.jsonXContent, ""), - new ContentPath() + new ContentPath(), + null ); expectThrows(UnsupportedOperationException.class, dotExpandedParser::list); } @@ -426,7 +436,8 @@ public void testParseListUOE() throws Exception { public void testParseListOrderedUOE() throws Exception { XContentParser dotExpandedParser = DotExpandingXContentParser.expandDots( createParser(JsonXContent.jsonXContent, ""), - new ContentPath() + new ContentPath(), + null ); expectThrows(UnsupportedOperationException.class, dotExpandedParser::listOrderedMap); } @@ -440,7 +451,8 @@ public void testParseList() throws Exception { contentPath.setWithinLeafObject(true); XContentParser dotExpandedParser = DotExpandingXContentParser.expandDots( createParser(JsonXContent.jsonXContent, jsonInput), - contentPath + contentPath, + null ); assertEquals(XContentParser.Token.START_OBJECT, dotExpandedParser.nextToken()); assertEquals(XContentParser.Token.FIELD_NAME, dotExpandedParser.nextToken()); @@ -450,4 +462,104 @@ public void testParseList() throws Exception { assertEquals("one", list.get(0)); assertEquals("two", list.get(1)); } + + private static DocumentParserContext createContext(XContentBuilder builder) throws IOException { + var documentMapper = new MapperServiceTestCase() { + }.createDocumentMapper(builder); + return new TestDocumentParserContext(documentMapper.mappers(), null); + } + + private static List getSubPaths(XContentBuilder builder, String... path) throws IOException { + DocumentParserContext context = createContext(builder); + return DotExpandingXContentParser.maybeFlattenPaths(Arrays.stream(path).toList(), context, new ContentPath()); + } + + private static List getSubPaths(XContentBuilder builder, List contentPath, List path) throws IOException { + DocumentParserContext context = createContext(builder); + ContentPath content = new ContentPath(); + for (String c : contentPath) { + content.add(c); + } + return DotExpandingXContentParser.maybeFlattenPaths(path, context, content); + } + + public void testAutoFlattening() throws Exception { + var b = XContentBuilder.builder(XContentType.JSON.xContent()); + b.startObject().startObject("_doc"); + { + b.field("subobjects", "auto"); + b.startObject("properties"); + { + b.startObject("path").startObject("properties"); + { + b.startObject("to").startObject("properties"); + { + b.startObject("field").field("type", "integer").endObject(); + } + b.endObject().endObject(); + } + b.endObject().endObject(); + b.startObject("path.auto").field("subobjects", "auto").startObject("properties"); + { + b.startObject("to").startObject("properties"); + { + b.startObject("some.field").field("type", "integer").endObject(); + } + b.endObject().endObject(); + b.startObject("inner.enabled").field("dynamic", "false").startObject("properties"); + { + b.startObject("field").field("type", "integer").endObject(); + } + b.endObject().endObject(); + } + b.endObject().endObject(); + b.startObject("path.disabled").field("subobjects", "false").startObject("properties"); + { + b.startObject("to").startObject("properties"); + { + b.startObject("some.field").field("type", "integer").endObject(); + } + b.endObject().endObject(); + } + b.endObject().endObject(); + } + b.endObject(); + } + b.endObject().endObject(); + + // inner [subobjects:enabled] gets flattened + assertThat(getSubPaths(b, "field"), Matchers.contains("field")); + assertThat(getSubPaths(b, "path", "field"), Matchers.contains("path.field")); + assertThat(getSubPaths(b, "path", "to", "field"), Matchers.contains("path.to.field")); + assertThat(getSubPaths(b, "path", "to", "any"), Matchers.contains("path.to.any")); + + // inner [subobjects:auto] does not get flattened + assertThat(getSubPaths(b, "path", "auto", "field"), Matchers.contains("path.auto", "field")); + assertThat(getSubPaths(b, "path", "auto", "some", "field"), Matchers.contains("path.auto", "some.field")); + assertThat(getSubPaths(b, "path", "auto", "to", "some", "field"), Matchers.contains("path.auto", "to.some.field")); + assertThat(getSubPaths(b, "path", "auto", "to", "some", "other"), Matchers.contains("path.auto", "to.some.other")); + assertThat(getSubPaths(b, "path", "auto", "inner", "enabled", "field"), Matchers.contains("path.auto", "inner.enabled", "field")); + assertThat( + getSubPaths(b, "path", "auto", "inner", "enabled", "to", "some", "field"), + Matchers.contains("path.auto", "inner.enabled", "to", "some", "field") + ); + + // inner [subobjects:disabled] gets flattened + assertThat(getSubPaths(b, "path", "disabled", "field"), Matchers.contains("path.disabled.field")); + assertThat(getSubPaths(b, "path", "disabled", "some", "field"), Matchers.contains("path.disabled.some.field")); + assertThat(getSubPaths(b, "path", "disabled", "to", "some", "field"), Matchers.contains("path.disabled.to.some.field")); + assertThat(getSubPaths(b, "path", "disabled", "to", "some", "other"), Matchers.contains("path.disabled.to.some.other")); + + // Non-empty content path. + assertThat(getSubPaths(b, List.of("path"), List.of("field")), Matchers.contains("field")); + assertThat(getSubPaths(b, List.of("path"), List.of("to", "field")), Matchers.contains("to", "field")); + assertThat(getSubPaths(b, List.of("path", "to"), List.of("field")), Matchers.contains("field")); + assertThat(getSubPaths(b, List.of("path"), List.of("auto", "field")), Matchers.contains("auto", "field")); + assertThat(getSubPaths(b, List.of("path", "auto"), List.of("to", "some", "field")), Matchers.contains("to.some.field")); + assertThat( + getSubPaths(b, List.of("path", "auto"), List.of("inner", "enabled", "to", "some", "field")), + Matchers.contains("inner.enabled", "to", "some", "field") + ); + assertThat(getSubPaths(b, List.of("path", "disabled"), List.of("to", "some", "field")), Matchers.contains("to", "some", "field")); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java index 7f430cf676809..43ee47245f492 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java @@ -1619,10 +1619,9 @@ public void testSubobjectsAutoWithInnerNestedFromDynamicTemplate() throws IOExce assertNotNull(doc.rootDoc().get("metrics.time.max")); assertNotNull(doc.docs().get(0).get("metrics.time.foo")); - assertThat( - ((ObjectMapper) doc.dynamicMappingsUpdate().getRoot().getMapper("metrics")).getMapper("time"), - instanceOf(NestedObjectMapper.class) - ); + var metrics = ((ObjectMapper) doc.dynamicMappingsUpdate().getRoot().getMapper("metrics")); + assertThat(metrics.getMapper("time"), instanceOf(NestedObjectMapper.class)); + assertThat(metrics.getMapper("time.max"), instanceOf(NumberFieldMapper.class)); } public void testDynamicSubobject() throws IOException { @@ -2057,7 +2056,7 @@ public void testSubobjectsAutoFlattened() throws IOException { "dynamic_templates": [ { "test": { - "path_match": "attributes.resource.*", + "path_match": "attributes.*", "match_mapping_type": "object", "mapping": { "type": "flattened" @@ -2070,7 +2069,7 @@ public void testSubobjectsAutoFlattened() throws IOException { """; String docJson = """ { - "attributes.resource": { + "attributes": { "complex.attribute": { "a": "b" }, @@ -2083,14 +2082,67 @@ public void testSubobjectsAutoFlattened() throws IOException { ParsedDocument parsedDoc = mapperService.documentMapper().parse(source(docJson)); merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); - Mapper fooBarMapper = mapperService.documentMapper().mappers().getMapper("attributes.resource.foo.bar"); + Mapper fooBarMapper = mapperService.documentMapper().mappers().getMapper("attributes.foo.bar"); assertNotNull(fooBarMapper); assertEquals("text", fooBarMapper.typeName()); - Mapper fooStructuredMapper = mapperService.documentMapper().mappers().getMapper("attributes.resource.complex.attribute"); + Mapper fooStructuredMapper = mapperService.documentMapper().mappers().getMapper("attributes.complex.attribute"); assertNotNull(fooStructuredMapper); assertEquals("flattened", fooStructuredMapper.typeName()); } + public void testSubobjectsAutoWithObjectInDynamicTemplate() throws IOException { + String mapping = """ + { + "_doc": { + "properties": { + "attributes": { + "type": "object", + "subobjects": "auto" + } + }, + "dynamic_templates": [ + { + "test": { + "path_match": "attributes.*", + "match_mapping_type": "object", + "mapping": { + "type": "object", + "dynamic": "false", + "properties": { + "id": { + "type": "integer" + } + } + } + } + } + ] + } + } + """; + String docJson = """ + { + "attributes": { + "to": { + "id": 10 + }, + "foo.bar": "baz" + } + } + """; + + MapperService mapperService = createMapperService(mapping); + ParsedDocument parsedDoc = mapperService.documentMapper().parse(source(docJson)); + merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); + + Mapper fooBarMapper = mapperService.documentMapper().mappers().getMapper("attributes.foo.bar"); + assertNotNull(fooBarMapper); + assertEquals("text", fooBarMapper.typeName()); + Mapper innerObject = mapperService.documentMapper().mappers().objectMappers().get("attributes.to"); + assertNotNull(innerObject); + assertEquals("integer", mapperService.documentMapper().mappers().getMapper("attributes.to.id").typeName()); + } + public void testMatchWithArrayOfFieldNames() throws IOException { String mapping = """ { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index eaa7bf6528203..5d5273f0fc788 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -1549,6 +1549,66 @@ public void testCopyToLogicInsideObject() throws IOException { assertEquals("{\"path\":{\"at\":\"A\"}}", syntheticSource); } + public void testCopyToRootWithSubobjectFlattening() throws IOException { + DocumentMapper documentMapper = createMapperService(topMapping(b -> { + b.startObject("_source").field("mode", "synthetic").endObject(); + b.field("subobjects", randomFrom("false", "auto")); + b.startObject("properties"); + { + b.startObject("k").field("type", "keyword").field("copy_to", "a.b.c").endObject(); + b.startObject("a").startObject("properties"); + { + b.startObject("b").startObject("properties"); + { + b.startObject("c").field("type", "keyword").endObject(); + } + b.endObject().endObject(); + } + b.endObject().endObject(); + } + b.endObject(); + })).documentMapper(); + + CheckedConsumer document = b -> b.field("k", "hey"); + + var doc = documentMapper.parse(source(document)); + assertNotNull(doc.docs().get(0).getField("a.b.c")); + + var syntheticSource = syntheticSource(documentMapper, document); + assertEquals("{\"k\":\"hey\"}", syntheticSource); + } + + public void testCopyToObjectWithSubobjectFlattening() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("path").field("subobjects", randomFrom("false", "auto")).startObject("properties"); + { + b.startObject("k").field("type", "keyword").field("copy_to", "path.a.b.c").endObject(); + b.startObject("a").startObject("properties"); + { + b.startObject("b").startObject("properties"); + { + b.startObject("c").field("type", "keyword").endObject(); + } + b.endObject().endObject(); + } + b.endObject().endObject(); + } + b.endObject().endObject(); + })).documentMapper(); + + CheckedConsumer document = b -> { + b.startObject("path"); + b.field("k", "hey"); + b.endObject(); + }; + + var doc = documentMapper.parse(source(document)); + assertNotNull(doc.docs().get(0).getField("path.a.b.c")); + + var syntheticSource = syntheticSource(documentMapper, document); + assertEquals("{\"path\":{\"k\":\"hey\"}}", syntheticSource); + } + protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) throws IOException { // We exclude ignored source field since in some cases it contains an exact copy of a part of document source. diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index 3312c94e8a0e1..4bc91b793d049 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -354,12 +354,8 @@ public void testSubobjectsFalse() throws Exception { b.field("subobjects", false); b.startObject("properties"); { - b.startObject("time"); - b.field("type", "long"); - b.endObject(); - b.startObject("time.max"); - b.field("type", "long"); - b.endObject(); + b.startObject("time").field("type", "long").endObject(); + b.startObject("time.max").field("type", "long").endObject(); } b.endObject(); } @@ -380,9 +376,7 @@ public void testSubobjectsFalseWithInnerObject() throws IOException { { b.startObject("properties"); { - b.startObject("max"); - b.field("type", "long"); - b.endObject(); + b.startObject("max").field("type", "long").endObject(); } b.endObject(); } @@ -403,9 +397,7 @@ public void testSubobjectsFalseWithInnerNested() { b.field("subobjects", false); b.startObject("properties"); { - b.startObject("time"); - b.field("type", "nested"); - b.endObject(); + b.startObject("time").field("type", "nested").endObject(); } b.endObject(); } @@ -419,12 +411,8 @@ public void testSubobjectsFalseWithInnerNested() { public void testSubobjectsFalseRoot() throws Exception { MapperService mapperService = createMapperService(mappingNoSubobjects(b -> { - b.startObject("metrics.service.time"); - b.field("type", "long"); - b.endObject(); - b.startObject("metrics.service.time.max"); - b.field("type", "long"); - b.endObject(); + b.startObject("metrics.service.time").field("type", "long").endObject(); + b.startObject("metrics.service.time.max").field("type", "long").endObject(); })); assertNotNull(mapperService.fieldType("metrics.service.time")); assertNotNull(mapperService.fieldType("metrics.service.time.max")); @@ -441,9 +429,7 @@ public void testSubobjectsFalseRootWithInnerObject() throws IOException { { b.startObject("properties"); { - b.startObject("max"); - b.field("type", "long"); - b.endObject(); + b.startObject("max").field("type", "long").endObject(); } b.endObject(); } @@ -455,9 +441,7 @@ public void testSubobjectsFalseRootWithInnerObject() throws IOException { public void testSubobjectsFalseRootWithInnerNested() { MapperParsingException exception = expectThrows(MapperParsingException.class, () -> createMapperService(mappingNoSubobjects(b -> { - b.startObject("metrics.service"); - b.field("type", "nested"); - b.endObject(); + b.startObject("metrics.service").field("type", "nested").endObject(); }))); assertEquals( "Failed to parse mapping: Tried to add nested object [metrics.service] to object [_doc] which does not support subobjects", @@ -473,8 +457,7 @@ public void testSubobjectsCannotBeUpdated() throws IOException { "_doc", MergeReason.MAPPING_UPDATE, new CompressedXContent(BytesReference.bytes(fieldMapping(b -> { - b.field("type", "object"); - b.field("subobjects", "false"); + b.field("type", "object").field("subobjects", "false"); }))) ); MapperException exception = expectThrows( @@ -509,12 +492,8 @@ public void testSubobjectsAuto() throws Exception { b.field("subobjects", "auto"); b.startObject("properties"); { - b.startObject("time"); - b.field("type", "long"); - b.endObject(); - b.startObject("time.max"); - b.field("type", "long"); - b.endObject(); + b.startObject("time").field("type", "long").endObject(); + b.startObject("time.max").field("type", "long").endObject(); b.startObject("attributes"); { b.field("type", "object"); @@ -531,7 +510,7 @@ public void testSubobjectsAuto() throws Exception { assertNotNull(mapperService.documentMapper().mappers().objectMappers().get("metrics.service.attributes")); } - public void testSubobjectsAutoWithInnerObject() throws IOException { + public void testSubobjectsAutoWithInnerFlattenableObject() throws IOException { MapperService mapperService = createMapperService(mapping(b -> { b.startObject("metrics.service"); { @@ -542,16 +521,12 @@ public void testSubobjectsAutoWithInnerObject() throws IOException { { b.startObject("properties"); { - b.startObject("max"); - b.field("type", "long"); - b.endObject(); + b.startObject("max").field("type", "long").endObject(); } b.endObject(); } b.endObject(); - b.startObject("foo"); - b.field("type", "keyword"); - b.endObject(); + b.startObject("foo").field("type", "keyword").endObject(); } b.endObject(); } @@ -560,11 +535,11 @@ public void testSubobjectsAutoWithInnerObject() throws IOException { assertNull(mapperService.fieldType("metrics.service.time")); assertNotNull(mapperService.fieldType("metrics.service.time.max")); assertNotNull(mapperService.fieldType("metrics.service.foo")); - assertNotNull(mapperService.documentMapper().mappers().objectMappers().get("metrics.service.time")); + assertNull(mapperService.documentMapper().mappers().objectMappers().get("metrics.service.time")); // Gets flattened. assertNotNull(mapperService.documentMapper().mappers().getMapper("metrics.service.foo")); } - public void testSubobjectsAutoWithInnerNested() throws IOException { + public void testSubobjectsAutoWithInnerNonFlattenableObject() throws IOException { MapperService mapperService = createMapperService(mapping(b -> { b.startObject("metrics.service"); { @@ -572,8 +547,36 @@ public void testSubobjectsAutoWithInnerNested() throws IOException { b.startObject("properties"); { b.startObject("time"); - b.field("type", "nested"); + { + b.field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true); + b.startObject("properties"); + { + b.startObject("max").field("type", "long").endObject(); + } + b.endObject(); + } b.endObject(); + b.startObject("foo").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + })); + assertNull(mapperService.fieldType("metrics.service.time")); + assertNotNull(mapperService.fieldType("metrics.service.time.max")); + assertNotNull(mapperService.fieldType("metrics.service.foo")); + assertNotNull(mapperService.documentMapper().mappers().objectMappers().get("metrics.service.time")); // Not flattened. + assertNotNull(mapperService.documentMapper().mappers().getMapper("metrics.service.foo")); + } + + public void testSubobjectsAutoWithInnerNested() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("metrics.service"); + { + b.field("subobjects", "auto"); + b.startObject("properties"); + { + b.startObject("time").field("type", "nested").endObject(); } b.endObject(); } @@ -587,12 +590,8 @@ public void testSubobjectsAutoWithInnerNested() throws IOException { public void testSubobjectsAutoRoot() throws Exception { MapperService mapperService = createMapperService(mappingWithSubobjects(b -> { - b.startObject("metrics.service.time"); - b.field("type", "long"); - b.endObject(); - b.startObject("metrics.service.time.max"); - b.field("type", "long"); - b.endObject(); + b.startObject("metrics.service.time").field("type", "long").endObject(); + b.startObject("metrics.service.time.max").field("type", "long").endObject(); b.startObject("metrics.attributes"); { b.field("type", "object"); @@ -605,15 +604,13 @@ public void testSubobjectsAutoRoot() throws Exception { assertNotNull(mapperService.documentMapper().mappers().objectMappers().get("metrics.attributes")); } - public void testSubobjectsAutoRootWithInnerObject() throws IOException { + public void testSubobjectsAutoRootWithInnerFlattenableObject() throws IOException { MapperService mapperService = createMapperService(mappingWithSubobjects(b -> { b.startObject("metrics.service.time"); { b.startObject("properties"); { - b.startObject("max"); - b.field("type", "long"); - b.endObject(); + b.startObject("max").field("type", "long").endObject(); } b.endObject(); } @@ -621,8 +618,48 @@ public void testSubobjectsAutoRootWithInnerObject() throws IOException { }, "auto")); assertNull(mapperService.fieldType("metrics.service.time")); assertNotNull(mapperService.fieldType("metrics.service.time.max")); - assertNotNull(mapperService.documentMapper().mappers().objectMappers().get("metrics.service.time")); - assertNotNull(mapperService.documentMapper().mappers().getMapper("metrics.service.time.max")); + assertNull(mapperService.documentMapper().mappers().objectMappers().get("metrics.service.time")); // Gets flattened. + + Mapper innerField = mapperService.documentMapper().mappers().getMapper("metrics.service.time.max"); + assertNotNull(innerField); + assertEquals("metrics.service.time.max", innerField.leafName()); + } + + public void testSubobjectsAutoRootWithInnerNonFlattenableObject() throws IOException { + MapperService mapperService = createMapperService(mappingWithSubobjects(b -> { + b.startObject("metrics").startObject("properties"); + { + b.startObject("service.time"); + { + b.field("subobjects", "auto"); + b.startObject("properties"); + { + b.startObject("path").startObject("properties"); + { + b.startObject("to").startObject("properties"); + { + b.startObject("max").field("type", "long").endObject(); + } + b.endObject().endObject(); + } + b.endObject().endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject().endObject(); + }, "auto")); + assertNull(mapperService.fieldType("metrics.service.time")); + assertNotNull(mapperService.fieldType("metrics.service.time.path.to.max")); + + ObjectMapper innerObject = mapperService.documentMapper().mappers().objectMappers().get("metrics.service.time"); // Not flattened. + assertNotNull(innerObject); + assertEquals("metrics.service.time", innerObject.leafName()); + + Mapper innerField = mapperService.documentMapper().mappers().getMapper("metrics.service.time.path.to.max"); + assertNotNull(innerField); + assertEquals("path.to.max", innerField.leafName()); } public void testSubobjectsAutoRootWithInnerNested() throws IOException { @@ -742,16 +779,7 @@ public void testFlatten() { ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Optional.empty()).add( new ObjectMapper.Builder("child", Optional.empty()).add(new KeywordFieldMapper.Builder("keyword2", IndexVersion.current())) ).add(new KeywordFieldMapper.Builder("keyword1", IndexVersion.current())).build(rootContext); - List fields = objectMapper.asFlattenedFieldMappers(rootContext).stream().map(FieldMapper::fullPath).toList(); - assertThat(fields, containsInAnyOrder("parent.keyword1", "parent.child.keyword2")); - } - - public void testFlattenSubobjectsAuto() { - MapperBuilderContext rootContext = MapperBuilderContext.root(false, false); - ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Optional.of(ObjectMapper.Subobjects.AUTO)).add( - new ObjectMapper.Builder("child", Optional.empty()).add(new KeywordFieldMapper.Builder("keyword2", IndexVersion.current())) - ).add(new KeywordFieldMapper.Builder("keyword1", IndexVersion.current())).build(rootContext); - List fields = objectMapper.asFlattenedFieldMappers(rootContext).stream().map(FieldMapper::fullPath).toList(); + List fields = objectMapper.asFlattenedFieldMappers(rootContext, true).stream().map(Mapper::fullPath).toList(); assertThat(fields, containsInAnyOrder("parent.keyword1", "parent.child.keyword2")); } @@ -760,7 +788,7 @@ public void testFlattenSubobjectsFalse() { ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Optional.of(ObjectMapper.Subobjects.DISABLED)).add( new ObjectMapper.Builder("child", Optional.empty()).add(new KeywordFieldMapper.Builder("keyword2", IndexVersion.current())) ).add(new KeywordFieldMapper.Builder("keyword1", IndexVersion.current())).build(rootContext); - List fields = objectMapper.asFlattenedFieldMappers(rootContext).stream().map(FieldMapper::fullPath).toList(); + List fields = objectMapper.asFlattenedFieldMappers(rootContext, true).stream().map(Mapper::fullPath).toList(); assertThat(fields, containsInAnyOrder("parent.keyword1", "parent.child.keyword2")); } @@ -772,7 +800,7 @@ public void testFlattenDynamicIncompatible() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> objectMapper.asFlattenedFieldMappers(rootContext) + () -> objectMapper.asFlattenedFieldMappers(rootContext, true) ); assertEquals( "Object mapper [parent.child] was found in a context where subobjects is set to false. " @@ -788,7 +816,7 @@ public void testFlattenEnabledFalse() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> objectMapper.asFlattenedFieldMappers(rootContext) + () -> objectMapper.asFlattenedFieldMappers(rootContext, true) ); assertEquals( "Object mapper [parent] was found in a context where subobjects is set to false. " @@ -797,13 +825,30 @@ public void testFlattenEnabledFalse() { ); } + public void testFlattenSubobjectsAuto() { + MapperBuilderContext rootContext = MapperBuilderContext.root(false, false); + ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Optional.of(ObjectMapper.Subobjects.AUTO)).add( + new ObjectMapper.Builder("child", Optional.empty()).add(new KeywordFieldMapper.Builder("keyword2", IndexVersion.current())) + ).add(new KeywordFieldMapper.Builder("keyword1", IndexVersion.current())).build(rootContext); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> objectMapper.asFlattenedFieldMappers(rootContext, true) + ); + assertEquals( + "Object mapper [parent] was found in a context where subobjects is set to false. " + + "Auto-flattening [parent] failed because the value of [subobjects] is [auto]", + exception.getMessage() + ); + } + public void testFlattenExplicitSubobjectsTrue() { MapperBuilderContext rootContext = MapperBuilderContext.root(false, false); ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Optional.of(ObjectMapper.Subobjects.ENABLED)).build(rootContext); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> objectMapper.asFlattenedFieldMappers(rootContext) + () -> objectMapper.asFlattenedFieldMappers(rootContext, true) ); assertEquals( "Object mapper [parent] was found in a context where subobjects is set to false. " From 4798037fba43d06672648a9bbfc348e574bd8ae4 Mon Sep 17 00:00:00 2001 From: Iraklis Psaroudakis Date: Thu, 26 Sep 2024 10:52:17 +0200 Subject: [PATCH 12/36] Log exceptions in IndicesLifecycleListener tests (#113547) Because the finally clause assertions did not finally print any exceptions that might have occurred. Happened in build scan qdorbubrxbqh6. And can be easily reproduced e.g., by using a custom metadata: metadata = IndexMetadata.builder(metadata).settings(Settings.builder() .put(metadata.getSettings()).put( IndexMetadata.INDEX_DATA_PATH_SETTING.getKey(), "/invalid/path")).build(); --- .../indices/IndicesLifecycleListenerSingleNodeTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java index 46d03275ac3ce..619714119a05e 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java @@ -128,6 +128,8 @@ public void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRem newRouting = ShardRoutingHelper.moveToStarted(newRouting); IndexShardTestCase.updateRoutingEntry(shard, newRouting); assertEquals(6, counter.get()); + } catch (Exception ex) { + logger.warn("unexpected exception", ex); } finally { indicesService.removeIndex(idx, DELETED, "simon says", EsExecutors.DIRECT_EXECUTOR_SERVICE, ActionListener.noop()); } From e9b303322c81acb4c1fc742fa67d2aa5b9f48910 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 26 Sep 2024 05:11:01 -0400 Subject: [PATCH 13/36] Fix packaging tests after addition of new wolfi-based image (#112831) * Add more missing wolfi references to fix tests * packaging tests require access to docker registry * Fix symlink for es distributions jdk cacerts in wolfi docker * Fix native support on wolfi images * Fix provided keystore packaging tests for wolfi * Add utils used for testing to wolfi image * Explicitly set default shell to bash in docker images * Fix docker config issues * Apply review feedback around docker login --------- Co-authored-by: Rene Groeschke --- .buildkite/hooks/pre-command | 12 ++++++++---- .buildkite/pipelines/periodic-packaging.yml | 3 ++- .ci/scripts/packaging-test.sh | 1 + .../gradle/internal/docker/DockerBuildTask.java | 11 +++++++++++ distribution/docker/src/docker/Dockerfile | 13 +++++++++++-- .../elasticsearch/packaging/test/DockerTests.java | 3 +++ .../packaging/test/KeystoreManagementTests.java | 5 ++++- .../packaging/test/PackagingTestCase.java | 6 ++++-- .../elasticsearch/packaging/util/Distribution.java | 7 +++++-- .../elasticsearch/packaging/util/docker/Docker.java | 4 ++-- .../packaging/util/docker/DockerRun.java | 1 + 11 files changed, 52 insertions(+), 14 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index b6b730fc3de8b..0c0ede8c3a076 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -78,11 +78,15 @@ if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then fi if [[ "${USE_PROD_DOCKER_CREDENTIALS:-}" == "true" ]]; then - DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)" - export DOCKER_REGISTRY_USERNAME + if which docker > /dev/null 2>&1; then + DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)" + export DOCKER_REGISTRY_USERNAME - DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)" - export DOCKER_REGISTRY_PASSWORD + DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)" + export DOCKER_REGISTRY_PASSWORD + + docker login --username "$DOCKER_REGISTRY_USERNAME" --password "$DOCKER_REGISTRY_PASSWORD" docker.elastic.co + fi fi if [[ "$BUILDKITE_AGENT_META_DATA_PROVIDER" != *"k8s"* ]]; then diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 8ef8f5954887e..76cc543a6898e 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -30,7 +30,8 @@ steps: image: family/elasticsearch-{{matrix.image}} diskSizeGb: 350 machineType: n1-standard-8 - env: {} + env: + USE_PROD_DOCKER_CREDENTIALS: "true" - group: packaging-tests-upgrade steps: - label: "{{matrix.image}} / 8.0.1 / packaging-tests-upgrade" diff --git a/.ci/scripts/packaging-test.sh b/.ci/scripts/packaging-test.sh index 6b9938dabffa8..bb7547933b213 100755 --- a/.ci/scripts/packaging-test.sh +++ b/.ci/scripts/packaging-test.sh @@ -77,5 +77,6 @@ sudo -E env \ --unset=ES_JAVA_HOME \ --unset=JAVA_HOME \ SYSTEM_JAVA_HOME=`readlink -f -n $BUILD_JAVA_HOME` \ + DOCKER_CONFIG="${HOME}/.docker" \ ./gradlew -g $HOME/.gradle --scan --parallel --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ --continue $@ diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java index 8971f27838578..9b28401994ee2 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java @@ -30,6 +30,7 @@ import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; import org.gradle.process.ExecOperations; +import org.gradle.process.ExecSpec; import org.gradle.workers.WorkAction; import org.gradle.workers.WorkParameters; import org.gradle.workers.WorkerExecutor; @@ -166,6 +167,7 @@ private void pullBaseImage(String baseImage) { for (int attempt = 1; attempt <= maxAttempts; attempt++) { try { LoggedExec.exec(execOperations, spec -> { + maybeConfigureDockerConfig(spec); spec.executable("docker"); spec.args("pull"); spec.args(baseImage); @@ -181,6 +183,13 @@ private void pullBaseImage(String baseImage) { throw new GradleException("Failed to pull Docker base image [" + baseImage + "], all attempts failed"); } + private void maybeConfigureDockerConfig(ExecSpec spec) { + String dockerConfig = System.getenv("DOCKER_CONFIG"); + if (dockerConfig != null) { + spec.environment("DOCKER_CONFIG", dockerConfig); + } + } + @Override public void execute() { final Parameters parameters = getParameters(); @@ -193,6 +202,8 @@ public void execute() { final boolean isCrossPlatform = isCrossPlatform(); LoggedExec.exec(execOperations, spec -> { + maybeConfigureDockerConfig(spec); + spec.executable("docker"); if (isCrossPlatform) { diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index 47f79749cbefa..fd2516f2fdc9a 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -163,9 +163,16 @@ RUN <%= retry.loop(package_manager, " ${package_manager} update && \n" + " ${package_manager} upgrade && \n" + " ${package_manager} add --no-cache \n" + - " bash ca-certificates curl libsystemd netcat-openbsd p11-kit p11-kit-trust shadow tini unzip zip zstd && \n" + + " bash java-cacerts curl libstdc++ libsystemd netcat-openbsd p11-kit p11-kit-trust posix-libc-utils shadow tini unzip zip zstd && \n" + " rm -rf /var/cache/apk/* " ) %> + +# Set Bash as the default shell for future commands +SHELL ["/bin/bash", "-c"] + +# Optionally set Bash as the default shell in the container at runtime +CMD ["/bin/bash"] + <% } else if (docker_base == "default" || docker_base == "cloud") { %> # Change default shell to bash, then install required packages with retries. @@ -224,7 +231,7 @@ COPY --from=builder --chown=0:0 /opt /opt <% } %> ENV PATH /usr/share/elasticsearch/bin:\$PATH - +ENV SHELL /bin/bash COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh # 1. Sync the user and group permissions of /etc/passwd @@ -249,6 +256,8 @@ RUN chmod g=u /etc/passwd && \\ # stays up-to-date with changes to Ubuntu's store) COPY bin/docker-openjdk /etc/ca-certificates/update.d/docker-openjdk RUN /etc/ca-certificates/update.d/docker-openjdk +<% } else if (docker_base == 'wolfi') { %> +RUN ln -sf /etc/ssl/certs/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts <% } else { %> RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts <% } %> diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index a9402c324f7fc..f588b78c78cc8 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -386,6 +386,9 @@ public void test040JavaUsesTheOsProvidedKeystore() { if (distribution.packaging == Packaging.DOCKER_UBI || distribution.packaging == Packaging.DOCKER_IRON_BANK) { // In these images, the `cacerts` file ought to be a symlink here assertThat(path, equalTo("/etc/pki/ca-trust/extracted/java/cacerts")); + } else if (distribution.packaging == Packaging.DOCKER_WOLFI) { + // In these images, the `cacerts` file ought to be a symlink here + assertThat(path, equalTo("/etc/ssl/certs/java/cacerts")); } else { // Whereas on other images, it's a real file so the real path is the same assertThat(path, equalTo("/usr/share/elasticsearch/jdk/lib/security/cacerts")); diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java index 5b86796aa80ca..a988a446f561f 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java @@ -436,7 +436,10 @@ private void verifyKeystorePermissions() { switch (distribution.packaging) { case TAR, ZIP -> assertThat(keystore, file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660)); case DEB, RPM -> assertThat(keystore, file(File, "root", "elasticsearch", p660)); - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS -> assertThat(keystore, DockerFileMatcher.file(p660)); + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> assertThat( + keystore, + DockerFileMatcher.file(p660) + ); default -> throw new IllegalStateException("Unknown Elasticsearch packaging type."); } } diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java index a1a9af3b6e307..644990105f60f 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java @@ -245,7 +245,7 @@ protected static void install() throws Exception { installation = Packages.installPackage(sh, distribution); Packages.verifyPackageInstallation(installation, distribution, sh); } - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS -> { + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> { installation = Docker.runContainer(distribution); Docker.verifyContainerInstallation(installation); } @@ -337,6 +337,7 @@ public Shell.Result runElasticsearchStartCommand(String password, boolean daemon case DOCKER_IRON_BANK: case DOCKER_CLOUD: case DOCKER_CLOUD_ESS: + case DOCKER_WOLFI: // nothing, "installing" docker image is running it return Shell.NO_OP; default: @@ -359,6 +360,7 @@ public void stopElasticsearch() throws Exception { case DOCKER_IRON_BANK: case DOCKER_CLOUD: case DOCKER_CLOUD_ESS: + case DOCKER_WOLFI: // nothing, "installing" docker image is running it break; default: @@ -371,7 +373,7 @@ public void awaitElasticsearchStartup(Shell.Result result) throws Exception { switch (distribution.packaging) { case TAR, ZIP -> Archives.assertElasticsearchStarted(installation); case DEB, RPM -> Packages.assertElasticsearchStarted(sh, installation); - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS -> Docker.waitForElasticsearchToStart(); + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> Docker.waitForElasticsearchToStart(); default -> throw new IllegalStateException("Unknown Elasticsearch packaging type."); } } diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java index b3ea54425af8e..05cef4a0818ba 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java @@ -37,6 +37,8 @@ public Distribution(Path path) { this.packaging = Packaging.DOCKER_CLOUD; } else if (filename.endsWith(".cloud-ess.tar")) { this.packaging = Packaging.DOCKER_CLOUD_ESS; + } else if (filename.endsWith(".wolfi.tar")) { + this.packaging = Packaging.DOCKER_WOLFI; } else { int lastDot = filename.lastIndexOf('.'); this.packaging = Packaging.valueOf(filename.substring(lastDot + 1).toUpperCase(Locale.ROOT)); @@ -61,7 +63,7 @@ public boolean isPackage() { */ public boolean isDocker() { return switch (packaging) { - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS -> true; + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> true; default -> false; }; } @@ -76,7 +78,8 @@ public enum Packaging { DOCKER_UBI(".ubi.tar", Platforms.isDocker()), DOCKER_IRON_BANK(".ironbank.tar", Platforms.isDocker()), DOCKER_CLOUD(".cloud.tar", Platforms.isDocker()), - DOCKER_CLOUD_ESS(".cloud-ess.tar", Platforms.isDocker()); + DOCKER_CLOUD_ESS(".cloud-ess.tar", Platforms.isDocker()), + DOCKER_WOLFI(".wolfi.tar", Platforms.isDocker()); /** The extension of this distribution's file */ public final String extension; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java index cb8a955a5972c..c38eaa58f0552 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java @@ -486,9 +486,9 @@ public static void verifyContainerInstallation(Installation es) { // Ensure the `elasticsearch` user and group exist. // These lines will both throw an exception if the command fails dockerShell.run("id elasticsearch"); - dockerShell.run("getent group elasticsearch"); + dockerShell.run("grep -E '^elasticsearch:' /etc/group"); - final Shell.Result passwdResult = dockerShell.run("getent passwd elasticsearch"); + final Shell.Result passwdResult = dockerShell.run("grep -E '^elasticsearch:' /etc/passwd"); final String homeDir = passwdResult.stdout().trim().split(":")[5]; assertThat("elasticsearch user's home directory is incorrect", homeDir, equalTo("/usr/share/elasticsearch")); diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java index 6c58bcba09879..2b3eb7ff7a617 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java @@ -167,6 +167,7 @@ public static String getImageName(Distribution distribution) { case DOCKER_IRON_BANK -> "-ironbank"; case DOCKER_CLOUD -> "-cloud"; case DOCKER_CLOUD_ESS -> "-cloud-ess"; + case DOCKER_WOLFI -> "-wolfi"; default -> throw new IllegalStateException("Unexpected distribution packaging type: " + distribution.packaging); }; From 11c0bf8d6ee4bbd6e982a04211cc3994c5dd48e0 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 26 Sep 2024 11:19:46 +0200 Subject: [PATCH 14/36] Fix cancellation race condition in `onIndexAvailableForSearch` (#113386) This PR fixes following race conditions in `onIndexAvailableForSearch` introduced in https://github.com/elastic/elasticsearch/pull/112813: 1. If the method is called when the index is already available, cancellation is still scheduled and may execute before successful completion (manifested in test failures https://github.com/elastic/elasticsearch/issues/113336) 2. If the cancel task runs _before_ `addStateListener`, it may fail to remove the listener (noticed while fixing the first issue) These race conditions only manifest for small timeout windows, and are completely bypassed for 0 timeout windows based on other checks in prod code, so the practical impact is fortunately limited. Resolves: https://github.com/elastic/elasticsearch/issues/113336 --- muted-tests.yml | 3 - .../SecurityIndexManagerIntegTests.java | 110 +++++++++++++++++- .../support/SecurityIndexManager.java | 86 ++++++++++---- 3 files changed, 168 insertions(+), 31 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 602d790246648..728f3d0bd6c72 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -251,9 +251,6 @@ tests: - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=transform/transforms_force_delete/Test force deleting a running transform} issue: https://github.com/elastic/elasticsearch/issues/113327 -- class: org.elasticsearch.xpack.security.support.SecurityIndexManagerIntegTests - method: testOnIndexAvailableForSearchIndexAlreadyAvailable - issue: https://github.com/elastic/elasticsearch/issues/113336 - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=analytics/top_metrics/sort by scaled float field} issue: https://github.com/elastic/elasticsearch/issues/113340 diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java index 32337f0d66896..44cbf03f220a1 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java @@ -26,18 +26,24 @@ import org.elasticsearch.xpack.core.security.action.user.PutUserResponse; import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore; import org.hamcrest.Matchers; +import org.junit.After; import org.junit.Before; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; @@ -45,6 +51,14 @@ public class SecurityIndexManagerIntegTests extends SecurityIntegTestCase { + private final int concurrentCallsToOnAvailable = 6; + private final ExecutorService executor = Executors.newFixedThreadPool(concurrentCallsToOnAvailable); + + @After + public void shutdownExecutor() { + executor.shutdown(); + } + public void testConcurrentOperationsTryingToCreateSecurityIndexAndAlias() throws Exception { final int processors = Runtime.getRuntime().availableProcessors(); final int numThreads = Math.min(50, scaledRandomIntBetween((processors + 1) / 2, 4 * processors)); // up to 50 threads @@ -110,6 +124,12 @@ public void testOnIndexAvailableForSearchIndexCompletesWithinTimeout() throws Ex // pick longer wait than in the assertBusy that waits for below to ensure index has had enough time to initialize securityIndexManager.onIndexAvailableForSearch((ActionListener) future, TimeValue.timeValueSeconds(40)); + // check listener added + assertThat( + securityIndexManager.getStateChangeListeners(), + hasItem(instanceOf(SecurityIndexManager.StateConsumerWithCancellable.class)) + ); + createSecurityIndexWithWaitForActiveShards(); assertBusy( @@ -121,6 +141,12 @@ public void testOnIndexAvailableForSearchIndexCompletesWithinTimeout() throws Ex // security index creation is complete and index is available for search; therefore whenIndexAvailableForSearch should report // success in time future.actionGet(); + + // check no remaining listeners + assertThat( + securityIndexManager.getStateChangeListeners(), + not(hasItem(instanceOf(SecurityIndexManager.StateConsumerWithCancellable.class))) + ); } @SuppressWarnings("unchecked") @@ -152,6 +178,69 @@ public void testOnIndexAvailableForSearchIndexAlreadyAvailable() throws Exceptio securityIndexManager.onIndexAvailableForSearch((ActionListener) future, TimeValue.timeValueSeconds(10)); future.actionGet(); } + + // check no remaining listeners + assertThat( + securityIndexManager.getStateChangeListeners(), + not(hasItem(instanceOf(SecurityIndexManager.StateConsumerWithCancellable.class))) + ); + } + + @SuppressWarnings("unchecked") + public void testOnIndexAvailableForSearchIndexUnderConcurrentLoad() throws Exception { + final SecurityIndexManager securityIndexManager = internalCluster().getInstances(NativePrivilegeStore.class) + .iterator() + .next() + .getSecurityIndexManager(); + // Long time out calls should all succeed + final List> futures = new ArrayList<>(); + for (int i = 0; i < concurrentCallsToOnAvailable / 2; i++) { + final Future future = executor.submit(() -> { + try { + final ActionFuture f = new PlainActionFuture<>(); + securityIndexManager.onIndexAvailableForSearch((ActionListener) f, TimeValue.timeValueSeconds(40)); + f.actionGet(); + } catch (Exception ex) { + fail(ex, "should not have encountered exception"); + } + return null; + }); + futures.add(future); + } + + // short time-out tasks should all time out + for (int i = 0; i < concurrentCallsToOnAvailable / 2; i++) { + final Future future = executor.submit(() -> { + expectThrows(ElasticsearchTimeoutException.class, () -> { + final ActionFuture f = new PlainActionFuture<>(); + securityIndexManager.onIndexAvailableForSearch((ActionListener) f, TimeValue.timeValueMillis(10)); + f.actionGet(); + }); + return null; + }); + futures.add(future); + } + + // Sleep a second for short-running calls to timeout + Thread.sleep(1000); + + createSecurityIndexWithWaitForActiveShards(); + // ensure security index manager state is fully in the expected precondition state for this test (ready for search) + assertBusy( + () -> assertThat(securityIndexManager.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS), is(true)), + 30, + TimeUnit.SECONDS + ); + + for (var future : futures) { + future.get(10, TimeUnit.SECONDS); + } + + // check no remaining listeners + assertThat( + securityIndexManager.getStateChangeListeners(), + not(hasItem(instanceOf(SecurityIndexManager.StateConsumerWithCancellable.class))) + ); } @SuppressWarnings("unchecked") @@ -163,9 +252,24 @@ public void testOnIndexAvailableForSearchIndexWaitTimeOut() { .next() .getSecurityIndexManager(); - final ActionFuture future = new PlainActionFuture<>(); - securityIndexManager.onIndexAvailableForSearch((ActionListener) future, TimeValue.timeValueMillis(100)); - expectThrows(ElasticsearchTimeoutException.class, future::actionGet); + { + final ActionFuture future = new PlainActionFuture<>(); + securityIndexManager.onIndexAvailableForSearch((ActionListener) future, TimeValue.timeValueMillis(100)); + expectThrows(ElasticsearchTimeoutException.class, future::actionGet); + } + + // Also works with 0 timeout + { + final ActionFuture future = new PlainActionFuture<>(); + securityIndexManager.onIndexAvailableForSearch((ActionListener) future, TimeValue.timeValueMillis(0)); + expectThrows(ElasticsearchTimeoutException.class, future::actionGet); + } + + // check no remaining listeners + assertThat( + securityIndexManager.getStateChangeListeners(), + not(hasItem(instanceOf(SecurityIndexManager.StateConsumerWithCancellable.class))) + ); } public void testSecurityIndexSettingsCannotBeChanged() throws Exception { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index a6c8de003c159..6d9b0ef6aeebe 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -54,6 +54,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -364,45 +365,80 @@ public void accept(State previousState, State nextState) { * Notifies {@code listener} once the security index is available, or calls {@code onFailure} on {@code timeout}. */ public void onIndexAvailableForSearch(ActionListener listener, TimeValue timeout) { - logger.info("Will wait for security index [{}] to become available for search", getConcreteIndexName()); + logger.info("Will wait for security index [{}] for [{}] to become available for search", getConcreteIndexName(), timeout); - final ActionListener notifyOnceListener = ActionListener.notifyOnce(listener); + if (state.indexAvailableForSearch) { + logger.debug("Security index [{}] is already available", getConcreteIndexName()); + listener.onResponse(null); + return; + } + final AtomicBoolean isDone = new AtomicBoolean(false); final var indexAvailableForSearchListener = new StateConsumerWithCancellable() { @Override public void accept(SecurityIndexManager.State previousState, SecurityIndexManager.State nextState) { if (nextState.indexAvailableForSearch) { - assert cancellable != null; - // cancel and removeStateListener are idempotent - cancellable.cancel(); - removeStateListener(this); - notifyOnceListener.onResponse(null); + if (isDone.compareAndSet(false, true)) { + cancel(); + removeStateListener(this); + listener.onResponse(null); + } } } }; + // add listener _before_ registering timeout -- this way we are guaranteed it gets removed (either by timeout below, or successful + // completion above) + addStateListener(indexAvailableForSearchListener); + // schedule failure handling on timeout -- keep reference to cancellable so a successful completion can cancel the timeout - indexAvailableForSearchListener.cancellable = client.threadPool().schedule(() -> { - removeStateListener(indexAvailableForSearchListener); - notifyOnceListener.onFailure( - new ElasticsearchTimeoutException( - "timed out waiting for security index [" + getConcreteIndexName() + "] to become available for search" - ) - ); - }, timeout, client.threadPool().generic()); + indexAvailableForSearchListener.setCancellable(client.threadPool().schedule(() -> { + if (isDone.compareAndSet(false, true)) { + removeStateListener(indexAvailableForSearchListener); + listener.onFailure( + new ElasticsearchTimeoutException( + "timed out waiting for security index [" + getConcreteIndexName() + "] to become available for search" + ) + ); + } + }, timeout, client.threadPool().generic())); + } - // in case the state has meanwhile changed to available, return immediately - if (state.indexAvailableForSearch) { - indexAvailableForSearchListener.cancellable.cancel(); - notifyOnceListener.onResponse(null); - } else { - addStateListener(indexAvailableForSearchListener); - } + // pkg-private for testing + List> getStateChangeListeners() { + return stateChangeListeners; } - private abstract static class StateConsumerWithCancellable + /** + * This class ensures that if cancel() is called _before_ setCancellable(), the passed-in cancellable is still correctly cancelled on + * a subsequent setCancellable() call. + */ + // pkg-private for testing + abstract static class StateConsumerWithCancellable implements - BiConsumer { - volatile Scheduler.ScheduledCancellable cancellable; + BiConsumer, + Scheduler.Cancellable { + private volatile Scheduler.ScheduledCancellable cancellable; + private volatile boolean cancelled = false; + + void setCancellable(Scheduler.ScheduledCancellable cancellable) { + this.cancellable = cancellable; + if (cancelled) { + cancel(); + } + } + + public boolean cancel() { + cancelled = true; + if (cancellable != null) { + // cancellable is idempotent, so it's fine to potentially call it multiple times + return cancellable.cancel(); + } + return isCancelled(); + } + + public boolean isCancelled() { + return cancelled; + } } private Tuple checkIndexAvailable(ClusterState state) { From 0d275c65dca4aeba90bd8ed632644a433c817408 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 26 Sep 2024 12:06:28 +0200 Subject: [PATCH 15/36] Log when clients are randomly reset in ITs (#113510) This is useful for debugging tests and should not be too noisy since it's a rare event. --- .../main/java/org/elasticsearch/test/InternalTestCluster.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index ff66d59a21c5b..7a04384298933 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -1536,7 +1536,9 @@ private void randomlyResetClients() { // only reset the clients on nightly tests, it causes heavy load... if (RandomizedTest.isNightly() && rarely(random)) { final Collection nodesAndClients = nodes.values(); + logger.info("Resetting [{}] node clients on internal test cluster", nodesAndClients.size()); for (NodeAndClient nodeAndClient : nodesAndClients) { + logger.info("Resetting [{}] node client on internal test cluster", nodeAndClient.name); nodeAndClient.resetClient(); } } From 98db01b271680732a0862db1a07f6677b49672c3 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Thu, 26 Sep 2024 12:22:49 +0200 Subject: [PATCH 16/36] ES|QL: Fix warnings for date tests (#113586) Fixes: https://github.com/elastic/elasticsearch/issues/113540 Fixes: https://github.com/elastic/elasticsearch/issues/113539 More generic warning regex for Java 23 date patterns (already fixed in 8.x, no need to backport) --- muted-tests.yml | 6 ------ .../resources/rest-api-spec/test/esql/70_locale.yml | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 728f3d0bd6c72..cf821355b1d3c 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -290,12 +290,6 @@ tests: - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates} issue: https://github.com/elastic/elasticsearch/issues/113537 -- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT - method: test {p0=esql/70_locale/Date format with default locale} - issue: https://github.com/elastic/elasticsearch/issues/113539 -- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT - method: test {p0=esql/70_locale/Date format with Italian locale} - issue: https://github.com/elastic/elasticsearch/issues/113540 - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT method: testPutE5WithTrainedModelAndInference issue: https://github.com/elastic/elasticsearch/issues/113565 diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml index 05edf6cdfb5a8..5a9a2a21e21bc 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml @@ -29,7 +29,7 @@ setup: - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" - - "Date format \\[MMMM\\] contains textual field specifiers that could change in JDK 23" + - "Date format \\[MMMM\\] contains textual field specifiers that could change in JDK 23.*" esql.query: body: query: 'FROM events | eval fixed_format = date_format("MMMM", @timestamp), variable_format = date_format(format, @timestamp) | sort @timestamp | keep @timestamp, fixed_format, variable_format' @@ -51,7 +51,7 @@ setup: - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" - - "Date format \\[MMMM\\] contains textual field specifiers that could change in JDK 23" + - "Date format \\[MMMM\\] contains textual field specifiers that could change in JDK 23.*" esql.query: body: query: 'FROM events | eval fixed_format = date_format("MMMM", @timestamp), variable_format = date_format(format, @timestamp) | sort @timestamp | keep @timestamp, fixed_format, variable_format' From b344493298a0db19f9e95d8252fe90e79102dd5a Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:57:31 +0300 Subject: [PATCH 17/36] Rest skipped tests after backporting (#113591) The skip test entries were added in https://github.com/elastic/elasticsearch/pull/113584, no longer needed after backporting it. --- modules/dot-prefix-validation/build.gradle | 10 ---------- rest-api-spec/build.gradle | 22 ---------------------- 2 files changed, 32 deletions(-) diff --git a/modules/dot-prefix-validation/build.gradle b/modules/dot-prefix-validation/build.gradle index b300cae20d717..6e232570b4a22 100644 --- a/modules/dot-prefix-validation/build.gradle +++ b/modules/dot-prefix-validation/build.gradle @@ -27,13 +27,3 @@ tasks.named('yamlRestTest') { tasks.named('yamlRestCompatTest') { usesDefaultDistribution() } - -tasks.named("yamlRestCompatTestTransform").configure( - { task -> - task.skipTest("tsdb/140_routing_path/multi-value routing path field", "Multi-value routing paths are allowed now. See #112645") - task.skipTest( - "dot_prefix/10_basic/Deprecated index template with a dot prefix index pattern", - "Tentantively disabled until #112092 gets backported to 8.x" - ) - } -) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 1d69c170d7553..a742e83255bbb 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -55,28 +55,6 @@ tasks.named("precommit").configure { } tasks.named("yamlRestCompatTestTransform").configure({task -> - task.skipTest("tsdb/140_routing_path/multi-value routing path field", "Multi-value routing paths are allowed now. See #112645") task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") - task.skipTest("search/540_ignore_above_synthetic_source/ignore_above mapping level setting on arrays", "Temporary mute while backporting to 8.x") - task.skipTest("indices.create/20_synthetic_source/subobjects auto", "Tentantively disabled until #112092 gets backported to 8.x") - task.skipTest( - "index/92_metrics_auto_subobjects/Metrics object indexing with synthetic source", - "Tentantively disabled until #112092 gets backported to 8.x" - ) - task.skipTest( - "index/92_metrics_auto_subobjects/Root without subobjects with synthetic source", - "Tentantively disabled until #112092 gets backported to 8.x" - ) - task.skipTest( - "indices.put_index_template/15_composition/Composable index templates that include subobjects: auto at root", - "Tentantively disabled until #112092 gets backported to 8.x" - ) - task.skipTest( - "indices.put_index_template/15_composition/Composable index templates that include subobjects: auto on arbitrary field", - "Tentantively disabled until #112092 gets backported to 8.x" - ) - task.skipTest("index/92_metrics_auto_subobjects/Metrics object indexing", "Tentantively disabled until #112092 gets backported to 8.x") - task.skipTest("index/92_metrics_auto_subobjects/Root with metrics", "Tentantively disabled until #112092 gets backported to 8.x") - task.skipTest("search/330_fetch_fields/Test with subobjects: auto", "Tentantively disabled until #112092 gets backported to 8.x") }) From 5c6778cc8632950a50da4f34151bd0403c806dc6 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 26 Sep 2024 22:26:40 +1000 Subject: [PATCH 18/36] Mute org.elasticsearch.integration.KibanaUserRoleIntegTests testFieldMappings #113592 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index cf821355b1d3c..528f1e3be8d17 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -299,6 +299,9 @@ tests: - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testCantCreateJobWithSameID issue: https://github.com/elastic/elasticsearch/issues/113581 +- class: org.elasticsearch.integration.KibanaUserRoleIntegTests + method: testFieldMappings + issue: https://github.com/elastic/elasticsearch/issues/113592 # Examples: # From fc9954e031e3f94fca58765755fbddae93115656 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 26 Sep 2024 22:26:58 +1000 Subject: [PATCH 19/36] Mute org.elasticsearch.integration.KibanaUserRoleIntegTests testSearchAndMSearch #113593 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 528f1e3be8d17..5d7474af06d86 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -302,6 +302,9 @@ tests: - class: org.elasticsearch.integration.KibanaUserRoleIntegTests method: testFieldMappings issue: https://github.com/elastic/elasticsearch/issues/113592 +- class: org.elasticsearch.integration.KibanaUserRoleIntegTests + method: testSearchAndMSearch + issue: https://github.com/elastic/elasticsearch/issues/113593 # Examples: # From e2281a1158976bd5fac73c8e77e9a67c769dd3b2 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:44:03 +0200 Subject: [PATCH 20/36] Introduce an `IndexSettingsProvider` to inject logsdb index mode (#113505) Here we introduce a new implementation of `IndexSettingProvider` whose goal is to "inject" the `index.mode` setting with value `logsdb` when a cluster setting `cluster.logsdb.enabled` is `true`. We also make sure that: * the existing `index.mode` is not set * the datastream name matches the `logs-*-*` pattern * `logs@settings` component template is used --- .../LogsIndexModeDisabledRestTestIT.java | 70 +++- .../LogsIndexModeEnabledRestTestIT.java | 87 ++++- .../logsdb/LogsIndexModeRestTestIT.java | 19 +- .../core/src/main/java/module-info.java | 1 + .../cluster/settings/ClusterSettings.java | 19 + .../main/resources/logs@settings-logsdb.json | 26 ++ .../src/main/resources/logs@settings.json | 1 - .../xpack/logsdb/LogsDBPlugin.java | 12 +- .../LogsdbIndexModeSettingsProvider.java | 89 +++++ .../LogsdbIndexModeSettingsProviderTests.java | 326 ++++++++++++++++++ .../stack/LegacyStackTemplateRegistry.java | 7 +- .../xpack/stack/StackPlugin.java | 2 +- .../xpack/stack/StackTemplateRegistry.java | 21 +- 13 files changed, 648 insertions(+), 32 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/settings/ClusterSettings.java create mode 100644 x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json create mode 100644 x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java create mode 100644 x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java index c9818a34169de..123ca3b806153 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.hamcrest.Matchers; @@ -23,6 +24,22 @@ public class LogsIndexModeDisabledRestTestIT extends LogsIndexModeRestTestIT { + private static final String MAPPINGS = """ + { + "template": { + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "message": { + "type": "text" + } + } + } + } + }"""; + @ClassRule() public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) @@ -50,8 +67,59 @@ public void setup() throws Exception { public void testLogsSettingsIndexModeDisabled() throws IOException { assertOK(createDataStream(client, "logs-custom-dev")); - final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); + final String indexMode = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); assertThat(indexMode, Matchers.not(equalTo(IndexMode.LOGSDB.getName()))); } + public void testTogglingLogsdb() throws IOException { + putComponentTemplate(client, "logs@settings", MAPPINGS); + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexModeBefore = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); + assertThat(indexModeBefore, Matchers.not(equalTo(IndexMode.LOGSDB.getName()))); + assertOK(putClusterSetting(client, "cluster.logsdb.enabled", "true")); + final String indexModeAfter = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); + assertThat(indexModeAfter, Matchers.not(equalTo(IndexMode.LOGSDB.getName()))); + assertOK(rolloverDataStream(client, "logs-custom-dev")); + final String indexModeLater = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 1), + IndexSettings.MODE.getKey() + ); + assertThat(indexModeLater, equalTo(IndexMode.LOGSDB.getName())); + assertOK(putClusterSetting(client, "cluster.logsdb.enabled", "false")); + assertOK(rolloverDataStream(client, "logs-custom-dev")); + final String indexModeFinal = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 2), + IndexSettings.MODE.getKey() + ); + assertThat(indexModeFinal, Matchers.not(equalTo(IndexMode.LOGSDB.getName()))); + + } + + public void testEnablingLogsdb() throws IOException { + putComponentTemplate(client, "logs@settings", MAPPINGS); + assertOK(putClusterSetting(client, "cluster.logsdb.enabled", true)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexMode = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); + assertThat(indexMode, equalTo(IndexMode.LOGSDB.getName())); + assertOK(putClusterSetting(client, "cluster.logsdb.enabled", false)); + } + } diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java index d7bdf54007d69..a024a2c0f303c 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java @@ -10,8 +10,10 @@ package org.elasticsearch.datastreams.logsdb; import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.hamcrest.Matchers; @@ -179,7 +181,11 @@ public void setup() throws Exception { public void testCreateDataStream() throws IOException { assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); assertOK(createDataStream(client, "logs-custom-dev")); - final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); + final String indexMode = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); assertThat(indexMode, equalTo(IndexMode.LOGSDB.getName())); } @@ -224,4 +230,83 @@ public void testRolloverDataStream() throws IOException { assertThat(firstBackingIndex, Matchers.not(equalTo(secondBackingIndex))); assertThat(getDataStreamBackingIndices(client, "logs-custom-dev").size(), equalTo(2)); } + + public void testLogsAtSettingWithStandardOverride() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", """ + { + "template": { + "settings": { + "index": { + "mode": "standard" + } + } + } + } + """)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexMode = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); + assertThat(indexMode, equalTo(IndexMode.STANDARD.getName())); + } + + public void testLogsAtSettingWithTimeSeriesOverride() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", """ + { + "template": { + "settings": { + "index": { + "routing_path": [ "hostname" ], + "mode": "time_series", + "sort.field": [], + "sort.order": [] + } + }, + "mappings": { + "properties": { + "hostname": { + "type": "keyword", + "time_series_dimension": true + } + } + } + } + } + """)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexMode = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); + assertThat(indexMode, equalTo(IndexMode.TIME_SERIES.getName())); + } + + public void testLogsAtSettingWithTimeSeriesOverrideFailure() { + // NOTE: apm@settings defines sorting on @timestamp and template composition results in index.mode "time_series" + // with a non-allowed index.sort.field '@timestamp'. This fails at template composition stage before the index is even created. + final ResponseException ex = assertThrows(ResponseException.class, () -> putComponentTemplate(client, "logs@custom", """ + { + "template": { + "settings": { + "index": { + "routing_path": [ "hostname" ], + "mode": "time_series" + } + }, + "mappings": { + "properties": { + "hostname": { + "type": "keyword", + "time_series_dimension": true + } + } + } + } + } + """)); + assertTrue(ex.getMessage().contains("[index.mode=time_series] is incompatible with [index.sort.field]")); + } } diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java index 7d65207794598..22ac2b6d7d239 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java @@ -33,10 +33,16 @@ protected static void waitForLogs(RestClient client) throws Exception { }); } - protected static Response putComponentTemplate(final RestClient client, final String templateName, final String mappings) + protected static Response putComponentTemplate(final RestClient client, final String componentTemplate, final String contends) throws IOException { - final Request request = new Request("PUT", "/_component_template/" + templateName); - request.setJsonEntity(mappings); + final Request request = new Request("PUT", "/_component_template/" + componentTemplate); + request.setJsonEntity(contends); + return client.performRequest(request); + } + + protected static Response putTemplate(final RestClient client, final String template, final String contents) throws IOException { + final Request request = new Request("PUT", "/_index_template/" + template); + request.setJsonEntity(contents); return client.performRequest(request); } @@ -87,4 +93,11 @@ protected static Response bulkIndex(final RestClient client, final String dataSt bulkRequest.addParameter("refresh", "true"); return client.performRequest(bulkRequest); } + + protected static Response putClusterSetting(final RestClient client, final String settingName, final Object settingValue) + throws IOException { + final Request request = new Request("PUT", "/_cluster/settings"); + request.setJsonEntity("{ \"transient\": { \"" + settingName + "\": " + settingValue + " } }"); + return client.performRequest(request); + } } diff --git a/x-pack/plugin/core/src/main/java/module-info.java b/x-pack/plugin/core/src/main/java/module-info.java index 72436bb9d5171..47848310fe781 100644 --- a/x-pack/plugin/core/src/main/java/module-info.java +++ b/x-pack/plugin/core/src/main/java/module-info.java @@ -228,6 +228,7 @@ exports org.elasticsearch.xpack.core.watcher.trigger; exports org.elasticsearch.xpack.core.watcher.watch; exports org.elasticsearch.xpack.core.watcher; + exports org.elasticsearch.xpack.cluster.settings; provides org.elasticsearch.action.admin.cluster.node.info.ComponentVersionNumber with diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/settings/ClusterSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/settings/ClusterSettings.java new file mode 100644 index 0000000000000..1127889783f16 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/settings/ClusterSettings.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.cluster.settings; + +import org.elasticsearch.common.settings.Setting; + +public class ClusterSettings { + public static final Setting CLUSTER_LOGSDB_ENABLED = Setting.boolSetting( + "cluster.logsdb.enabled", + false, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); +} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json new file mode 100644 index 0000000000000..eabdd6fb9fad2 --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json @@ -0,0 +1,26 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "logs" + }, + "mode": "logsdb", + "codec": "best_compression", + "mapping": { + "ignore_malformed": true, + "total_fields": { + "ignore_dynamic_beyond_limit": true + } + }, + "default_pipeline": "logs@default-pipeline" + } + } + }, + "_meta": { + "description": "default settings for the logs index template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version}, + "deprecated": ${xpack.stack.template.deprecated} +} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json index e9a9f2611ad7b..ca2659b8d8dea 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json @@ -5,7 +5,6 @@ "lifecycle": { "name": "logs" }, - "mode": "${xpack.stack.template.logsdb.index.mode}", "codec": "best_compression", "mapping": { "ignore_malformed": true, diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java index e38f953be96a3..833555a7884ea 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java @@ -17,6 +17,7 @@ import java.util.Collection; import java.util.List; +import static org.elasticsearch.xpack.cluster.settings.ClusterSettings.CLUSTER_LOGSDB_ENABLED; import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseService.FALLBACK_SETTING; public class LogsDBPlugin extends Plugin { @@ -24,9 +25,12 @@ public class LogsDBPlugin extends Plugin { private final Settings settings; private final SyntheticSourceLicenseService licenseService; + private final LogsdbIndexModeSettingsProvider logsdbIndexModeSettingsProvider; + public LogsDBPlugin(Settings settings) { this.settings = settings; this.licenseService = new SyntheticSourceLicenseService(settings); + this.logsdbIndexModeSettingsProvider = new LogsdbIndexModeSettingsProvider(settings); } @Override @@ -34,6 +38,10 @@ public Collection createComponents(PluginServices services) { licenseService.setLicenseState(XPackPlugin.getSharedLicenseState()); var clusterSettings = services.clusterService().getClusterSettings(); clusterSettings.addSettingsUpdateConsumer(FALLBACK_SETTING, licenseService::setSyntheticSourceFallback); + clusterSettings.addSettingsUpdateConsumer( + CLUSTER_LOGSDB_ENABLED, + logsdbIndexModeSettingsProvider::updateClusterIndexModeLogsdbEnabled + ); // Nothing to share here: return super.createComponents(services); } @@ -43,11 +51,11 @@ public Collection getAdditionalIndexSettingProviders(Index if (DiscoveryNode.isStateless(settings)) { return List.of(); } - return List.of(new SyntheticSourceIndexSettingsProvider(licenseService)); + return List.of(new SyntheticSourceIndexSettingsProvider(licenseService), logsdbIndexModeSettingsProvider); } @Override public List> getSettings() { - return List.of(FALLBACK_SETTING); + return List.of(FALLBACK_SETTING, CLUSTER_LOGSDB_ENABLED); } } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java new file mode 100644 index 0000000000000..3f6bb66dfa438 --- /dev/null +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettingProvider; +import org.elasticsearch.index.IndexSettings; + +import java.time.Instant; +import java.util.List; +import java.util.Locale; + +import static org.elasticsearch.xpack.cluster.settings.ClusterSettings.CLUSTER_LOGSDB_ENABLED; + +final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider { + private static final String LOGS_PATTERN = "logs-*-*"; + private volatile boolean isLogsdbEnabled; + + LogsdbIndexModeSettingsProvider(final Settings settings) { + this.isLogsdbEnabled = CLUSTER_LOGSDB_ENABLED.get(settings); + } + + void updateClusterIndexModeLogsdbEnabled(boolean isLogsdbEnabled) { + this.isLogsdbEnabled = isLogsdbEnabled; + } + + @Override + public Settings getAdditionalIndexSettings( + final String indexName, + final String dataStreamName, + boolean isTimeSeries, + final Metadata metadata, + final Instant resolvedAt, + final Settings settings, + final List combinedTemplateMappings + ) { + if (isLogsdbEnabled == false || dataStreamName == null) { + return Settings.EMPTY; + } + + final IndexMode indexMode = resolveIndexMode(settings.get(IndexSettings.MODE.getKey())); + if (indexMode != null) { + return Settings.EMPTY; + } + + if (usesLogsAtSettingsComponentTemplate(metadata, dataStreamName) && matchesLogsPattern(dataStreamName)) { + return Settings.builder().put("index.mode", IndexMode.LOGSDB.getName()).build(); + } + + return Settings.EMPTY; + } + + private static boolean matchesLogsPattern(final String name) { + return Regex.simpleMatch(LOGS_PATTERN, name); + } + + private IndexMode resolveIndexMode(final String mode) { + return mode != null ? Enum.valueOf(IndexMode.class, mode.toUpperCase(Locale.ROOT)) : null; + } + + private boolean usesLogsAtSettingsComponentTemplate(final Metadata metadata, final String name) { + final String template = MetadataIndexTemplateService.findV2Template(metadata, name, false); + if (template == null) { + return false; + } + final ComposableIndexTemplate composableIndexTemplate = metadata.templatesV2().get(template); + if (composableIndexTemplate == null) { + return false; + } + for (final String componentTemplate : composableIndexTemplate.composedOf()) { + if ("logs@settings".equals(componentTemplate)) { + return true; + } + } + return false; + } + +} diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java new file mode 100644 index 0000000000000..eeb5389644c02 --- /dev/null +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -0,0 +1,326 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Map; + +public class LogsdbIndexModeSettingsProviderTests extends ESTestCase { + + public static final String DEFAULT_MAPPING = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + }, + "message": { + "type": "keyword" + }, + "host.name": { + "type": "keyword" + } + } + } + } + """; + + public void testLogsDbDisabled() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", false).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testOnIndexCreation() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + "logs-apache-production", + null, + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testOnExplicitStandardIndex() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.getName()).build(), + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testOnExplicitTimeSeriesIndex() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()).build(), + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testNonLogsDataStream() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs", + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testWithoutLogsComponentTemplate() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of()), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testWithLogsComponentTemplate() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of("logs@settings")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); + } + + public void testWithMultipleComponentTemplates() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of("logs@settings", "logs@custom")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); + } + + public void testWithCustomComponentTemplatesOnly() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of("logs@custom", "custom-component-template")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testNonMatchingTemplateIndexPattern() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("standard-apache-production"), List.of("logs@settings")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testCaseSensitivity() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "LOGS-apache-production", + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testMultipleHyphensInDataStreamName() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production-eu", + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testBeforeAndAFterSettingUpdate() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", false).build() + ); + + final Settings beforeSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of("logs@settings")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(beforeSettings.isEmpty()); + + provider.updateClusterIndexModeLogsdbEnabled(true); + + final Settings afterSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of("logs@settings")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertIndexMode(afterSettings, IndexMode.LOGSDB.getName()); + + provider.updateClusterIndexModeLogsdbEnabled(false); + + final Settings laterSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of("logs@settings")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(laterSettings.isEmpty()); + } + + private static Metadata buildMetadata(final List indexPatterns, final List componentTemplates) throws IOException { + final Template template = new Template(Settings.EMPTY, new CompressedXContent(DEFAULT_MAPPING), null); + final ComposableIndexTemplate composableTemplate = ComposableIndexTemplate.builder() + .indexPatterns(indexPatterns) + .template(template) + .componentTemplates(componentTemplates) + .priority(1_000L) + .version(1L) + .build(); + return Metadata.builder() + .putCustom(ComposableIndexTemplateMetadata.TYPE, new ComposableIndexTemplateMetadata(Map.of("composable", composableTemplate))) + .build(); + } + + private void assertIndexMode(final Settings settings, final String expectedIndexMode) { + assertEquals(expectedIndexMode, settings.get(IndexSettings.MODE.getKey())); + } + +} diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java index 62d22c0c0a9cc..b2dc04c1178e4 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java @@ -51,12 +51,7 @@ public class LegacyStackTemplateRegistry extends IndexTemplateRegistry { private final FeatureService featureService; private volatile boolean stackTemplateEnabled; - private static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of( - "xpack.stack.template.deprecated", - "true", - "xpack.stack.template.logsdb.index.mode", - "standard" - ); + private static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of("xpack.stack.template.deprecated", "true"); // General mappings conventions for any data that ends up in a data stream public static final String DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "data-streams-mappings"; diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java index cc127883652af..71d01798323d3 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java @@ -23,7 +23,7 @@ public StackPlugin(Settings settings) { @Override public List> getSettings() { - return List.of(StackTemplateRegistry.STACK_TEMPLATES_ENABLED, StackTemplateRegistry.CLUSTER_LOGSDB_ENABLED); + return List.of(StackTemplateRegistry.STACK_TEMPLATES_ENABLED); } @Override diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 592842f61eee8..b45f17e434388 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -18,7 +18,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.features.FeatureService; import org.elasticsearch.features.NodeFeature; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -36,6 +35,8 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.xpack.cluster.settings.ClusterSettings.CLUSTER_LOGSDB_ENABLED; + public class StackTemplateRegistry extends IndexTemplateRegistry { private static final Logger logger = LogManager.getLogger(StackTemplateRegistry.class); @@ -58,15 +59,6 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { Setting.Property.Dynamic ); - /** - * if index.mode "logsdb" is applied by default in logs@settings for 'logs-*-*' - */ - public static final Setting CLUSTER_LOGSDB_ENABLED = Setting.boolSetting( - "cluster.logsdb.enabled", - false, - Setting.Property.NodeScope - ); - private final ClusterService clusterService; private final FeatureService featureService; private final Map componentTemplateConfigs; @@ -167,15 +159,10 @@ private Map loadComponentTemplateConfigs(boolean logs ), new IndexTemplateConfig( LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, - "/logs@settings.json", + logsDbEnabled ? "/logs@settings-logsdb.json" : "/logs@settings.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE, - Map.of( - "xpack.stack.template.deprecated", - "false", - "xpack.stack.template.logsdb.index.mode", - logsDbEnabled ? IndexMode.LOGSDB.getName() : IndexMode.STANDARD.getName() - ) + Map.of("xpack.stack.template.deprecated", "false") ), new IndexTemplateConfig( METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, From 1b67dabadb35b4c79d67ff30ccdfce1cd5929ce8 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Thu, 26 Sep 2024 08:51:02 -0400 Subject: [PATCH 21/36] Fix collapse interaction with stored fields (#112761) Collapse dynamically will add values to the DocumentField values array. There are a few scenarios where this is immutable and most of these are OK. However, we get in trouble when we create an immutable set for StoredValues which collapse later tries to update. The other option for this fix was to make an array copy for `values` in every `DocumentField` ctor, this seemed very expensive and could get out of hand. So, I decided to fix this one bug instead. closes https://github.com/elastic/elasticsearch/issues/112646 --- docs/changelog/112761.yaml | 6 ++++ .../search/CollapseSearchResultsIT.java | 30 +++++++++++++++++++ .../fetch/subphase/StoredFieldsPhase.java | 4 ++- 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/112761.yaml diff --git a/docs/changelog/112761.yaml b/docs/changelog/112761.yaml new file mode 100644 index 0000000000000..fe63f38f365a4 --- /dev/null +++ b/docs/changelog/112761.yaml @@ -0,0 +1,6 @@ +pr: 112761 +summary: Fix collapse interaction with stored fields +area: Search +type: bug +issues: + - 112646 diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/CollapseSearchResultsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/CollapseSearchResultsIT.java index 48dda7fd30068..89474a0181597 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/CollapseSearchResultsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/CollapseSearchResultsIT.java @@ -14,6 +14,7 @@ import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentType; import java.util.Map; import java.util.Set; @@ -85,4 +86,33 @@ public void testCollapseWithFields() { } ); } + + public void testCollapseWithStoredFields() { + final String indexName = "test_collapse"; + createIndex(indexName); + final String collapseField = "collapse_field"; + assertAcked(indicesAdmin().preparePutMapping(indexName).setSource(""" + { + "dynamic": "strict", + "properties": { + "collapse_field": { "type": "keyword", "store": true }, + "ts": { "type": "date", "store": true } + } + } + """, XContentType.JSON)); + index(indexName, "id_1_0", Map.of(collapseField, "value1", "ts", 0)); + index(indexName, "id_1_1", Map.of(collapseField, "value1", "ts", 1)); + index(indexName, "id_2_0", Map.of(collapseField, "value2", "ts", 2)); + refresh(indexName); + + assertNoFailuresAndResponse( + prepareSearch(indexName).setQuery(new MatchAllQueryBuilder()) + .setFetchSource(false) + .storedFields("*") + .setCollapse(new CollapseBuilder(collapseField)), + searchResponse -> { + assertEquals(collapseField, searchResponse.getHits().getCollapseField()); + } + ); + } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/StoredFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/StoredFieldsPhase.java index b3211f0b1e31c..17b57645d7d5f 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/StoredFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/StoredFieldsPhase.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** * Process stored fields loaded from a HitContext into DocumentFields @@ -42,7 +43,8 @@ List process(Map> loadedFields) { if (inputs == null) { return List.of(); } - return inputs.stream().map(ft::valueForDisplay).toList(); + // This is eventually provided to DocumentField, which needs this collection to be mutable + return inputs.stream().map(ft::valueForDisplay).collect(Collectors.toList()); } boolean hasValue(Map> loadedFields) { From 701ed61fd31aed60e9e1ae7ba5ab4d24f266f928 Mon Sep 17 00:00:00 2001 From: Dan Rubinstein Date: Thu, 26 Sep 2024 09:35:54 -0400 Subject: [PATCH 22/36] Adding inference endpoint creation validation for MistralService, GoogleAiStudioService, and HuggingFaceService (#113492) * Adding inference endpoint creation validation for MistralService, GoogleAiStudioService, and HuggingFaceService * Moving invalid model type exception to shared ServiceUtils function * Fixing naming inconsistency * Updating HuggingFaceIT ELSER tests for inference endpoint validation --- .../qa/mixed/HuggingFaceServiceMixedIT.java | 1 + .../HuggingFaceServiceUpgradeIT.java | 2 + .../inference/services/ServiceUtils.java | 7 +++ .../googleaistudio/GoogleAiStudioService.java | 40 ++++++++--------- .../huggingface/HuggingFaceService.java | 42 +++++++++--------- .../services/mistral/MistralService.java | 43 +++++++++--------- .../services/openai/OpenAiService.java | 7 +-- .../SimpleServiceIntegrationValidator.java | 20 +++++++-- .../GoogleAiStudioServiceTests.java | 39 ++++++++++++++++ .../huggingface/HuggingFaceServiceTests.java | 39 ++++++++++++++++ .../services/mistral/MistralServiceTests.java | 44 +++++++++++++++++++ .../services/openai/OpenAiServiceTests.java | 5 ++- ...impleServiceIntegrationValidatorTests.java | 1 - 13 files changed, 213 insertions(+), 77 deletions(-) diff --git a/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/HuggingFaceServiceMixedIT.java b/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/HuggingFaceServiceMixedIT.java index a2793f9060d8a..59d3faf6489a6 100644 --- a/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/HuggingFaceServiceMixedIT.java +++ b/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/HuggingFaceServiceMixedIT.java @@ -84,6 +84,7 @@ public void testElser() throws IOException { final String inferenceId = "mixed-cluster-elser"; final String upgradedClusterId = "upgraded-cluster-elser"; + elserServer.enqueue(new MockResponse().setResponseCode(200).setBody(elserResponse())); put(inferenceId, elserConfig(getUrl(elserServer)), TaskType.SPARSE_EMBEDDING); var configs = (List>) get(TaskType.SPARSE_EMBEDDING, inferenceId).get("endpoints"); diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java index 36ee472cc0a13..9c9a377bbb001 100644 --- a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java +++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java @@ -117,6 +117,7 @@ public void testElser() throws IOException { var testTaskType = TaskType.SPARSE_EMBEDDING; if (isOldCluster()) { + elserServer.enqueue(new MockResponse().setResponseCode(200).setBody(elserResponse())); put(oldClusterId, elserConfig(getUrl(elserServer)), testTaskType); var configs = (List>) get(testTaskType, oldClusterId).get(old_cluster_endpoint_identifier); assertThat(configs, hasSize(1)); @@ -136,6 +137,7 @@ public void testElser() throws IOException { assertElser(oldClusterId); // New endpoint + elserServer.enqueue(new MockResponse().setResponseCode(200).setBody(elserResponse())); put(upgradedClusterId, elserConfig(getUrl(elserServer)), testTaskType); configs = (List>) get(upgradedClusterId).get("endpoints"); assertThat(configs, hasSize(1)); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java index 6c4904f8918a7..6eb0331913009 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java @@ -202,6 +202,13 @@ public static ElasticsearchStatusException unknownSettingsError(Map invalidModelType) { + throw new ElasticsearchStatusException( + Strings.format("Can't update embedding details for model with unexpected type %s", invalidModelType), + RestStatus.BAD_REQUEST + ); + } + public static String missingSettingErrorMsg(String settingName, String scope) { return Strings.format("[%s] does not contain the required setting [%s]", scope, settingName); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java index 08eb67ca744a4..422fc5b0ed720 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java @@ -35,6 +35,7 @@ import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModel; import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsModel; import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.validation.ModelValidatorBuilder; import java.util.List; import java.util.Map; @@ -187,30 +188,29 @@ public TransportVersion getMinimalSupportedVersion() { @Override public void checkModelConfig(Model model, ActionListener listener) { - if (model instanceof GoogleAiStudioEmbeddingsModel embeddingsModel) { - ServiceUtils.getEmbeddingSize( - model, - this, - listener.delegateFailureAndWrap((l, size) -> l.onResponse(updateModelWithEmbeddingDetails(embeddingsModel, size))) - ); - } else { - listener.onResponse(model); - } + // TODO: Remove this function once all services have been updated to use the new model validators + ModelValidatorBuilder.buildModelValidator(model.getTaskType()).validate(this, model, listener); } - private GoogleAiStudioEmbeddingsModel updateModelWithEmbeddingDetails(GoogleAiStudioEmbeddingsModel model, int embeddingSize) { - var similarityFromModel = model.getServiceSettings().similarity(); - var similarityToUse = similarityFromModel == null ? SimilarityMeasure.DOT_PRODUCT : similarityFromModel; + @Override + public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { + if (model instanceof GoogleAiStudioEmbeddingsModel embeddingsModel) { + var serviceSettings = embeddingsModel.getServiceSettings(); + var similarityFromModel = serviceSettings.similarity(); + var similarityToUse = similarityFromModel == null ? SimilarityMeasure.DOT_PRODUCT : similarityFromModel; - GoogleAiStudioEmbeddingsServiceSettings serviceSettings = new GoogleAiStudioEmbeddingsServiceSettings( - model.getServiceSettings().modelId(), - model.getServiceSettings().maxInputTokens(), - embeddingSize, - similarityToUse, - model.getServiceSettings().rateLimitSettings() - ); + var updatedServiceSettings = new GoogleAiStudioEmbeddingsServiceSettings( + serviceSettings.modelId(), + serviceSettings.maxInputTokens(), + embeddingSize, + similarityToUse, + serviceSettings.rateLimitSettings() + ); - return new GoogleAiStudioEmbeddingsModel(model, serviceSettings); + return new GoogleAiStudioEmbeddingsModel(embeddingsModel, updatedServiceSettings); + } else { + throw ServiceUtils.invalidModelTypeForUpdateModelWithEmbeddingDetails(model.getClass()); + } } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java index bdfa87e77b708..6b142edca80aa 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java @@ -29,6 +29,7 @@ import org.elasticsearch.xpack.inference.services.ServiceUtils; import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserModel; import org.elasticsearch.xpack.inference.services.huggingface.embeddings.HuggingFaceEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.validation.ModelValidatorBuilder; import java.util.List; import java.util.Map; @@ -67,34 +68,31 @@ protected HuggingFaceModel createModel( @Override public void checkModelConfig(Model model, ActionListener listener) { + // TODO: Remove this function once all services have been updated to use the new model validators + ModelValidatorBuilder.buildModelValidator(model.getTaskType()).validate(this, model, listener); + } + + @Override + public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { if (model instanceof HuggingFaceEmbeddingsModel embeddingsModel) { - ServiceUtils.getEmbeddingSize( - model, - this, - listener.delegateFailureAndWrap((l, size) -> l.onResponse(updateModelWithEmbeddingDetails(embeddingsModel, size))) + var serviceSettings = embeddingsModel.getServiceSettings(); + var similarityFromModel = serviceSettings.similarity(); + var similarityToUse = similarityFromModel == null ? SimilarityMeasure.COSINE : similarityFromModel; + + var updatedServiceSettings = new HuggingFaceServiceSettings( + serviceSettings.uri(), + similarityToUse, + embeddingSize, + embeddingsModel.getTokenLimit(), + serviceSettings.rateLimitSettings() ); + + return new HuggingFaceEmbeddingsModel(embeddingsModel, updatedServiceSettings); } else { - listener.onResponse(model); + throw ServiceUtils.invalidModelTypeForUpdateModelWithEmbeddingDetails(model.getClass()); } } - private static HuggingFaceEmbeddingsModel updateModelWithEmbeddingDetails(HuggingFaceEmbeddingsModel model, int embeddingSize) { - // default to cosine similarity - var similarity = model.getServiceSettings().similarity() == null - ? SimilarityMeasure.COSINE - : model.getServiceSettings().similarity(); - - var serviceSettings = new HuggingFaceServiceSettings( - model.getServiceSettings().uri(), - similarity, - embeddingSize, - model.getTokenLimit(), - model.getServiceSettings().rateLimitSettings() - ); - - return new HuggingFaceEmbeddingsModel(model, serviceSettings); - } - @Override protected void doChunkedInfer( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java index 1acc13f50778b..221951f7a621e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.inference.services.ServiceUtils; import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsModel; import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.validation.ModelValidatorBuilder; import java.util.List; import java.util.Map; @@ -214,32 +215,28 @@ private MistralEmbeddingsModel createModelFromPersistent( @Override public void checkModelConfig(Model model, ActionListener listener) { - if (model instanceof MistralEmbeddingsModel embeddingsModel) { - ServiceUtils.getEmbeddingSize( - model, - this, - listener.delegateFailureAndWrap((l, size) -> l.onResponse(updateEmbeddingModelConfig(embeddingsModel, size))) - ); - } else { - listener.onResponse(model); - } + // TODO: Remove this function once all services have been updated to use the new model validators + ModelValidatorBuilder.buildModelValidator(model.getTaskType()).validate(this, model, listener); } - private MistralEmbeddingsModel updateEmbeddingModelConfig(MistralEmbeddingsModel embeddingsModel, int embeddingsSize) { - var embeddingServiceSettings = embeddingsModel.getServiceSettings(); - - var similarityFromModel = embeddingsModel.getServiceSettings().similarity(); - var similarityToUse = similarityFromModel == null ? SimilarityMeasure.DOT_PRODUCT : similarityFromModel; + @Override + public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { + if (model instanceof MistralEmbeddingsModel embeddingsModel) { + var serviceSettings = embeddingsModel.getServiceSettings(); - MistralEmbeddingsServiceSettings serviceSettings = new MistralEmbeddingsServiceSettings( - embeddingServiceSettings.modelId(), - embeddingsSize, - embeddingServiceSettings.maxInputTokens(), - similarityToUse, - embeddingServiceSettings.rateLimitSettings() - ); + var similarityFromModel = embeddingsModel.getServiceSettings().similarity(); + var similarityToUse = similarityFromModel == null ? SimilarityMeasure.DOT_PRODUCT : similarityFromModel; - return new MistralEmbeddingsModel(embeddingsModel, serviceSettings); + MistralEmbeddingsServiceSettings updatedServiceSettings = new MistralEmbeddingsServiceSettings( + serviceSettings.modelId(), + embeddingSize, + serviceSettings.maxInputTokens(), + similarityToUse, + serviceSettings.rateLimitSettings() + ); + return new MistralEmbeddingsModel(embeddingsModel, updatedServiceSettings); + } else { + throw ServiceUtils.invalidModelTypeForUpdateModelWithEmbeddingDetails(model.getClass()); + } } - } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java index 7cea1ec7df46c..f9565a915124f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java @@ -12,7 +12,6 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.inference.ChunkingOptions; @@ -35,6 +34,7 @@ import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.SenderService; import org.elasticsearch.xpack.inference.services.ServiceComponents; +import org.elasticsearch.xpack.inference.services.ServiceUtils; import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModel; import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsModel; import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsServiceSettings; @@ -307,10 +307,7 @@ public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { return new OpenAiEmbeddingsModel(embeddingsModel, updatedServiceSettings); } else { - throw new ElasticsearchStatusException( - Strings.format("Can't update embedding details for model with unexpected type %s", model.getClass()), - RestStatus.BAD_REQUEST - ); + throw ServiceUtils.invalidModelTypeForUpdateModelWithEmbeddingDetails(model.getClass()); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidator.java index 9fc5748746085..6233a7e0b6b29 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidator.java @@ -1,3 +1,4 @@ + /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License @@ -33,14 +34,25 @@ public void validate(InferenceService service, Model model, ActionListener { + ActionListener.wrap(r -> { if (r != null) { - delegate.onResponse(r); + listener.onResponse(r); } else { - delegate.onFailure( - new ElasticsearchStatusException("Could not make a validation call to the selected service", RestStatus.BAD_REQUEST) + listener.onFailure( + new ElasticsearchStatusException( + "Could not complete inference endpoint creation as validation call to service returned null response.", + RestStatus.BAD_REQUEST + ) ); } + }, e -> { + listener.onFailure( + new ElasticsearchStatusException( + "Could not complete inference endpoint creation as validation call to service threw an exception.", + RestStatus.BAD_REQUEST, + e + ) + ); }) ); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java index a8882bb244512..89d6a010bbc07 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java @@ -914,6 +914,45 @@ public void testCheckModelConfig_DoesNotUpdateSimilarity_WhenItIsSpecifiedAsCosi } } + public void testUpdateModelWithEmbeddingDetails_InvalidModelProvided() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new GoogleAiStudioService(senderFactory, createWithEmptySettings(threadPool))) { + var model = GoogleAiStudioCompletionModelTests.createModel(randomAlphaOfLength(10), randomAlphaOfLength(10)); + assertThrows( + ElasticsearchStatusException.class, + () -> { service.updateModelWithEmbeddingDetails(model, randomNonNegativeInt()); } + ); + } + } + + public void testUpdateModelWithEmbeddingDetails_NullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(null); + } + + public void testUpdateModelWithEmbeddingDetails_NonNullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(randomFrom(SimilarityMeasure.values())); + } + + private void testUpdateModelWithEmbeddingDetails_Successful(SimilarityMeasure similarityMeasure) throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new GoogleAiStudioService(senderFactory, createWithEmptySettings(threadPool))) { + var embeddingSize = randomNonNegativeInt(); + var model = GoogleAiStudioEmbeddingsModelTests.createModel( + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomNonNegativeInt(), + similarityMeasure + ); + + Model updatedModel = service.updateModelWithEmbeddingDetails(model, embeddingSize); + + SimilarityMeasure expectedSimilarityMeasure = similarityMeasure == null ? SimilarityMeasure.DOT_PRODUCT : similarityMeasure; + assertEquals(expectedSimilarityMeasure, updatedModel.getServiceSettings().similarity()); + assertEquals(embeddingSize, updatedModel.getServiceSettings().dimensions().intValue()); + } + } + public static Map buildExpectationCompletions(List completions) { return Map.of( ChatCompletionResults.COMPLETION, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java index f68aedd69f365..5ea9f82e5b60c 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java @@ -595,6 +595,45 @@ public void testCheckModelConfig_DefaultsSimilarityToCosine() throws IOException } } + public void testUpdateModelWithEmbeddingDetails_InvalidModelProvided() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new HuggingFaceService(senderFactory, createWithEmptySettings(threadPool))) { + var model = HuggingFaceElserModelTests.createModel(randomAlphaOfLength(10), randomAlphaOfLength(10)); + assertThrows( + ElasticsearchStatusException.class, + () -> { service.updateModelWithEmbeddingDetails(model, randomNonNegativeInt()); } + ); + } + } + + public void testUpdateModelWithEmbeddingDetails_NullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(null); + } + + public void testUpdateModelWithEmbeddingDetails_NonNullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(randomFrom(SimilarityMeasure.values())); + } + + private void testUpdateModelWithEmbeddingDetails_Successful(SimilarityMeasure similarityMeasure) throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new HuggingFaceService(senderFactory, createWithEmptySettings(threadPool))) { + var embeddingSize = randomNonNegativeInt(); + var model = HuggingFaceEmbeddingsModelTests.createModel( + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomNonNegativeInt(), + randomNonNegativeInt(), + similarityMeasure + ); + + Model updatedModel = service.updateModelWithEmbeddingDetails(model, embeddingSize); + + SimilarityMeasure expectedSimilarityMeasure = similarityMeasure == null ? SimilarityMeasure.COSINE : similarityMeasure; + assertEquals(expectedSimilarityMeasure, updatedModel.getServiceSettings().similarity()); + assertEquals(embeddingSize, updatedModel.getServiceSettings().dimensions().intValue()); + } + } + public void testChunkedInfer_CallsInfer_TextEmbedding_ConvertsFloatResponse() throws IOException { var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java index c833f00c4c433..9d0fd954c44f9 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; +import org.elasticsearch.xpack.inference.ModelConfigurationsTests; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; @@ -38,6 +39,7 @@ import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingModelTests; import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsModel; import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettingsTests; import org.hamcrest.CoreMatchers; import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; @@ -388,6 +390,48 @@ public void testCheckModelConfig_ForEmbeddingsModel_Works() throws IOException { } } + public void testUpdateModelWithEmbeddingDetails_InvalidModelProvided() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new MistralService(senderFactory, createWithEmptySettings(threadPool))) { + var model = new Model(ModelConfigurationsTests.createRandomInstance()); + + assertThrows( + ElasticsearchStatusException.class, + () -> { service.updateModelWithEmbeddingDetails(model, randomNonNegativeInt()); } + ); + } + } + + public void testUpdateModelWithEmbeddingDetails_NullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(null); + } + + public void testUpdateModelWithEmbeddingDetails_NonNullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(randomFrom(SimilarityMeasure.values())); + } + + private void testUpdateModelWithEmbeddingDetails_Successful(SimilarityMeasure similarityMeasure) throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new MistralService(senderFactory, createWithEmptySettings(threadPool))) { + var embeddingSize = randomNonNegativeInt(); + var model = MistralEmbeddingModelTests.createModel( + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomNonNegativeInt(), + randomNonNegativeInt(), + similarityMeasure, + RateLimitSettingsTests.createRandom() + ); + + Model updatedModel = service.updateModelWithEmbeddingDetails(model, embeddingSize); + + SimilarityMeasure expectedSimilarityMeasure = similarityMeasure == null ? SimilarityMeasure.DOT_PRODUCT : similarityMeasure; + assertEquals(expectedSimilarityMeasure, updatedModel.getServiceSettings().similarity()); + assertEquals(embeddingSize, updatedModel.getServiceSettings().dimensions().intValue()); + } + } + public void testInfer_ThrowsErrorWhenModelIsNotMistralEmbeddingsModel() throws IOException { var sender = mock(Sender.class); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java index e4a304f818328..a5e8c1d7eb26e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java @@ -1433,7 +1433,7 @@ private void testUpdateModelWithEmbeddingDetails_Successful(SimilarityMeasure si randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10), - null, + similarityMeasure, randomNonNegativeInt(), randomNonNegativeInt(), randomBoolean() @@ -1441,7 +1441,8 @@ private void testUpdateModelWithEmbeddingDetails_Successful(SimilarityMeasure si Model updatedModel = service.updateModelWithEmbeddingDetails(model, embeddingSize); - assertEquals(SimilarityMeasure.DOT_PRODUCT, updatedModel.getServiceSettings().similarity()); + SimilarityMeasure expectedSimilarityMeasure = similarityMeasure == null ? SimilarityMeasure.DOT_PRODUCT : similarityMeasure; + assertEquals(expectedSimilarityMeasure, updatedModel.getServiceSettings().similarity()); assertEquals(embeddingSize, updatedModel.getServiceSettings().dimensions().intValue()); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java index 23000ce431e7b..ef295e4070cc3 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java @@ -122,7 +122,6 @@ private void verifyCallToService(boolean withQuery) { eq(InferenceAction.Request.DEFAULT_TIMEOUT), any() ); - verify(mockActionListener).delegateFailureAndWrap(any()); verifyNoMoreInteractions(mockInferenceService, mockModel, mockActionListener, mockInferenceServiceResults); } } From 1faa35176020e7d500a7ca823705c0d87c9335c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Cea=20Fontenla?= Date: Thu, 26 Sep 2024 16:03:29 +0200 Subject: [PATCH 23/36] Add CircuitBreaker to TDigest, Step 2: Add CB to array wrappers (#113105) Part of https://github.com/elastic/elasticsearch/issues/99815 ## Steps 1. Migrate TDigest classes to use a custom Array implementation. Temporarily use a simple array wrapper (https://github.com/elastic/elasticsearch/pull/112810) 2. Implement CircuitBreaking in the `MemoryTrackingTDigestArrays` class. Add `Releasable` and ensure it's always closed within TDigest (This PR) 3. Pass the CircuitBreaker as a parameter to TDigestState from wherever it's being used 4. Account remaining TDigest classes size ("SHALLOW_SIZE") Every step should be safely mergeable to main: - The first and second steps should have no impact. - The third and fourth ones will start increasing the CB count partially. ## Remarks To simplify testing the CircuitBreaker, added a helper method + `@After` to ESTestCase. Right now CBs are usually tested through MockBigArrays. E.g: https://github.com/elastic/elasticsearch/blob/f7a0196b454b17f7928728a26084000238c4efaa/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java#L1263-L1265 So I guess there was no need for this yet. But I may have missed something somewhere. Also, I'm separating this PR from the "step 3" as integrating this (CB) in the current usages may require some refactor of external code, which may be somewhat more _dangerous_ --- .../benchmark/tdigest/SortBench.java | 9 +- .../benchmark/tdigest/TDigestBench.java | 11 +- libs/tdigest/build.gradle | 2 + libs/tdigest/src/main/java/module-info.java | 2 + .../elasticsearch/tdigest/AVLGroupTree.java | 8 +- .../elasticsearch/tdigest/AVLTreeDigest.java | 41 +- .../elasticsearch/tdigest/HybridDigest.java | 7 + .../org/elasticsearch/tdigest/IntAVLTree.java | 8 +- .../elasticsearch/tdigest/MergingDigest.java | 13 +- .../elasticsearch/tdigest/SortingDigest.java | 6 + .../org/elasticsearch/tdigest/TDigest.java | 3 +- .../tdigest/arrays/TDigestByteArray.java | 4 +- .../tdigest/arrays/TDigestDoubleArray.java | 4 +- .../tdigest/arrays/TDigestIntArray.java | 4 +- .../tdigest/arrays/TDigestLongArray.java | 4 +- .../tdigest/arrays/WrapperTDigestArrays.java | 258 ----------- .../tdigest/AVLGroupTreeTests.java | 15 +- .../tdigest/AVLTreeDigestTests.java | 4 +- .../tdigest/AlternativeMergeTests.java | 9 +- .../elasticsearch/tdigest/BigCountTests.java | 4 +- .../BigCountTestsMergingDigestTests.java | 4 +- .../tdigest/BigCountTestsTreeDigestTests.java | 4 +- .../tdigest/ComparisonTests.java | 13 +- .../tdigest/IntAVLTreeTests.java | 13 +- .../elasticsearch/tdigest/MedianTests.java | 13 +- .../org/elasticsearch/tdigest/SortTests.java | 14 +- .../tdigest/TDigestTestCase.java | 109 +++++ .../elasticsearch/tdigest/TDigestTests.java | 10 +- .../metrics/EmptyTDigestState.java | 4 +- .../metrics/MemoryTrackingTDigestArrays.java | 401 ++++++++++++++++++ .../aggregations/metrics/TDigestState.java | 18 +- .../MemoryTrackingTDigestArraysTests.java | 360 ++++++++++++++++ .../metrics/TDigestStateTests.java | 235 +++++----- .../org/elasticsearch/test/ESTestCase.java | 16 + 34 files changed, 1147 insertions(+), 483 deletions(-) delete mode 100644 libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/WrapperTDigestArrays.java create mode 100644 libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTestCase.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArrays.java create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArraysTests.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/SortBench.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/SortBench.java index 423db48337586..4bec6a183fe94 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/SortBench.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/SortBench.java @@ -21,10 +21,12 @@ package org.elasticsearch.benchmark.tdigest; +import org.elasticsearch.common.breaker.NoopCircuitBreaker; +import org.elasticsearch.search.aggregations.metrics.MemoryTrackingTDigestArrays; import org.elasticsearch.tdigest.Sort; +import org.elasticsearch.tdigest.arrays.TDigestArrays; import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; import org.elasticsearch.tdigest.arrays.TDigestIntArray; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -51,7 +53,8 @@ @State(Scope.Thread) public class SortBench { private final int size = 100000; - private final TDigestDoubleArray values = WrapperTDigestArrays.INSTANCE.newDoubleArray(size); + private final TDigestArrays arrays = new MemoryTrackingTDigestArrays(new NoopCircuitBreaker("default-wrapper-tdigest-arrays")); + private final TDigestDoubleArray values = arrays.newDoubleArray(size); @Param({ "0", "1", "-1" }) public int sortDirection; @@ -72,7 +75,7 @@ public void setup() { @Benchmark public void stableSort() { - TDigestIntArray order = WrapperTDigestArrays.INSTANCE.newIntArray(size); + TDigestIntArray order = arrays.newIntArray(size); for (int i = 0; i < size; i++) { order.set(i, i); } diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java index 58bb5b07d22cd..36ffc34c482d7 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java @@ -21,9 +21,11 @@ package org.elasticsearch.benchmark.tdigest; +import org.elasticsearch.common.breaker.NoopCircuitBreaker; +import org.elasticsearch.search.aggregations.metrics.MemoryTrackingTDigestArrays; import org.elasticsearch.tdigest.MergingDigest; import org.elasticsearch.tdigest.TDigest; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestArrays; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -56,24 +58,25 @@ @Threads(1) @State(Scope.Thread) public class TDigestBench { + private static final TDigestArrays arrays = new MemoryTrackingTDigestArrays(new NoopCircuitBreaker("default-wrapper-tdigest-arrays")); public enum TDigestFactory { MERGE { @Override TDigest create(double compression) { - return new MergingDigest(WrapperTDigestArrays.INSTANCE, compression, (int) (10 * compression)); + return new MergingDigest(arrays, compression, (int) (10 * compression)); } }, AVL_TREE { @Override TDigest create(double compression) { - return TDigest.createAvlTreeDigest(WrapperTDigestArrays.INSTANCE, compression); + return TDigest.createAvlTreeDigest(arrays, compression); } }, HYBRID { @Override TDigest create(double compression) { - return TDigest.createHybridDigest(WrapperTDigestArrays.INSTANCE, compression); + return TDigest.createHybridDigest(arrays, compression); } }; diff --git a/libs/tdigest/build.gradle b/libs/tdigest/build.gradle index 771df2e83d85d..df60862b27386 100644 --- a/libs/tdigest/build.gradle +++ b/libs/tdigest/build.gradle @@ -22,6 +22,8 @@ apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.publish' dependencies { + api project(':libs:elasticsearch-core') + testImplementation(project(":test:framework")) { exclude group: 'org.elasticsearch', module: 'elasticsearch-tdigest' } diff --git a/libs/tdigest/src/main/java/module-info.java b/libs/tdigest/src/main/java/module-info.java index 8edaff3f31d8c..cc7ff1810905f 100644 --- a/libs/tdigest/src/main/java/module-info.java +++ b/libs/tdigest/src/main/java/module-info.java @@ -18,6 +18,8 @@ */ module org.elasticsearch.tdigest { + requires org.elasticsearch.base; + exports org.elasticsearch.tdigest; exports org.elasticsearch.tdigest.arrays; } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLGroupTree.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLGroupTree.java index 8528db2128729..a1a65e1e71cde 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLGroupTree.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLGroupTree.java @@ -21,6 +21,8 @@ package org.elasticsearch.tdigest; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; import org.elasticsearch.tdigest.arrays.TDigestLongArray; @@ -31,7 +33,7 @@ /** * A tree of t-digest centroids. */ -final class AVLGroupTree extends AbstractCollection { +final class AVLGroupTree extends AbstractCollection implements Releasable { /* For insertions into the tree */ private double centroid; private long count; @@ -267,4 +269,8 @@ private void checkAggregates(int node) { } } + @Override + public void close() { + Releasables.close(centroids, counts, aggregatedCounts, tree); + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLTreeDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLTreeDigest.java index c28f86b9b8edc..f6b027edb1e9c 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLTreeDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLTreeDigest.java @@ -21,6 +21,7 @@ package org.elasticsearch.tdigest; +import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; import java.util.Collection; @@ -153,26 +154,27 @@ public void compress() { } needsCompression = false; - AVLGroupTree centroids = summary; - this.summary = new AVLGroupTree(arrays); + try (AVLGroupTree centroids = summary) { + this.summary = new AVLGroupTree(arrays); - final int[] nodes = new int[centroids.size()]; - nodes[0] = centroids.first(); - for (int i = 1; i < nodes.length; ++i) { - nodes[i] = centroids.next(nodes[i - 1]); - assert nodes[i] != IntAVLTree.NIL; - } - assert centroids.next(nodes[nodes.length - 1]) == IntAVLTree.NIL; + final int[] nodes = new int[centroids.size()]; + nodes[0] = centroids.first(); + for (int i = 1; i < nodes.length; ++i) { + nodes[i] = centroids.next(nodes[i - 1]); + assert nodes[i] != IntAVLTree.NIL; + } + assert centroids.next(nodes[nodes.length - 1]) == IntAVLTree.NIL; - for (int i = centroids.size() - 1; i > 0; --i) { - final int other = gen.nextInt(i + 1); - final int tmp = nodes[other]; - nodes[other] = nodes[i]; - nodes[i] = tmp; - } + for (int i = centroids.size() - 1; i > 0; --i) { + final int other = gen.nextInt(i + 1); + final int tmp = nodes[other]; + nodes[other] = nodes[i]; + nodes[i] = tmp; + } - for (int node : nodes) { - add(centroids.mean(node), centroids.count(node)); + for (int node : nodes) { + add(centroids.mean(node), centroids.count(node)); + } } } @@ -356,4 +358,9 @@ public int byteSize() { compress(); return 64 + summary.size() * 13; } + + @Override + public void close() { + Releasables.close(summary); + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/HybridDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/HybridDigest.java index c28a99fbd6d44..8d03ce4e303a6 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/HybridDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/HybridDigest.java @@ -19,6 +19,7 @@ package org.elasticsearch.tdigest; +import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; import java.util.Collection; @@ -110,6 +111,7 @@ public void reserve(long size) { } mergingDigest.reserve(size); // Release the allocated SortingDigest. + sortingDigest.close(); sortingDigest = null; } else { sortingDigest.reserve(size); @@ -196,4 +198,9 @@ public int byteSize() { } return sortingDigest.byteSize(); } + + @Override + public void close() { + Releasables.close(sortingDigest, mergingDigest); + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/IntAVLTree.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/IntAVLTree.java index cda8aecdb2ccc..b4a82257693d8 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/IntAVLTree.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/IntAVLTree.java @@ -21,6 +21,8 @@ package org.elasticsearch.tdigest; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; import org.elasticsearch.tdigest.arrays.TDigestByteArray; import org.elasticsearch.tdigest.arrays.TDigestIntArray; @@ -33,7 +35,7 @@ * want to add data to the nodes, typically by using arrays and node * identifiers as indices. */ -abstract class IntAVLTree { +abstract class IntAVLTree implements Releasable { /** * We use 0 instead of -1 so that left(NIL) works without * condition. @@ -586,4 +588,8 @@ int size() { } + @Override + public void close() { + Releasables.close(parent, left, right, depth); + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java index 1649af041ee19..f2ccfc33aa2a9 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java @@ -21,6 +21,7 @@ package org.elasticsearch.tdigest; +import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; import org.elasticsearch.tdigest.arrays.TDigestIntArray; @@ -66,8 +67,6 @@ * what the AVLTreeDigest uses and no dynamic allocation is required at all. */ public class MergingDigest extends AbstractTDigest { - private final TDigestArrays arrays; - private int mergeCount = 0; private final double publicCompression; @@ -138,8 +137,6 @@ public MergingDigest(TDigestArrays arrays, double compression, int bufferSize) { * @param size Size of main buffer */ public MergingDigest(TDigestArrays arrays, double compression, int bufferSize, int size) { - this.arrays = arrays; - // ensure compression >= 10 // default size = 2 * ceil(compression) // default bufferSize = 5 * size @@ -274,9 +271,6 @@ private void merge( incomingWeight.set(incomingCount, weight, 0, lastUsedCell); incomingCount += lastUsedCell; - if (incomingOrder == null) { - incomingOrder = arrays.newIntArray(incomingCount); - } Sort.stableSort(incomingOrder, incomingMean, incomingCount); totalWeight += unmergedWeight; @@ -581,4 +575,9 @@ public String toString() { + "-" + (useTwoLevelCompression ? "twoLevel" : "oneLevel"); } + + @Override + public void close() { + Releasables.close(weight, mean, tempWeight, tempMean, order); + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/SortingDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/SortingDigest.java index 94b5c667e0672..f063ca9a511c6 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/SortingDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/SortingDigest.java @@ -19,6 +19,7 @@ package org.elasticsearch.tdigest; +import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; @@ -137,4 +138,9 @@ public void reserve(long size) { public int byteSize() { return values.size() * 8; } + + @Override + public void close() { + Releasables.close(values); + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/TDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/TDigest.java index 4e79f9e68cd02..e578a688738cb 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/TDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/TDigest.java @@ -21,6 +21,7 @@ package org.elasticsearch.tdigest; +import org.elasticsearch.core.Releasable; import org.elasticsearch.tdigest.arrays.TDigestArrays; import java.util.Collection; @@ -37,7 +38,7 @@ * - test coverage roughly at 90% * - easy to adapt for use with map-reduce */ -public abstract class TDigest { +public abstract class TDigest implements Releasable { protected ScaleFunction scale = ScaleFunction.K_2; double min = Double.POSITIVE_INFINITY; double max = Double.NEGATIVE_INFINITY; diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestByteArray.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestByteArray.java index 481dde9784008..ae8e84800b433 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestByteArray.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestByteArray.java @@ -21,10 +21,12 @@ package org.elasticsearch.tdigest.arrays; +import org.elasticsearch.core.Releasable; + /** * Minimal interface for ByteArray-like classes used within TDigest. */ -public interface TDigestByteArray { +public interface TDigestByteArray extends Releasable { int size(); byte get(int index); diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestDoubleArray.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestDoubleArray.java index 92530db5e7dc4..1699dbd9beaf1 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestDoubleArray.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestDoubleArray.java @@ -21,10 +21,12 @@ package org.elasticsearch.tdigest.arrays; +import org.elasticsearch.core.Releasable; + /** * Minimal interface for DoubleArray-like classes used within TDigest. */ -public interface TDigestDoubleArray { +public interface TDigestDoubleArray extends Releasable { int size(); double get(int index); diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestIntArray.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestIntArray.java index c944a4f8faf07..44e366aacd173 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestIntArray.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestIntArray.java @@ -21,10 +21,12 @@ package org.elasticsearch.tdigest.arrays; +import org.elasticsearch.core.Releasable; + /** * Minimal interface for IntArray-like classes used within TDigest. */ -public interface TDigestIntArray { +public interface TDigestIntArray extends Releasable { int size(); int get(int index); diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestLongArray.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestLongArray.java index 7e75dd512e86d..5deea6b28b1ed 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestLongArray.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestLongArray.java @@ -21,10 +21,12 @@ package org.elasticsearch.tdigest.arrays; +import org.elasticsearch.core.Releasable; + /** * Minimal interface for LongArray-like classes used within TDigest. */ -public interface TDigestLongArray { +public interface TDigestLongArray extends Releasable { int size(); long get(int index); diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/WrapperTDigestArrays.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/WrapperTDigestArrays.java deleted file mode 100644 index ce2dd4f8d8e1d..0000000000000 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/WrapperTDigestArrays.java +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - * This project is based on a modification of https://github.com/tdunning/t-digest which is licensed under the Apache 2.0 License. - */ - -package org.elasticsearch.tdigest.arrays; - -import java.util.Arrays; - -/** - * Temporal TDigestArrays with raw arrays. - * - *

- * Delete after the right implementation for BigArrays is made. - *

- */ -public class WrapperTDigestArrays implements TDigestArrays { - - public static final WrapperTDigestArrays INSTANCE = new WrapperTDigestArrays(); - - private WrapperTDigestArrays() {} - - @Override - public WrapperTDigestDoubleArray newDoubleArray(int initialCapacity) { - return new WrapperTDigestDoubleArray(initialCapacity); - } - - @Override - public WrapperTDigestIntArray newIntArray(int initialSize) { - return new WrapperTDigestIntArray(initialSize); - } - - @Override - public TDigestLongArray newLongArray(int initialSize) { - return new WrapperTDigestLongArray(initialSize); - } - - @Override - public TDigestByteArray newByteArray(int initialSize) { - return new WrapperTDigestByteArray(initialSize); - } - - public WrapperTDigestDoubleArray newDoubleArray(double[] array) { - return new WrapperTDigestDoubleArray(array); - } - - public WrapperTDigestIntArray newIntArray(int[] array) { - return new WrapperTDigestIntArray(array); - } - - public static class WrapperTDigestDoubleArray implements TDigestDoubleArray { - private double[] array; - private int size; - - public WrapperTDigestDoubleArray(int initialSize) { - this(new double[initialSize]); - } - - public WrapperTDigestDoubleArray(double[] array) { - this.array = array; - this.size = array.length; - } - - @Override - public int size() { - return size; - } - - @Override - public double get(int index) { - assert index >= 0 && index < size; - return array[index]; - } - - @Override - public void set(int index, double value) { - assert index >= 0 && index < size; - array[index] = value; - } - - @Override - public void add(double value) { - ensureCapacity(size + 1); - array[size++] = value; - } - - @Override - public void sort() { - Arrays.sort(array, 0, size); - } - - @Override - public void ensureCapacity(int requiredCapacity) { - if (requiredCapacity > array.length) { - int newSize = array.length + (array.length >> 1); - if (newSize < requiredCapacity) { - newSize = requiredCapacity; - } - double[] newArray = new double[newSize]; - System.arraycopy(array, 0, newArray, 0, size); - array = newArray; - } - } - - @Override - public void resize(int newSize) { - if (newSize > array.length) { - array = Arrays.copyOf(array, newSize); - } - if (newSize > size) { - Arrays.fill(array, size, newSize, 0); - } - size = newSize; - } - } - - public static class WrapperTDigestIntArray implements TDigestIntArray { - private int[] array; - private int size; - - public WrapperTDigestIntArray(int initialSize) { - this(new int[initialSize]); - } - - public WrapperTDigestIntArray(int[] array) { - this.array = array; - this.size = array.length; - } - - @Override - public int size() { - return size; - } - - @Override - public int get(int index) { - assert index >= 0 && index < size; - return array[index]; - } - - @Override - public void set(int index, int value) { - assert index >= 0 && index < size; - array[index] = value; - } - - @Override - public void resize(int newSize) { - if (newSize > array.length) { - array = Arrays.copyOf(array, newSize); - } - if (newSize > size) { - Arrays.fill(array, size, newSize, 0); - } - size = newSize; - } - } - - public static class WrapperTDigestLongArray implements TDigestLongArray { - private long[] array; - private int size; - - public WrapperTDigestLongArray(int initialSize) { - this(new long[initialSize]); - } - - public WrapperTDigestLongArray(long[] array) { - this.array = array; - this.size = array.length; - } - - @Override - public int size() { - return size; - } - - @Override - public long get(int index) { - assert index >= 0 && index < size; - return array[index]; - } - - @Override - public void set(int index, long value) { - assert index >= 0 && index < size; - array[index] = value; - } - - @Override - public void resize(int newSize) { - if (newSize > array.length) { - array = Arrays.copyOf(array, newSize); - } - if (newSize > size) { - Arrays.fill(array, size, newSize, 0); - } - size = newSize; - } - } - - public static class WrapperTDigestByteArray implements TDigestByteArray { - private byte[] array; - private int size; - - public WrapperTDigestByteArray(int initialSize) { - this(new byte[initialSize]); - } - - public WrapperTDigestByteArray(byte[] array) { - this.array = array; - this.size = array.length; - } - - @Override - public int size() { - return size; - } - - @Override - public byte get(int index) { - assert index >= 0 && index < size; - return array[index]; - } - - @Override - public void set(int index, byte value) { - assert index >= 0 && index < size; - array[index] = value; - } - - @Override - public void resize(int newSize) { - if (newSize > array.length) { - array = Arrays.copyOf(array, newSize); - } - if (newSize > size) { - Arrays.fill(array, size, newSize, (byte) 0); - } - size = newSize; - } - } -} diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLGroupTreeTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLGroupTreeTests.java index 71be849f401f4..7ac55afd87808 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLGroupTreeTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLGroupTreeTests.java @@ -21,13 +21,10 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; -import org.elasticsearch.test.ESTestCase; - -public class AVLGroupTreeTests extends ESTestCase { +public class AVLGroupTreeTests extends TDigestTestCase { public void testSimpleAdds() { - AVLGroupTree x = new AVLGroupTree(WrapperTDigestArrays.INSTANCE); + AVLGroupTree x = new AVLGroupTree(arrays()); assertEquals(IntAVLTree.NIL, x.floor(34)); assertEquals(IntAVLTree.NIL, x.first()); assertEquals(IntAVLTree.NIL, x.last()); @@ -46,7 +43,7 @@ public void testSimpleAdds() { } public void testBalancing() { - AVLGroupTree x = new AVLGroupTree(WrapperTDigestArrays.INSTANCE); + AVLGroupTree x = new AVLGroupTree(arrays()); for (int i = 0; i < 101; i++) { x.add(new Centroid(i)); } @@ -60,7 +57,7 @@ public void testBalancing() { public void testFloor() { // mostly tested in other tests - AVLGroupTree x = new AVLGroupTree(WrapperTDigestArrays.INSTANCE); + AVLGroupTree x = new AVLGroupTree(arrays()); for (int i = 0; i < 101; i++) { x.add(new Centroid(i / 2)); } @@ -73,7 +70,7 @@ public void testFloor() { } public void testHeadSum() { - AVLGroupTree x = new AVLGroupTree(WrapperTDigestArrays.INSTANCE); + AVLGroupTree x = new AVLGroupTree(arrays()); for (int i = 0; i < 1000; ++i) { x.add(randomDouble(), randomIntBetween(1, 10)); } @@ -88,7 +85,7 @@ public void testHeadSum() { } public void testFloorSum() { - AVLGroupTree x = new AVLGroupTree(WrapperTDigestArrays.INSTANCE); + AVLGroupTree x = new AVLGroupTree(arrays()); int total = 0; for (int i = 0; i < 1000; ++i) { int count = randomIntBetween(1, 10); diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLTreeDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLTreeDigestTests.java index 3cd89de4746f1..f6dde4e168291 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLTreeDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLTreeDigestTests.java @@ -21,13 +21,11 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; - public class AVLTreeDigestTests extends TDigestTests { protected DigestFactory factory(final double compression) { return () -> { - AVLTreeDigest digest = new AVLTreeDigest(WrapperTDigestArrays.INSTANCE, compression); + AVLTreeDigest digest = new AVLTreeDigest(arrays(), compression); digest.setRandomSeed(randomLong()); return digest; }; diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AlternativeMergeTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AlternativeMergeTests.java index 4b95e9c0ee695..0d095ec37fa45 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AlternativeMergeTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AlternativeMergeTests.java @@ -21,15 +21,12 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; -import org.elasticsearch.test.ESTestCase; - import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Random; -public class AlternativeMergeTests extends ESTestCase { +public class AlternativeMergeTests extends TDigestTestCase { /** * Computes size using the alternative scaling limit for both an idealized merge and for * a MergingDigest. @@ -37,8 +34,8 @@ public class AlternativeMergeTests extends ESTestCase { public void testMerges() { for (int n : new int[] { 100, 1000, 10000, 100000 }) { for (double compression : new double[] { 50, 100, 200, 400 }) { - MergingDigest mergingDigest = new MergingDigest(WrapperTDigestArrays.INSTANCE, compression); - AVLTreeDigest treeDigest = new AVLTreeDigest(WrapperTDigestArrays.INSTANCE, compression); + MergingDigest mergingDigest = new MergingDigest(arrays(), compression); + AVLTreeDigest treeDigest = new AVLTreeDigest(arrays(), compression); List data = new ArrayList<>(); Random gen = random(); for (int i = 0; i < n; i++) { diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTests.java index 68b07f1096eea..7520d76172ef9 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTests.java @@ -21,9 +21,7 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.test.ESTestCase; - -public abstract class BigCountTests extends ESTestCase { +public abstract class BigCountTests extends TDigestTestCase { public void testBigMerge() { TDigest digest = createDigest(); diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsMergingDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsMergingDigestTests.java index 25cd1af05a0ba..ab28628200cce 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsMergingDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsMergingDigestTests.java @@ -21,11 +21,9 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; - public class BigCountTestsMergingDigestTests extends BigCountTests { @Override public TDigest createDigest() { - return new MergingDigest(WrapperTDigestArrays.INSTANCE, 100); + return new MergingDigest(arrays(), 100); } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsTreeDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsTreeDigestTests.java index a2cdf49d8f8ad..a9af82164c2ba 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsTreeDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsTreeDigestTests.java @@ -21,11 +21,9 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; - public class BigCountTestsTreeDigestTests extends BigCountTests { @Override public TDigest createDigest() { - return new AVLTreeDigest(WrapperTDigestArrays.INSTANCE, 100); + return new AVLTreeDigest(arrays(), 100); } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/ComparisonTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/ComparisonTests.java index f5df0c2f86ea1..82620459891ec 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/ComparisonTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/ComparisonTests.java @@ -21,13 +21,10 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; -import org.elasticsearch.test.ESTestCase; - import java.util.Arrays; import java.util.function.Supplier; -public class ComparisonTests extends ESTestCase { +public class ComparisonTests extends TDigestTestCase { private static final int SAMPLE_COUNT = 1_000_000; @@ -40,10 +37,10 @@ public class ComparisonTests extends ESTestCase { private void loadData(Supplier sampleGenerator) { final int COMPRESSION = 100; - avlTreeDigest = TDigest.createAvlTreeDigest(WrapperTDigestArrays.INSTANCE, COMPRESSION); - mergingDigest = TDigest.createMergingDigest(WrapperTDigestArrays.INSTANCE, COMPRESSION); - sortingDigest = TDigest.createSortingDigest(WrapperTDigestArrays.INSTANCE); - hybridDigest = TDigest.createHybridDigest(WrapperTDigestArrays.INSTANCE, COMPRESSION); + avlTreeDigest = TDigest.createAvlTreeDigest(arrays(), COMPRESSION); + mergingDigest = TDigest.createMergingDigest(arrays(), COMPRESSION); + sortingDigest = TDigest.createSortingDigest(arrays()); + hybridDigest = TDigest.createHybridDigest(arrays(), COMPRESSION); samples = new double[SAMPLE_COUNT]; for (int i = 0; i < SAMPLE_COUNT; i++) { diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/IntAVLTreeTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/IntAVLTreeTests.java index 58c91ae6e03e6..5178701e96c2c 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/IntAVLTreeTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/IntAVLTreeTests.java @@ -21,8 +21,7 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.tdigest.arrays.TDigestArrays; import java.util.Arrays; import java.util.Iterator; @@ -30,7 +29,7 @@ import java.util.Random; import java.util.TreeMap; -public class IntAVLTreeTests extends ESTestCase { +public class IntAVLTreeTests extends TDigestTestCase { static class IntegerBag extends IntAVLTree { @@ -38,8 +37,8 @@ static class IntegerBag extends IntAVLTree { int[] values; int[] counts; - IntegerBag() { - super(WrapperTDigestArrays.INSTANCE); + IntegerBag(TDigestArrays arrays) { + super(arrays); values = new int[capacity()]; counts = new int[capacity()]; } @@ -89,7 +88,7 @@ protected void merge(int node) { public void testDualAdd() { Random r = random(); TreeMap map = new TreeMap<>(); - IntegerBag bag = new IntegerBag(); + IntegerBag bag = new IntegerBag(arrays()); for (int i = 0; i < 100000; ++i) { final int v = r.nextInt(100000); if (map.containsKey(v)) { @@ -112,7 +111,7 @@ public void testDualAdd() { public void testDualAddRemove() { Random r = random(); TreeMap map = new TreeMap<>(); - IntegerBag bag = new IntegerBag(); + IntegerBag bag = new IntegerBag(arrays()); for (int i = 0; i < 100000; ++i) { final int v = r.nextInt(1000); if (r.nextBoolean()) { diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MedianTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MedianTests.java index dd455b307344e..c8acec935c040 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MedianTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MedianTests.java @@ -21,14 +21,11 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; -import org.elasticsearch.test.ESTestCase; - -public class MedianTests extends ESTestCase { +public class MedianTests extends TDigestTestCase { public void testAVL() { double[] data = new double[] { 7, 15, 36, 39, 40, 41 }; - TDigest digest = new AVLTreeDigest(WrapperTDigestArrays.INSTANCE, 100); + TDigest digest = new AVLTreeDigest(arrays(), 100); for (double value : data) { digest.add(value); } @@ -39,7 +36,7 @@ public void testAVL() { public void testMergingDigest() { double[] data = new double[] { 7, 15, 36, 39, 40, 41 }; - TDigest digest = new MergingDigest(WrapperTDigestArrays.INSTANCE, 100); + TDigest digest = new MergingDigest(arrays(), 100); for (double value : data) { digest.add(value); } @@ -50,7 +47,7 @@ public void testMergingDigest() { public void testSortingDigest() { double[] data = new double[] { 7, 15, 36, 39, 40, 41 }; - TDigest digest = new SortingDigest(WrapperTDigestArrays.INSTANCE); + TDigest digest = new SortingDigest(arrays()); for (double value : data) { digest.add(value); } @@ -61,7 +58,7 @@ public void testSortingDigest() { public void testHybridDigest() { double[] data = new double[] { 7, 15, 36, 39, 40, 41 }; - TDigest digest = new HybridDigest(WrapperTDigestArrays.INSTANCE, 100); + TDigest digest = new HybridDigest(arrays(), 100); for (double value : data) { digest.add(value); } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortTests.java index 7327dfb5aac3c..425e4d1497eda 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortTests.java @@ -22,22 +22,20 @@ package org.elasticsearch.tdigest; import org.elasticsearch.tdigest.arrays.TDigestIntArray; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; -import org.elasticsearch.test.ESTestCase; import java.util.HashMap; import java.util.Map; import java.util.Random; -public class SortTests extends ESTestCase { +public class SortTests extends TDigestTestCase { public void testReverse() { - TDigestIntArray x = WrapperTDigestArrays.INSTANCE.newIntArray(0); + TDigestIntArray x = arrays().newIntArray(0); // don't crash with no input Sort.reverse(x, 0, x.size()); // reverse stuff! - x = WrapperTDigestArrays.INSTANCE.newIntArray(new int[] { 1, 2, 3, 4, 5 }); + x = arrays().newIntArray(new int[] { 1, 2, 3, 4, 5 }); Sort.reverse(x, 0, x.size()); for (int i = 0; i < 5; i++) { assertEquals(5 - i, x.get(i)); @@ -59,7 +57,7 @@ public void testReverse() { assertEquals(4, x.get(3)); assertEquals(1, x.get(4)); - x = WrapperTDigestArrays.INSTANCE.newIntArray(new int[] { 1, 2, 3, 4, 5, 6 }); + x = arrays().newIntArray(new int[] { 1, 2, 3, 4, 5, 6 }); Sort.reverse(x, 0, x.size()); for (int i = 0; i < 6; i++) { assertEquals(6 - i, x.get(i)); @@ -229,8 +227,8 @@ private void checkOrder(int[] order, double[] values) { } private void sort(int[] order, double[] values, int n) { - var wrappedOrder = WrapperTDigestArrays.INSTANCE.newIntArray(order); - var wrappedValues = WrapperTDigestArrays.INSTANCE.newDoubleArray(values); + var wrappedOrder = arrays().newIntArray(order); + var wrappedValues = arrays().newDoubleArray(values); Sort.stableSort(wrappedOrder, wrappedValues, n); } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTestCase.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTestCase.java new file mode 100644 index 0000000000000..76db01d5dd0bf --- /dev/null +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTestCase.java @@ -0,0 +1,109 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * This project is based on a modification of https://github.com/tdunning/t-digest which is licensed under the Apache 2.0 License. + */ + +package org.elasticsearch.tdigest; + +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.search.aggregations.metrics.MemoryTrackingTDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestByteArray; +import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; +import org.elasticsearch.tdigest.arrays.TDigestIntArray; +import org.elasticsearch.tdigest.arrays.TDigestLongArray; +import org.elasticsearch.test.ESTestCase; +import org.junit.After; + +import java.util.Collection; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Base class for TDigest tests that require {@link TDigestArrays} instances. + *

+ * This class provides arrays that will be automatically closed after the test. + * It will also test that all memory have been freed, as the arrays use a counting CircuitBreaker. + *

+ */ +public abstract class TDigestTestCase extends ESTestCase { + private final Collection trackedArrays = ConcurrentHashMap.newKeySet(); + + /** + * Create a new TDigestArrays instance with a limited breaker. This method may be called multiple times. + * + *

+ * The arrays created by this method will be automatically released after the test. + *

+ */ + protected DelegatingTDigestArrays arrays() { + return new DelegatingTDigestArrays(); + } + + /** + * Release all arrays before {@link ESTestCase} checks for unreleased bytes. + */ + @After + public void releaseArrays() { + Releasables.close(trackedArrays); + trackedArrays.clear(); + } + + private T register(T releasable) { + trackedArrays.add(releasable); + return releasable; + } + + protected final class DelegatingTDigestArrays implements TDigestArrays { + private final MemoryTrackingTDigestArrays delegate; + + DelegatingTDigestArrays() { + this.delegate = new MemoryTrackingTDigestArrays(newLimitedBreaker(ByteSizeValue.ofMb(100))); + } + + public TDigestDoubleArray newDoubleArray(double[] data) { + return register(delegate.newDoubleArray(data)); + } + + @Override + public TDigestDoubleArray newDoubleArray(int size) { + return register(delegate.newDoubleArray(size)); + } + + public TDigestIntArray newIntArray(int[] data) { + return register(delegate.newIntArray(data)); + } + + @Override + public TDigestIntArray newIntArray(int size) { + return register(delegate.newIntArray(size)); + } + + @Override + public TDigestLongArray newLongArray(int size) { + return register(delegate.newLongArray(size)); + } + + @Override + public TDigestByteArray newByteArray(int initialSize) { + return register(delegate.newByteArray(initialSize)); + } + } +} diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java index 43f1e36afb314..89a0c037dc864 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java @@ -21,10 +21,6 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.tdigest.arrays.TDigestArrays; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; -import org.elasticsearch.test.ESTestCase; - import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -34,7 +30,7 @@ /** * Base test case for TDigests, just extend this class and implement the abstract methods. */ -public abstract class TDigestTests extends ESTestCase { +public abstract class TDigestTests extends TDigestTestCase { public interface DigestFactory { TDigest create(); @@ -544,8 +540,4 @@ public void testMonotonicity() { lastQuantile = q; } } - - protected static TDigestArrays arrays() { - return WrapperTDigestArrays.INSTANCE; - } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/EmptyTDigestState.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/EmptyTDigestState.java index 6ae9c655df3e8..56ac38a70cf07 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/EmptyTDigestState.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/EmptyTDigestState.java @@ -9,12 +9,10 @@ package org.elasticsearch.search.aggregations.metrics; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; - public final class EmptyTDigestState extends TDigestState { public EmptyTDigestState() { // Use the sorting implementation to minimize memory allocation. - super(WrapperTDigestArrays.INSTANCE, Type.SORTING, 1.0D); + super(MemoryTrackingTDigestArrays.INSTANCE, Type.SORTING, 1.0D); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArrays.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArrays.java new file mode 100644 index 0000000000000..e99bfbfe534c8 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArrays.java @@ -0,0 +1,401 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.search.aggregations.metrics; + +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.breaker.NoopCircuitBreaker; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.tdigest.arrays.TDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestByteArray; +import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; +import org.elasticsearch.tdigest.arrays.TDigestIntArray; +import org.elasticsearch.tdigest.arrays.TDigestLongArray; + +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * TDigestArrays with raw arrays and circuit breaking. + */ +public class MemoryTrackingTDigestArrays implements TDigestArrays { + + /** + * Default no-op CB instance of the wrapper. + * + * @deprecated This instance shouldn't be used, and will be removed after all usages are replaced. + */ + @Deprecated + public static final MemoryTrackingTDigestArrays INSTANCE = new MemoryTrackingTDigestArrays( + new NoopCircuitBreaker("default-wrapper-tdigest-arrays") + ); + + private final CircuitBreaker breaker; + + public MemoryTrackingTDigestArrays(CircuitBreaker breaker) { + this.breaker = breaker; + } + + @Override + public MemoryTrackingTDigestDoubleArray newDoubleArray(int initialSize) { + breaker.addEstimateBytesAndMaybeBreak( + MemoryTrackingTDigestDoubleArray.estimatedRamBytesUsed(initialSize), + "tdigest-new-double-array" + ); + return new MemoryTrackingTDigestDoubleArray(breaker, initialSize); + } + + @Override + public MemoryTrackingTDigestIntArray newIntArray(int initialSize) { + breaker.addEstimateBytesAndMaybeBreak(MemoryTrackingTDigestIntArray.estimatedRamBytesUsed(initialSize), "tdigest-new-int-array"); + return new MemoryTrackingTDigestIntArray(breaker, initialSize); + } + + @Override + public TDigestLongArray newLongArray(int initialSize) { + breaker.addEstimateBytesAndMaybeBreak(MemoryTrackingTDigestLongArray.estimatedRamBytesUsed(initialSize), "tdigest-new-long-array"); + return new MemoryTrackingTDigestLongArray(breaker, initialSize); + } + + @Override + public TDigestByteArray newByteArray(int initialSize) { + breaker.addEstimateBytesAndMaybeBreak(MemoryTrackingTDigestByteArray.estimatedRamBytesUsed(initialSize), "tdigest-new-byte-array"); + return new MemoryTrackingTDigestByteArray(breaker, initialSize); + } + + public MemoryTrackingTDigestDoubleArray newDoubleArray(double[] array) { + breaker.addEstimateBytesAndMaybeBreak( + MemoryTrackingTDigestDoubleArray.estimatedRamBytesUsed(array.length), + "tdigest-new-double-array" + ); + return new MemoryTrackingTDigestDoubleArray(breaker, array); + } + + public MemoryTrackingTDigestIntArray newIntArray(int[] array) { + breaker.addEstimateBytesAndMaybeBreak(MemoryTrackingTDigestIntArray.estimatedRamBytesUsed(array.length), "tdigest-new-int-array"); + return new MemoryTrackingTDigestIntArray(breaker, array); + } + + private static long estimatedArraySize(long arrayLength, long bytesPerElement) { + return RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + arrayLength * bytesPerElement); + } + + private abstract static class AbstractMemoryTrackingArray implements Releasable, Accountable { + protected final CircuitBreaker breaker; + private final AtomicBoolean closed = new AtomicBoolean(false); + + AbstractMemoryTrackingArray(CircuitBreaker breaker) { + this.breaker = breaker; + } + + @Override + public final void close() { + if (closed.compareAndSet(false, true)) { + breaker.addWithoutBreaking(-ramBytesUsed()); + } + } + } + + public static class MemoryTrackingTDigestDoubleArray extends AbstractMemoryTrackingArray implements TDigestDoubleArray { + static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(MemoryTrackingTDigestDoubleArray.class); + + private double[] array; + private int size; + + public MemoryTrackingTDigestDoubleArray(CircuitBreaker breaker, int initialSize) { + this(breaker, new double[initialSize]); + } + + public MemoryTrackingTDigestDoubleArray(CircuitBreaker breaker, double[] array) { + super(breaker); + this.array = array; + this.size = array.length; + } + + public static long estimatedRamBytesUsed(int size) { + return SHALLOW_SIZE + estimatedArraySize(size, Double.BYTES); + } + + @Override + public long ramBytesUsed() { + return estimatedRamBytesUsed(array.length); + } + + @Override + public int size() { + return size; + } + + @Override + public double get(int index) { + assert index >= 0 && index < size; + return array[index]; + } + + @Override + public void set(int index, double value) { + assert index >= 0 && index < size; + array[index] = value; + } + + @Override + public void add(double value) { + ensureCapacity(size + 1); + array[size++] = value; + } + + @Override + public void sort() { + Arrays.sort(array, 0, size); + } + + @Override + public void resize(int newSize) { + ensureCapacity(newSize); + + if (newSize > size) { + Arrays.fill(array, size, newSize, 0); + } + + size = newSize; + } + + @Override + public void ensureCapacity(int requiredCapacity) { + if (requiredCapacity > array.length) { + double[] oldArray = array; + // Used for used bytes assertion + long oldRamBytesUsed = ramBytesUsed(); + long oldArraySize = RamUsageEstimator.sizeOf(oldArray); + + int newSize = ArrayUtil.oversize(requiredCapacity, Double.BYTES); + long newArraySize = estimatedArraySize(newSize, Double.BYTES); + breaker.addEstimateBytesAndMaybeBreak(newArraySize, "tdigest-new-capacity-double-array"); + array = Arrays.copyOf(array, newSize); + breaker.addWithoutBreaking(-RamUsageEstimator.sizeOf(oldArray)); + + assert ramBytesUsed() - oldRamBytesUsed == newArraySize - oldArraySize + : "ramBytesUsed() should be aligned with manual array calculations"; + } + } + } + + public static class MemoryTrackingTDigestIntArray extends AbstractMemoryTrackingArray implements TDigestIntArray { + static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(MemoryTrackingTDigestIntArray.class); + + private int[] array; + private int size; + + public MemoryTrackingTDigestIntArray(CircuitBreaker breaker, int initialSize) { + this(breaker, new int[initialSize]); + } + + public MemoryTrackingTDigestIntArray(CircuitBreaker breaker, int[] array) { + super(breaker); + this.array = array; + this.size = array.length; + } + + public static long estimatedRamBytesUsed(int size) { + return SHALLOW_SIZE + estimatedArraySize(size, Integer.BYTES); + } + + @Override + public long ramBytesUsed() { + return estimatedRamBytesUsed(array.length); + } + + @Override + public int size() { + return size; + } + + @Override + public int get(int index) { + assert index >= 0 && index < size; + return array[index]; + } + + @Override + public void set(int index, int value) { + assert index >= 0 && index < size; + array[index] = value; + } + + @Override + public void resize(int newSize) { + ensureCapacity(newSize); + if (newSize > size) { + Arrays.fill(array, size, newSize, 0); + } + size = newSize; + } + + private void ensureCapacity(int requiredCapacity) { + if (requiredCapacity > array.length) { + int[] oldArray = array; + // Used for used bytes assertion + long oldRamBytesUsed = ramBytesUsed(); + long oldArraySize = RamUsageEstimator.sizeOf(oldArray); + + int newSize = ArrayUtil.oversize(requiredCapacity, Integer.BYTES); + long newArraySize = estimatedArraySize(newSize, Integer.BYTES); + breaker.addEstimateBytesAndMaybeBreak(newArraySize, "tdigest-new-capacity-int-array"); + array = Arrays.copyOf(array, newSize); + breaker.addWithoutBreaking(-RamUsageEstimator.sizeOf(oldArray)); + + assert ramBytesUsed() - oldRamBytesUsed == newArraySize - oldArraySize + : "ramBytesUsed() should be aligned with manual array calculations"; + } + } + } + + public static class MemoryTrackingTDigestLongArray extends AbstractMemoryTrackingArray implements TDigestLongArray { + static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(MemoryTrackingTDigestLongArray.class); + + private long[] array; + private int size; + + public MemoryTrackingTDigestLongArray(CircuitBreaker breaker, int initialSize) { + this(breaker, new long[initialSize]); + } + + public MemoryTrackingTDigestLongArray(CircuitBreaker breaker, long[] array) { + super(breaker); + this.array = array; + this.size = array.length; + } + + public static long estimatedRamBytesUsed(int size) { + return SHALLOW_SIZE + estimatedArraySize(size, Long.BYTES); + } + + @Override + public long ramBytesUsed() { + return estimatedRamBytesUsed(array.length); + } + + @Override + public int size() { + return size; + } + + @Override + public long get(int index) { + assert index >= 0 && index < size; + return array[index]; + } + + @Override + public void set(int index, long value) { + assert index >= 0 && index < size; + array[index] = value; + } + + @Override + public void resize(int newSize) { + ensureCapacity(newSize); + if (newSize > size) { + Arrays.fill(array, size, newSize, 0); + } + size = newSize; + } + + private void ensureCapacity(int requiredCapacity) { + if (requiredCapacity > array.length) { + long[] oldArray = array; + // Used for used bytes assertion + long oldRamBytesUsed = ramBytesUsed(); + long oldArraySize = RamUsageEstimator.sizeOf(oldArray); + + int newSize = ArrayUtil.oversize(requiredCapacity, Long.BYTES); + long newArraySize = estimatedArraySize(newSize, Long.BYTES); + breaker.addEstimateBytesAndMaybeBreak(newArraySize, "tdigest-new-capacity-long-array"); + array = Arrays.copyOf(array, newSize); + breaker.addWithoutBreaking(-RamUsageEstimator.sizeOf(oldArray)); + + assert ramBytesUsed() - oldRamBytesUsed == newArraySize - oldArraySize + : "ramBytesUsed() should be aligned with manual array calculations"; + } + } + } + + public static class MemoryTrackingTDigestByteArray extends AbstractMemoryTrackingArray implements TDigestByteArray { + static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(MemoryTrackingTDigestByteArray.class); + + private byte[] array; + private int size; + + public MemoryTrackingTDigestByteArray(CircuitBreaker breaker, int initialSize) { + this(breaker, new byte[initialSize]); + } + + public MemoryTrackingTDigestByteArray(CircuitBreaker breaker, byte[] array) { + super(breaker); + this.array = array; + this.size = array.length; + } + + public static long estimatedRamBytesUsed(int size) { + return SHALLOW_SIZE + estimatedArraySize(size, Byte.BYTES); + } + + @Override + public long ramBytesUsed() { + return estimatedRamBytesUsed(array.length); + } + + @Override + public int size() { + return size; + } + + @Override + public byte get(int index) { + assert index >= 0 && index < size; + return array[index]; + } + + @Override + public void set(int index, byte value) { + assert index >= 0 && index < size; + array[index] = value; + } + + @Override + public void resize(int newSize) { + ensureCapacity(newSize); + if (newSize > size) { + Arrays.fill(array, size, newSize, (byte) 0); + } + size = newSize; + } + + private void ensureCapacity(int requiredCapacity) { + if (requiredCapacity > array.length) { + byte[] oldArray = array; + // Used for used bytes assertion + long oldRamBytesUsed = ramBytesUsed(); + long oldArraySize = RamUsageEstimator.sizeOf(oldArray); + + int newSize = ArrayUtil.oversize(requiredCapacity, Byte.BYTES); + long newArraySize = estimatedArraySize(newSize, Byte.BYTES); + breaker.addEstimateBytesAndMaybeBreak(newArraySize, "tdigest-new-capacity-byte-array"); + array = Arrays.copyOf(array, newSize); + breaker.addWithoutBreaking(-RamUsageEstimator.sizeOf(oldArray)); + + assert ramBytesUsed() - oldRamBytesUsed == newArraySize - oldArraySize + : "ramBytesUsed() should be aligned with manual array calculations"; + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestState.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestState.java index 48bdb59e430a5..78ef81684a256 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestState.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestState.java @@ -11,10 +11,11 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.Centroid; import org.elasticsearch.tdigest.TDigest; import org.elasticsearch.tdigest.arrays.TDigestArrays; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; import java.io.IOException; import java.util.Collection; @@ -25,7 +26,7 @@ * through factory method params, providing one optimized for performance (e.g. MergingDigest or HybridDigest) by default, or optionally one * that produces highly accurate results regardless of input size but its construction over the sample population takes 2x-10x longer. */ -public class TDigestState { +public class TDigestState implements Releasable { private final double compression; @@ -54,7 +55,7 @@ static Type valueForHighAccuracy() { */ @Deprecated public static TDigestState create(double compression) { - return create(WrapperTDigestArrays.INSTANCE, compression); + return create(MemoryTrackingTDigestArrays.INSTANCE, compression); } /** @@ -81,7 +82,7 @@ public static TDigestState createOptimizedForAccuracy(TDigestArrays arrays, doub */ @Deprecated public static TDigestState create(double compression, TDigestExecutionHint executionHint) { - return create(WrapperTDigestArrays.INSTANCE, compression, executionHint); + return create(MemoryTrackingTDigestArrays.INSTANCE, compression, executionHint); } /** @@ -106,7 +107,7 @@ public static TDigestState create(TDigestArrays arrays, double compression, TDig * @return a TDigestState object */ public static TDigestState createUsingParamsFrom(TDigestState state) { - return new TDigestState(WrapperTDigestArrays.INSTANCE, state.type, state.compression); + return new TDigestState(MemoryTrackingTDigestArrays.INSTANCE, state.type, state.compression); } protected TDigestState(TDigestArrays arrays, Type type, double compression) { @@ -143,7 +144,7 @@ public static void write(TDigestState state, StreamOutput out) throws IOExceptio */ @Deprecated public static TDigestState read(StreamInput in) throws IOException { - return read(WrapperTDigestArrays.INSTANCE, in); + return read(MemoryTrackingTDigestArrays.INSTANCE, in); } public static TDigestState read(TDigestArrays arrays, StreamInput in) throws IOException { @@ -267,4 +268,9 @@ public final double getMin() { public final double getMax() { return tdigest.getMax(); } + + @Override + public void close() { + Releasables.close(tdigest); + } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArraysTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArraysTests.java new file mode 100644 index 0000000000000..e57f39becef7c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArraysTests.java @@ -0,0 +1,360 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.search.aggregations.metrics; + +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.tdigest.arrays.TDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestByteArray; +import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; +import org.elasticsearch.tdigest.arrays.TDigestIntArray; +import org.elasticsearch.tdigest.arrays.TDigestLongArray; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + +public class MemoryTrackingTDigestArraysTests extends ESTestCase { + // Int arrays + + public void testIntEmpty() { + try (TDigestIntArray array = intArray(0)) { + assertThat(array.size(), equalTo(0)); + } + } + + public void testIntGetAndSet() { + int initialSize = randomIntBetween(10, 1000); + try (TDigestIntArray array = intArray(initialSize)) { + assertThat(array.size(), equalTo(initialSize)); + + int value = randomInt(); + for (int i = 9; i < initialSize; i++) { + array.set(i, value); + } + + for (int i = 0; i < initialSize; i++) { + if (i < 9) { + assertThat(array.get(i), equalTo(0)); + } else { + assertThat(array.get(i), equalTo(value)); + } + } + } + } + + public void testIntResize() { + int initialSize = randomIntBetween(10, 1000); + try (TDigestIntArray array = intArray(initialSize)) { + assertThat(array.size(), equalTo(initialSize)); + + // Fill with a non-zero value + int value = randomBoolean() ? randomIntBetween(Integer.MIN_VALUE, -1) : randomIntBetween(1, Integer.MAX_VALUE); + for (int i = 0; i < initialSize; i++) { + array.set(i, value); + } + + // Resize to a size-1 + array.resize(initialSize - 1); + assertThat(array.size(), equalTo(initialSize - 1)); + + for (int i = 0; i < initialSize - 1; i++) { + assertThat(array.get(i), equalTo(value)); + } + + // Resize to the original size + 1 + array.resize(initialSize + 1); + assertThat(array.size(), equalTo(initialSize + 1)); + + // Ensure all new elements are 0 + for (int i = 0; i < initialSize - 1; i++) { + if (i < initialSize) { + assertThat(array.get(i), equalTo(value)); + } else { + assertThat(array.get(i), equalTo(0)); + } + } + } + } + + public void testIntBulkSet() { + int initialSize = randomIntBetween(10, 1000); + int sourceArraySize = randomIntBetween(0, initialSize); + + try (TDigestIntArray array = intArray(initialSize); TDigestIntArray source = intArray(sourceArraySize)) { + assertThat(array.size(), equalTo(initialSize)); + assertThat(source.size(), equalTo(sourceArraySize)); + + int value = randomInt(); + for (int i = 0; i < sourceArraySize; i++) { + source.set(i, value); + } + + int initialOffset = randomIntBetween(0, initialSize - sourceArraySize); + int sourceOffset = randomIntBetween(0, sourceArraySize - 1); + int elementsToCopy = randomIntBetween(1, sourceArraySize - sourceOffset); + + array.set(initialOffset, source, sourceOffset, elementsToCopy); + + for (int i = 0; i < initialSize; i++) { + if (i < initialOffset || i >= initialOffset + elementsToCopy) { + assertThat(array.get(i), equalTo(0)); + } else { + assertThat(array.get(i), equalTo(value)); + } + } + } + } + + // Long arrays + + public void testLongEmpty() { + try (TDigestIntArray array = intArray(0)) { + assertThat(array.size(), equalTo(0)); + } + } + + public void testLongGetAndSet() { + int initialSize = randomIntBetween(10, 1000); + try (TDigestLongArray array = longArray(initialSize)) { + assertThat(array.size(), equalTo(initialSize)); + + long value = randomLong(); + for (int i = 9; i < initialSize; i++) { + array.set(i, value); + } + + for (int i = 0; i < initialSize; i++) { + if (i < 9) { + assertThat(array.get(i), equalTo(0L)); + } else { + assertThat(array.get(i), equalTo(value)); + } + } + } + } + + public void testLongResize() { + int initialSize = randomIntBetween(10, 1000); + try (TDigestLongArray array = longArray(initialSize)) { + assertThat(array.size(), equalTo(initialSize)); + + // Fill with a non-zero value + long value = randomBoolean() ? randomLongBetween(Long.MIN_VALUE, -1) : randomLongBetween(1, Long.MAX_VALUE); + for (int i = 0; i < initialSize; i++) { + array.set(i, value); + } + + // Resize to a size-1 + array.resize(initialSize - 1); + assertThat(array.size(), equalTo(initialSize - 1)); + + for (int i = 0; i < initialSize - 1; i++) { + assertThat(array.get(i), equalTo(value)); + } + + // Resize to the original size + 1 + array.resize(initialSize + 1); + assertThat(array.size(), equalTo(initialSize + 1)); + + // Ensure all new elements are 0 + for (int i = 0; i < initialSize - 1; i++) { + if (i < initialSize) { + assertThat(array.get(i), equalTo(value)); + } else { + assertThat(array.get(i), equalTo(0L)); + } + } + } + } + + // Byte arrays + + public void testByteEmpty() { + try (TDigestByteArray array = byteArray(0)) { + assertThat(array.size(), equalTo(0)); + } + } + + public void testByteGetAndSet() { + int initialSize = randomIntBetween(10, 1000); + try (TDigestByteArray array = byteArray(initialSize)) { + assertThat(array.size(), equalTo(initialSize)); + + byte value = randomByte(); + for (int i = 9; i < initialSize; i++) { + array.set(i, value); + } + + for (int i = 0; i < initialSize; i++) { + if (i < 9) { + assertThat(array.get(i), equalTo((byte) 0)); + } else { + assertThat(array.get(i), equalTo(value)); + } + } + } + } + + public void testByteResize() { + int initialSize = randomIntBetween(10, 1000); + try (TDigestByteArray array = byteArray(initialSize)) { + assertThat(array.size(), equalTo(initialSize)); + + // Fill with a non-zero value + byte value = randomBoolean() ? randomByteBetween(Byte.MIN_VALUE, (byte) -1) : randomByteBetween((byte) 1, Byte.MAX_VALUE); + for (int i = 0; i < initialSize; i++) { + array.set(i, value); + } + + // Resize to a size-1 + array.resize(initialSize - 1); + assertThat(array.size(), equalTo(initialSize - 1)); + + for (int i = 0; i < initialSize - 1; i++) { + assertThat(array.get(i), equalTo(value)); + } + + // Resize to the original size + 1 + array.resize(initialSize + 1); + assertThat(array.size(), equalTo(initialSize + 1)); + + // Ensure all new elements are 0 + for (int i = 0; i < initialSize - 1; i++) { + if (i < initialSize) { + assertThat(array.get(i), equalTo(value)); + } else { + assertThat(array.get(i), equalTo((byte) 0)); + } + } + } + } + + // Double arrays + + public void testDoubleEmpty() { + try (TDigestDoubleArray array = doubleArray(0)) { + assertThat(array.size(), equalTo(0)); + } + } + + public void testDoubleGetAndSet() { + int initialSize = randomIntBetween(10, 1000); + try (TDigestDoubleArray array = doubleArray(initialSize)) { + assertThat(array.size(), equalTo(initialSize)); + + double value = randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true); + for (int i = 9; i < initialSize; i++) { + array.set(i, value); + } + + for (int i = 0; i < initialSize; i++) { + if (i < 9) { + assertThat(array.get(i), equalTo(0.0)); + } else { + assertThat(array.get(i), equalTo(value)); + } + } + } + } + + public void testDoubleAdd() { + int initialSize = randomIntBetween(10, 1000); + try (TDigestDoubleArray array = doubleArray(initialSize)) { + assertThat(array.size(), equalTo(initialSize)); + + int newValueCount = randomIntBetween(1, 100); + if (randomBoolean()) { + array.ensureCapacity(initialSize + newValueCount); + } + double value = randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true); + for (int i = 0; i < newValueCount; i++) { + array.add(value); + } + + for (int i = 0; i < newValueCount; i++) { + if (i < initialSize) { + assertThat(array.get(i), equalTo(0.0)); + } else { + assertThat(array.get(i), equalTo(value)); + } + } + } + } + + public void testDoubleBulkSet() { + int initialSize = randomIntBetween(10, 1000); + int sourceArraySize = randomIntBetween(0, initialSize); + + try (TDigestDoubleArray array = doubleArray(initialSize); TDigestDoubleArray source = doubleArray(sourceArraySize)) { + assertThat(array.size(), equalTo(initialSize)); + assertThat(source.size(), equalTo(sourceArraySize)); + + double value = randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true); + for (int i = 0; i < sourceArraySize; i++) { + source.set(i, value); + } + + int initialOffset = randomIntBetween(0, initialSize - sourceArraySize); + int sourceOffset = randomIntBetween(0, sourceArraySize - 1); + int elementsToCopy = randomIntBetween(1, sourceArraySize - sourceOffset); + + array.set(initialOffset, source, sourceOffset, elementsToCopy); + + for (int i = 0; i < initialSize; i++) { + if (i < initialOffset || i >= initialOffset + elementsToCopy) { + assertThat(array.get(i), equalTo(0.0)); + } else { + assertThat(array.get(i), equalTo(value)); + } + } + } + } + + public void testDoubleSort() { + try (TDigestDoubleArray array = doubleArray(0)) { + int elementsToAdd = randomIntBetween(0, 100); + array.ensureCapacity(elementsToAdd); + for (int i = 0; i < elementsToAdd; i++) { + array.add(randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true)); + } + + array.sort(); + + double previous = -Double.MAX_VALUE; + for (int i = 0; i < array.size(); i++) { + double current = array.get(i); + assertThat(current, greaterThanOrEqualTo(previous)); + previous = current; + } + } + } + + // Helpers + + private TDigestIntArray intArray(int initialSize) { + return arrays().newIntArray(initialSize); + } + + private TDigestLongArray longArray(int initialSize) { + return arrays().newLongArray(initialSize); + } + + private TDigestByteArray byteArray(int initialSize) { + return arrays().newByteArray(initialSize); + } + + private TDigestDoubleArray doubleArray(int initialSize) { + return arrays().newDoubleArray(initialSize); + } + + private TDigestArrays arrays() { + return new MemoryTrackingTDigestArrays(newLimitedBreaker(ByteSizeValue.ofMb(100))); + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestStateTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestStateTests.java index e7799a133b5af..56d3d674b28ca 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestStateTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestStateTests.java @@ -16,8 +16,9 @@ import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; @@ -33,140 +34,150 @@ public class TDigestStateTests extends ESTestCase { public void testMoreThan4BValues() { // Regression test for #19528 // See https://github.com/tdunning/t-digest/pull/70/files#diff-4487072cee29b939694825647928f742R439 - TDigestState digest = TDigestState.create(arrays(), 100); - for (int i = 0; i < 1000; ++i) { - digest.add(randomDouble()); - } - final int count = 1 << 29; - for (int i = 0; i < 10; ++i) { - digest.add(randomDouble(), count); - } - assertEquals(1000 + 10L * (1 << 29), digest.size()); - assertTrue(digest.size() > 2L * Integer.MAX_VALUE); - final double[] quantiles = new double[] { 0, 0.1, 0.5, 0.9, 1, randomDouble() }; - Arrays.sort(quantiles); - double prev = Double.NEGATIVE_INFINITY; - for (double q : quantiles) { - final double v = digest.quantile(q); - logger.trace("q=" + q + ", v=" + v); - assertThat(v, Matchers.either(Matchers.closeTo(prev, 0.0000001D)).or(Matchers.greaterThan(prev))); - assertTrue("Unexpectedly low value: " + v, v >= 0.0); - assertTrue("Unexpectedly high value: " + v, v <= 1.0); - prev = v; + try (TDigestState digest = TDigestState.create(arrays(), 100)) { + for (int i = 0; i < 1000; ++i) { + digest.add(randomDouble()); + } + final int count = 1 << 29; + for (int i = 0; i < 10; ++i) { + digest.add(randomDouble(), count); + } + assertEquals(1000 + 10L * (1 << 29), digest.size()); + assertTrue(digest.size() > 2L * Integer.MAX_VALUE); + final double[] quantiles = new double[] { 0, 0.1, 0.5, 0.9, 1, randomDouble() }; + Arrays.sort(quantiles); + double prev = Double.NEGATIVE_INFINITY; + for (double q : quantiles) { + final double v = digest.quantile(q); + logger.trace("q=" + q + ", v=" + v); + assertThat(v, Matchers.either(Matchers.closeTo(prev, 0.0000001D)).or(Matchers.greaterThan(prev))); + assertTrue("Unexpectedly low value: " + v, v >= 0.0); + assertTrue("Unexpectedly high value: " + v, v <= 1.0); + prev = v; + } } } public void testEqualsHashCode() { - final TDigestState empty1 = new EmptyTDigestState(); - final TDigestState empty2 = new EmptyTDigestState(); - final TDigestState a = TDigestState.create(arrays(), 200); - final TDigestState b = TDigestState.create(arrays(), 100); - final TDigestState c = TDigestState.create(arrays(), 100); + try ( + TDigestState empty1 = new EmptyTDigestState(); + TDigestState empty2 = new EmptyTDigestState(); + TDigestState a = TDigestState.create(arrays(), 200); + TDigestState b = TDigestState.create(arrays(), 100); + TDigestState c = TDigestState.create(arrays(), 100); + ) { - assertEquals(empty1, empty2); - assertEquals(empty1.hashCode(), empty2.hashCode()); + assertEquals(empty1, empty2); + assertEquals(empty1.hashCode(), empty2.hashCode()); - assertNotEquals(a, b); - assertNotEquals(a.hashCode(), b.hashCode()); + assertNotEquals(a, b); + assertNotEquals(a.hashCode(), b.hashCode()); - assertNotEquals(a, c); - assertNotEquals(a.hashCode(), c.hashCode()); + assertNotEquals(a, c); + assertNotEquals(a.hashCode(), c.hashCode()); - assertEquals(b, c); - assertEquals(b.hashCode(), c.hashCode()); + assertEquals(b, c); + assertEquals(b.hashCode(), c.hashCode()); - for (int i = 0; i < 100; i++) { - double value = randomDouble(); - a.add(value); - b.add(value); - c.add(value); - } + for (int i = 0; i < 100; i++) { + double value = randomDouble(); + a.add(value); + b.add(value); + c.add(value); + } - assertNotEquals(a, b); - assertNotEquals(a.hashCode(), b.hashCode()); + assertNotEquals(a, b); + assertNotEquals(a.hashCode(), b.hashCode()); - assertNotEquals(a, c); - assertNotEquals(a.hashCode(), c.hashCode()); + assertNotEquals(a, c); + assertNotEquals(a.hashCode(), c.hashCode()); - assertEquals(b, c); - assertEquals(b.hashCode(), c.hashCode()); + assertEquals(b, c); + assertEquals(b.hashCode(), c.hashCode()); - b.add(randomDouble()); - c.add(randomDouble()); + b.add(randomDouble()); + c.add(randomDouble()); - assertNotEquals(b, c); - assertNotEquals(b.hashCode(), c.hashCode()); + assertNotEquals(b, c); + assertNotEquals(b.hashCode(), c.hashCode()); + } } public void testHash() { final HashMap map = new HashMap<>(); final Set set = new HashSet<>(); - final TDigestState empty1 = new EmptyTDigestState(); - final TDigestState empty2 = new EmptyTDigestState(); - final TDigestState a = TDigestState.create(arrays(), 200); - final TDigestState b = TDigestState.create(arrays(), 100); - final TDigestState c = TDigestState.create(arrays(), 100); - - a.add(randomDouble()); - b.add(randomDouble()); - c.add(randomDouble()); - expectThrows(UnsupportedOperationException.class, () -> empty1.add(randomDouble())); - expectThrows(UnsupportedOperationException.class, () -> empty2.add(a)); - - map.put("empty1", empty1); - map.put("empty2", empty2); - map.put("a", a); - map.put("b", b); - map.put("c", c); - set.add(empty1); - set.add(empty2); - set.add(a); - set.add(b); - set.add(c); - - assertEquals(5, map.size()); - assertEquals(4, set.size()); - - assertEquals(empty1, map.get("empty1")); - assertEquals(empty2, map.get("empty2")); - assertEquals(a, map.get("a")); - assertEquals(b, map.get("b")); - assertEquals(c, map.get("c")); - - assertTrue(set.stream().anyMatch(digest -> digest.equals(a))); - assertTrue(set.stream().anyMatch(digest -> digest.equals(b))); - assertTrue(set.stream().anyMatch(digest -> digest.equals(c))); - assertTrue(set.stream().anyMatch(digest -> digest.equals(empty1))); - assertTrue(set.stream().anyMatch(digest -> digest.equals(empty2))); + try ( + TDigestState empty1 = new EmptyTDigestState(); + TDigestState empty2 = new EmptyTDigestState(); + TDigestState a = TDigestState.create(arrays(), 200); + TDigestState b = TDigestState.create(arrays(), 100); + TDigestState c = TDigestState.create(arrays(), 100); + ) { + + a.add(randomDouble()); + b.add(randomDouble()); + c.add(randomDouble()); + expectThrows(UnsupportedOperationException.class, () -> empty1.add(randomDouble())); + expectThrows(UnsupportedOperationException.class, () -> empty2.add(a)); + + map.put("empty1", empty1); + map.put("empty2", empty2); + map.put("a", a); + map.put("b", b); + map.put("c", c); + set.add(empty1); + set.add(empty2); + set.add(a); + set.add(b); + set.add(c); + + assertEquals(5, map.size()); + assertEquals(4, set.size()); + + assertEquals(empty1, map.get("empty1")); + assertEquals(empty2, map.get("empty2")); + assertEquals(a, map.get("a")); + assertEquals(b, map.get("b")); + assertEquals(c, map.get("c")); + + assertTrue(set.stream().anyMatch(digest -> digest.equals(a))); + assertTrue(set.stream().anyMatch(digest -> digest.equals(b))); + assertTrue(set.stream().anyMatch(digest -> digest.equals(c))); + assertTrue(set.stream().anyMatch(digest -> digest.equals(empty1))); + assertTrue(set.stream().anyMatch(digest -> digest.equals(empty2))); + } } public void testFactoryMethods() { - TDigestState fast = TDigestState.create(arrays(), 100); - TDigestState anotherFast = TDigestState.create(arrays(), 100); - TDigestState accurate = TDigestState.createOptimizedForAccuracy(arrays(), 100); - TDigestState anotherAccurate = TDigestState.createUsingParamsFrom(accurate); - - for (int i = 0; i < 100; i++) { - fast.add(i); - anotherFast.add(i); - accurate.add(i); - anotherAccurate.add(i); - } + try ( + TDigestState fast = TDigestState.create(arrays(), 100); + TDigestState anotherFast = TDigestState.create(arrays(), 100); + TDigestState accurate = TDigestState.createOptimizedForAccuracy(arrays(), 100); + TDigestState anotherAccurate = TDigestState.createUsingParamsFrom(accurate); + ) { - for (double p : new double[] { 0.1, 1, 10, 25, 50, 75, 90, 99, 99.9 }) { - double q = p / 100; - assertEquals(fast.quantile(q), accurate.quantile(q), 0.5); - assertEquals(fast.quantile(q), anotherFast.quantile(q), 1e-5); - assertEquals(accurate.quantile(q), anotherAccurate.quantile(q), 1e-5); + for (int i = 0; i < 100; i++) { + fast.add(i); + anotherFast.add(i); + accurate.add(i); + anotherAccurate.add(i); + } + + for (double p : new double[] { 0.1, 1, 10, 25, 50, 75, 90, 99, 99.9 }) { + double q = p / 100; + assertEquals(fast.quantile(q), accurate.quantile(q), 0.5); + assertEquals(fast.quantile(q), anotherFast.quantile(q), 1e-5); + assertEquals(accurate.quantile(q), anotherAccurate.quantile(q), 1e-5); + } + + assertEquals(fast, anotherFast); + assertEquals(accurate, anotherAccurate); + assertNotEquals(fast, accurate); + assertNotEquals(anotherFast, anotherAccurate); } - - assertEquals(fast, anotherFast); - assertEquals(accurate, anotherAccurate); - assertNotEquals(fast, accurate); - assertNotEquals(anotherFast, anotherAccurate); } - private static TDigestState writeToAndReadFrom(TDigestState state, TransportVersion version) throws IOException { + private TDigestState writeToAndReadFrom(TDigestState state, TransportVersion version) throws IOException { BytesRef serializedAggs = serialize(state, version); try ( StreamInput in = new NamedWriteableAwareStreamInput( @@ -203,9 +214,11 @@ public void testSerialization() throws IOException { TDigestState serializedBackwardsCompatible = writeToAndReadFrom(state, TransportVersions.V_8_8_1); assertNotEquals(serializedBackwardsCompatible, state); assertEquals(serializedBackwardsCompatible, backwardsCompatible); + + Releasables.close(state, backwardsCompatible, serialized, serializedBackwardsCompatible); } - private static TDigestArrays arrays() { - return WrapperTDigestArrays.INSTANCE; + private TDigestArrays arrays() { + return new MemoryTrackingTDigestArrays(newLimitedBreaker(ByteSizeValue.ofMb(100))); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 00cfedb257187..e6fc32a8ebe1b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -57,6 +57,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.CompositeBytesReference; @@ -577,6 +578,21 @@ public final void before() { } } + private final List breakers = Collections.synchronizedList(new ArrayList<>()); + + protected final CircuitBreaker newLimitedBreaker(ByteSizeValue max) { + CircuitBreaker breaker = new MockBigArrays.LimitedBreaker("", max); + breakers.add(breaker); + return breaker; + } + + @After + public final void allBreakersMemoryReleased() { + for (CircuitBreaker breaker : breakers) { + assertThat(breaker.getUsed(), equalTo(0L)); + } + } + /** * Whether or not we check after each test whether it has left warnings behind. That happens if any deprecated feature or syntax * was used by the test and the test didn't assert on it using {@link #assertWarnings(String...)}. From 5e019998ef0e57a95ba73d114123e46ad52af9bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?= Date: Thu, 26 Sep 2024 16:09:28 +0200 Subject: [PATCH 24/36] [DOCS] Improves semantic text documentation. (#113606) --- .../inference/delete-inference.asciidoc | 7 ++-- .../mapping/types/semantic-text.asciidoc | 39 ++++++++++++++++--- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/docs/reference/inference/delete-inference.asciidoc b/docs/reference/inference/delete-inference.asciidoc index 4df72ba672092..bee39bf9b9851 100644 --- a/docs/reference/inference/delete-inference.asciidoc +++ b/docs/reference/inference/delete-inference.asciidoc @@ -49,13 +49,12 @@ The type of {infer} task that the model performs. `dry_run`:: (Optional, Boolean) -When `true`, checks the {infer} processors that reference the endpoint and -returns them in a list, but does not delete the endpoint. Defaults to `false`. +When `true`, checks the `semantic_text` fields and {infer} processors that reference the endpoint and returns them in a list, but does not delete the endpoint. +Defaults to `false`. `force`:: (Optional, Boolean) -Deletes the endpoint regardless if it's used in an {infer} pipeline or in a -`semantic_text` field. +Deletes the endpoint regardless if it's used in a `semantic_text` field or in an {infer} pipeline. [discrete] diff --git a/docs/reference/mapping/types/semantic-text.asciidoc b/docs/reference/mapping/types/semantic-text.asciidoc index a006f288dc66d..d0fdf0145aa58 100644 --- a/docs/reference/mapping/types/semantic-text.asciidoc +++ b/docs/reference/mapping/types/semantic-text.asciidoc @@ -14,9 +14,8 @@ The `semantic_text` field type specifies an inference endpoint identifier that w You can create the inference endpoint by using the <>. This field type and the <> type make it simpler to perform semantic search on your data. -Using `semantic_text`, you won't need to specify how to generate embeddings for -your data, or how to index it. The inference endpoint automatically determines -the embedding generation, indexing, and query to use. +Using `semantic_text`, you won't need to specify how to generate embeddings for your data, or how to index it. +The {infer} endpoint automatically determines the embedding generation, indexing, and query to use. [source,console] ------------------------------------------------------------ @@ -32,7 +31,29 @@ PUT my-index-000001 } } ------------------------------------------------------------ -// TEST[skip:TBD] +// TEST[skip:Requires inference endpoint] + + +The recommended way to use semantic_text is by having dedicated {infer} endpoints for ingestion and search. +This ensures that search speed remains unaffected by ingestion workloads, and vice versa. +After creating dedicated {infer} endpoints for both, you can reference them using the `inference_id` and `search_inference_id` parameters when setting up the index mapping for an index that uses the `semantic_text` field. + +[source,console] +------------------------------------------------------------ +PUT my-index-000002 +{ + "mappings": { + "properties": { + "inference_field": { + "type": "semantic_text", + "inference_id": "my-elser-endpoint-for-ingest", + "search_inference_id": "my-elser-endpoint-for-search" + } + } + } +} +------------------------------------------------------------ +// TEST[skip:Requires inference endpoint] [discrete] @@ -41,9 +62,15 @@ PUT my-index-000001 `inference_id`:: (Required, string) -Inference endpoint that will be used to generate the embeddings for the field. +{infer-cap} endpoint that will be used to generate the embeddings for the field. Use the <> to create the endpoint. +If `search_inference_id` is specified, the {infer} endpoint defined by `inference_id` will only be used at index time. +`search_inference_id`:: +(Optional, string) +{infer-cap} endpoint that will be used to generate embeddings at query time. +Use the <> to create the endpoint. +If not specified, the {infer} endpoint defined by `inference_id` will be used at both index and query time. [discrete] [[infer-endpoint-validation]] @@ -55,6 +82,7 @@ When the first document is indexed, the `inference_id` will be used to generate WARNING: Removing an {infer} endpoint will cause ingestion of documents and semantic queries to fail on indices that define `semantic_text` fields with that {infer} endpoint as their `inference_id`. Trying to <> that is used on a `semantic_text` field will result in an error. + [discrete] [[auto-text-chunking]] ==== Automatic text chunking @@ -183,6 +211,7 @@ PUT test-index/_bulk Notice that both the `semantic_text` field and the source field are updated in the bulk request. + [discrete] [[limitations]] ==== Limitations From 6e73c1423b6c940fab9867346b1af708c9096be7 Mon Sep 17 00:00:00 2001 From: kosabogi <105062005+kosabogi@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:12:28 +0200 Subject: [PATCH 25/36] Adds text_similarity task type to inference processor documentation (#113517) --- .../ingest/processors/inference.asciidoc | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/reference/ingest/processors/inference.asciidoc b/docs/reference/ingest/processors/inference.asciidoc index c942959d34e53..fa4f246cdd7c8 100644 --- a/docs/reference/ingest/processors/inference.asciidoc +++ b/docs/reference/ingest/processors/inference.asciidoc @@ -455,6 +455,29 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio ======= ===== +[discrete] +[[inference-processor-text-similarity-opt]] +==== Text similarity configuration options + +`text_similarity`::: +(Object, optional) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity] ++ +.Properties of text_similarity inference +[%collapsible%open] +===== +`span_score_combination_function`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity-span-score-func] + +`tokenization`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] ++ +Refer to <> to review the properties of the +`tokenization` object. +===== + [discrete] [[inference-processor-zero-shot-opt]] From f0339ed30c054664ed4180dbab98f8efabb713d4 Mon Sep 17 00:00:00 2001 From: Jan Kuipers <148754765+jan-elastic@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:18:18 +0200 Subject: [PATCH 26/36] Adaptive allocations: scale to zero allocations (#113455) --- .../test/cluster/FeatureFlag.java | 3 +- .../UpdateTrainedModelDeploymentAction.java | 2 +- .../xpack/ml/MachineLearning.java | 14 ++- .../xpack/ml/MlInitializationService.java | 7 +- .../TransportExternalInferModelAction.java | 7 +- .../TransportInternalInferModelAction.java | 23 +++-- .../AdaptiveAllocationsScaler.java | 23 ++++- .../AdaptiveAllocationsScalerService.java | 95 +++++++++++-------- .../ScaleToZeroFeatureFlag.java | 20 ++++ .../ml/MlInitializationServiceTests.java | 14 +-- .../AdaptiveAllocationsScalerTests.java | 43 +++++++++ 11 files changed, 180 insertions(+), 71 deletions(-) create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/ScaleToZeroFeatureFlag.java diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java index cb98f9de31ff5..7df791bf11559 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java @@ -18,7 +18,8 @@ public enum FeatureFlag { TIME_SERIES_MODE("es.index_mode_feature_flag_registered=true", Version.fromString("8.0.0"), null), FAILURE_STORE_ENABLED("es.failure_store_feature_flag_enabled=true", Version.fromString("8.12.0"), null), - CHUNKING_SETTINGS_ENABLED("es.inference_chunking_settings_feature_flag_enabled=true", Version.fromString("8.16.0"), null); + CHUNKING_SETTINGS_ENABLED("es.inference_chunking_settings_feature_flag_enabled=true", Version.fromString("8.16.0"), null), + INFERENCE_SCALE_TO_ZERO("es.inference_scale_to_zero_feature_flag_enabled=true", Version.fromString("8.16.0"), null); public final String systemProperty; public final Version from; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentAction.java index 7fca223b2ee7e..cb578fdb157de 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentAction.java @@ -161,7 +161,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public ActionRequestValidationException validate() { ActionRequestValidationException validationException = new ActionRequestValidationException(); if (numberOfAllocations != null) { - if (numberOfAllocations < 1) { + if (numberOfAllocations < 0 || (isInternal == false && numberOfAllocations == 0)) { validationException.addValidationError("[" + NUMBER_OF_ALLOCATIONS + "] must be a positive integer"); } if (isInternal == false diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 1bc867a849090..f8a590a23a2c1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -327,6 +327,7 @@ import org.elasticsearch.xpack.ml.dataframe.process.results.AnalyticsResult; import org.elasticsearch.xpack.ml.dataframe.process.results.MemoryUsageEstimationResult; import org.elasticsearch.xpack.ml.inference.TrainedModelStatsService; +import org.elasticsearch.xpack.ml.inference.adaptiveallocations.AdaptiveAllocationsScalerService; import org.elasticsearch.xpack.ml.inference.assignment.TrainedModelAssignmentClusterService; import org.elasticsearch.xpack.ml.inference.assignment.TrainedModelAssignmentService; import org.elasticsearch.xpack.ml.inference.deployment.DeploymentManager; @@ -1285,13 +1286,21 @@ public Collection createComponents(PluginServices services) { new MlAutoscalingDeciderService(memoryTracker, settings, nodeAvailabilityZoneMapper, clusterService) ); - MlInitializationService mlInitializationService = new MlInitializationService( - settings, + AdaptiveAllocationsScalerService adaptiveAllocationsScalerService = new AdaptiveAllocationsScalerService( threadPool, clusterService, client, inferenceAuditor, telemetryProvider.getMeterRegistry(), + machineLearningExtension.get().isNlpEnabled() + ); + + MlInitializationService mlInitializationService = new MlInitializationService( + settings, + threadPool, + clusterService, + client, + adaptiveAllocationsScalerService, mlAssignmentNotifier, machineLearningExtension.get().isAnomalyDetectionEnabled(), machineLearningExtension.get().isDataFrameAnalyticsEnabled(), @@ -1317,6 +1326,7 @@ public Collection createComponents(PluginServices services) { jobManagerHolder, autodetectProcessManager, mlInitializationService, + adaptiveAllocationsScalerService, jobDataCountsPersister, datafeedRunner, datafeedManager, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java index 98dfb13d9e3e4..45a71a80de077 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java @@ -30,11 +30,9 @@ import org.elasticsearch.common.component.LifecycleListener; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.gateway.GatewayService; -import org.elasticsearch.telemetry.metric.MeterRegistry; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex; import org.elasticsearch.xpack.ml.inference.adaptiveallocations.AdaptiveAllocationsScalerService; -import org.elasticsearch.xpack.ml.notifications.InferenceAuditor; import java.util.Collections; import java.util.Map; @@ -67,8 +65,7 @@ public final class MlInitializationService implements ClusterStateListener { ThreadPool threadPool, ClusterService clusterService, Client client, - InferenceAuditor inferenceAuditor, - MeterRegistry meterRegistry, + AdaptiveAllocationsScalerService adaptiveAllocationsScalerService, MlAssignmentNotifier mlAssignmentNotifier, boolean isAnomalyDetectionEnabled, boolean isDataFrameAnalyticsEnabled, @@ -88,7 +85,7 @@ public final class MlInitializationService implements ClusterStateListener { isDataFrameAnalyticsEnabled, isNlpEnabled ), - new AdaptiveAllocationsScalerService(threadPool, clusterService, client, inferenceAuditor, meterRegistry, isNlpEnabled), + adaptiveAllocationsScalerService, clusterService ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java index 545dcfbefecec..5603e9c4dca8d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.action.InferModelAction; +import org.elasticsearch.xpack.ml.inference.adaptiveallocations.AdaptiveAllocationsScalerService; import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService; import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider; @@ -25,7 +26,8 @@ public TransportExternalInferModelAction( Client client, ClusterService clusterService, XPackLicenseState licenseState, - TrainedModelProvider trainedModelProvider + TrainedModelProvider trainedModelProvider, + AdaptiveAllocationsScalerService adaptiveAllocationsScalerService ) { super( InferModelAction.EXTERNAL_NAME, @@ -35,7 +37,8 @@ public TransportExternalInferModelAction( client, clusterService, licenseState, - trainedModelProvider + trainedModelProvider, + adaptiveAllocationsScalerService ); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java index 0c4064348b3f6..b69f8c7d62eb2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java @@ -42,6 +42,7 @@ import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.inference.adaptiveallocations.AdaptiveAllocationsScalerService; import org.elasticsearch.xpack.ml.inference.loadingservice.LocalModel; import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService; import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider; @@ -66,6 +67,7 @@ public class TransportInternalInferModelAction extends HandledTransportAction format("[%s] model deployment not allocated to any node", assignment.getDeploymentId())); - listener.onFailure( - ExceptionsHelper.conflictStatusException("Trained model deployment [" + request.getId() + "] is not allocated to any nodes") - ); + String message = "Trained model deployment [" + request.getId() + "] is not allocated to any nodes"; + boolean starting = adaptiveAllocationsScalerService.maybeStartAllocation(assignment); + if (starting) { + message += "; starting deployment of one allocation"; + } + logger.debug(message); + listener.onFailure(ExceptionsHelper.conflictStatusException(message)); return; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java index 044556d1b30ac..05e7202b8efe9 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.common.Strings; +import org.elasticsearch.core.TimeValue; /** * Processes measured requests counts and inference times and decides whether @@ -21,6 +22,12 @@ public class AdaptiveAllocationsScaler { static final double SCALE_UP_THRESHOLD = 0.9; private static final double SCALE_DOWN_THRESHOLD = 0.85; + /** + * The time interval without any requests that has to pass, before scaling down + * to zero allocations (in case min_allocations = 0). + */ + private static final long SCALE_TO_ZERO_AFTER_NO_REQUESTS_TIME_SECONDS = TimeValue.timeValueMinutes(15).getSeconds(); + /** * If the max_number_of_allocations is not set, use this value for now to prevent scaling up * to high numbers due to possible bugs or unexpected behaviour in the scaler. @@ -33,6 +40,7 @@ public class AdaptiveAllocationsScaler { private final String deploymentId; private final KalmanFilter1d requestRateEstimator; private final KalmanFilter1d inferenceTimeEstimator; + private double timeWithoutRequestsSeconds; private int numberOfAllocations; private int neededNumberOfAllocations; @@ -55,6 +63,7 @@ public class AdaptiveAllocationsScaler { // the number of allocations changes, which is passed explicitly to the estimator. requestRateEstimator = new KalmanFilter1d(deploymentId + ":rate", 100, true); inferenceTimeEstimator = new KalmanFilter1d(deploymentId + ":time", 100, false); + timeWithoutRequestsSeconds = 0.0; this.numberOfAllocations = numberOfAllocations; neededNumberOfAllocations = numberOfAllocations; minNumberOfAllocations = null; @@ -73,6 +82,11 @@ void setMinMaxNumberOfAllocations(Integer minNumberOfAllocations, Integer maxNum void process(AdaptiveAllocationsScalerService.Stats stats, double timeIntervalSeconds, int numberOfAllocations) { lastMeasuredQueueSize = stats.pendingCount(); + if (stats.requestCount() > 0) { + timeWithoutRequestsSeconds = 0.0; + } else { + timeWithoutRequestsSeconds += timeIntervalSeconds; + } // The request rate (per second) is the request count divided by the time. // Assuming a Poisson process for the requests, the variance in the request @@ -145,7 +159,7 @@ Integer scale() { numberOfAllocations--; } - this.neededNumberOfAllocations = numberOfAllocations; + neededNumberOfAllocations = numberOfAllocations; if (maxNumberOfAllocations == null) { numberOfAllocations = Math.min(numberOfAllocations, MAX_NUMBER_OF_ALLOCATIONS_SAFEGUARD); @@ -156,6 +170,13 @@ Integer scale() { if (maxNumberOfAllocations != null) { numberOfAllocations = Math.min(numberOfAllocations, maxNumberOfAllocations); } + if (ScaleToZeroFeatureFlag.isEnabled() + && (minNumberOfAllocations == null || minNumberOfAllocations == 0) + && timeWithoutRequestsSeconds > SCALE_TO_ZERO_AFTER_NO_REQUESTS_TIME_SECONDS) { + logger.debug("[{}] adaptive allocations scaler: scaling down to zero, because of no requests.", deploymentId); + numberOfAllocations = 0; + neededNumberOfAllocations = 0; + } if (numberOfAllocations != oldNumberOfAllocations) { logger.debug( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java index bbe90f769818b..775279a6b2553 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java @@ -415,49 +415,60 @@ private void processDeploymentStats(GetDeploymentStatsAction.Response statsRespo if (newNumberOfAllocations > numberOfAllocations.get(deploymentId)) { lastScaleUpTimesMillis.put(deploymentId, now); } - UpdateTrainedModelDeploymentAction.Request updateRequest = new UpdateTrainedModelDeploymentAction.Request(deploymentId); - updateRequest.setNumberOfAllocations(newNumberOfAllocations); - updateRequest.setIsInternal(true); - ClientHelper.executeAsyncWithOrigin( - client, - ClientHelper.ML_ORIGIN, - UpdateTrainedModelDeploymentAction.INSTANCE, - updateRequest, - ActionListener.wrap(updateResponse -> { - logger.info("adaptive allocations scaler: scaled [{}] to [{}] allocations.", deploymentId, newNumberOfAllocations); - threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME) - .execute( - () -> inferenceAuditor.info( - deploymentId, - Strings.format( - "adaptive allocations scaler: scaled [%s] to [%s] allocations.", - deploymentId, - newNumberOfAllocations - ) - ) - ); - }, e -> { - logger.atLevel(Level.WARN) - .withThrowable(e) - .log( - "adaptive allocations scaler: scaling [{}] to [{}] allocations failed.", - deploymentId, - newNumberOfAllocations - ); - threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME) - .execute( - () -> inferenceAuditor.warning( - deploymentId, - Strings.format( - "adaptive allocations scaler: scaling [%s] to [%s] allocations failed.", - deploymentId, - newNumberOfAllocations - ) - ) - ); - }) - ); + updateNumberOfAllocations(deploymentId, newNumberOfAllocations); } } } + + public boolean maybeStartAllocation(TrainedModelAssignment assignment) { + if (ScaleToZeroFeatureFlag.isEnabled() + && assignment.getAdaptiveAllocationsSettings() != null + && assignment.getAdaptiveAllocationsSettings().getEnabled() == Boolean.TRUE) { + lastScaleUpTimesMillis.put(assignment.getDeploymentId(), System.currentTimeMillis()); + updateNumberOfAllocations(assignment.getDeploymentId(), 1); + return true; + } + return false; + } + + private void updateNumberOfAllocations(String deploymentId, int numberOfAllocations) { + UpdateTrainedModelDeploymentAction.Request updateRequest = new UpdateTrainedModelDeploymentAction.Request(deploymentId); + updateRequest.setNumberOfAllocations(numberOfAllocations); + updateRequest.setIsInternal(true); + ClientHelper.executeAsyncWithOrigin( + client, + ClientHelper.ML_ORIGIN, + UpdateTrainedModelDeploymentAction.INSTANCE, + updateRequest, + ActionListener.wrap(updateResponse -> { + logger.info("adaptive allocations scaler: scaled [{}] to [{}] allocations.", deploymentId, numberOfAllocations); + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME) + .execute( + () -> inferenceAuditor.info( + deploymentId, + Strings.format( + "adaptive allocations scaler: scaled [%s] to [%s] allocations.", + deploymentId, + numberOfAllocations + ) + ) + ); + }, e -> { + logger.atLevel(Level.WARN) + .withThrowable(e) + .log("adaptive allocations scaler: scaling [{}] to [{}] allocations failed.", deploymentId, numberOfAllocations); + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME) + .execute( + () -> inferenceAuditor.warning( + deploymentId, + Strings.format( + "adaptive allocations scaler: scaling [%s] to [%s] allocations failed.", + deploymentId, + numberOfAllocations + ) + ) + ); + }) + ); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/ScaleToZeroFeatureFlag.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/ScaleToZeroFeatureFlag.java new file mode 100644 index 0000000000000..072b8c5593c93 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/ScaleToZeroFeatureFlag.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.inference.adaptiveallocations; + +import org.elasticsearch.common.util.FeatureFlag; + +public class ScaleToZeroFeatureFlag { + private ScaleToZeroFeatureFlag() {} + + private static final FeatureFlag FEATURE_FLAG = new FeatureFlag("inference_scale_to_zero"); + + public static boolean isEnabled() { + return FEATURE_FLAG.isEnabled(); + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java index a5b9597886e15..80c957ecb7a09 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java @@ -17,11 +17,9 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue; -import org.elasticsearch.telemetry.metric.MeterRegistry; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.ml.inference.adaptiveallocations.AdaptiveAllocationsScalerService; -import org.elasticsearch.xpack.ml.notifications.InferenceAuditor; import org.junit.Before; import java.util.Map; @@ -40,8 +38,7 @@ public class MlInitializationServiceTests extends ESTestCase { private ThreadPool threadPool; private ClusterService clusterService; private Client client; - private InferenceAuditor inferenceAuditor; - private MeterRegistry meterRegistry; + private AdaptiveAllocationsScalerService adaptiveAllocationsScalerService; private MlAssignmentNotifier mlAssignmentNotifier; @Before @@ -50,8 +47,7 @@ public void setUpMocks() { threadPool = deterministicTaskQueue.getThreadPool(); clusterService = mock(ClusterService.class); client = mock(Client.class); - inferenceAuditor = mock(InferenceAuditor.class); - meterRegistry = mock(MeterRegistry.class); + adaptiveAllocationsScalerService = mock(AdaptiveAllocationsScalerService.class); mlAssignmentNotifier = mock(MlAssignmentNotifier.class); when(clusterService.getClusterName()).thenReturn(CLUSTER_NAME); @@ -77,8 +73,7 @@ public void testInitialize() { threadPool, clusterService, client, - inferenceAuditor, - meterRegistry, + adaptiveAllocationsScalerService, mlAssignmentNotifier, true, true, @@ -94,8 +89,7 @@ public void testInitialize_noMasterNode() { threadPool, clusterService, client, - inferenceAuditor, - meterRegistry, + adaptiveAllocationsScalerService, mlAssignmentNotifier, true, true, diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java index 08097357725d0..44aaba88c58a8 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java @@ -146,4 +146,47 @@ public void testAutoscaling_maxAllocationsSafeguard() { adaptiveAllocationsScaler.setMinMaxNumberOfAllocations(2, 77); assertThat(adaptiveAllocationsScaler.scale(), equalTo(77)); } + + public void testAutoscaling_scaleDownToZeroAllocations() { + AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1); + // 1 hour with 1 request per 1 seconds, so don't scale. + for (int i = 0; i < 3600; i++) { + adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(1, 0, 0, 0.05), 1, 1); + assertThat(adaptiveAllocationsScaler.scale(), nullValue()); + } + // 15 minutes with no requests, so don't scale. + for (int i = 0; i < 900; i++) { + adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(0, 0, 0, 0.05), 1, 1); + assertThat(adaptiveAllocationsScaler.scale(), nullValue()); + } + // 1 second with a request, so don't scale. + adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(1, 0, 0, 0.05), 1, 1); + assertThat(adaptiveAllocationsScaler.scale(), nullValue()); + // 15 minutes with no requests, so don't scale. + for (int i = 0; i < 900; i++) { + adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(0, 0, 0, 0.05), 1, 1); + assertThat(adaptiveAllocationsScaler.scale(), nullValue()); + } + // another second with no requests, so scale to zero allocations. + adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(0, 0, 0, 0.05), 1, 1); + assertThat(adaptiveAllocationsScaler.scale(), equalTo(0)); + // 15 minutes with no requests, so don't scale. + for (int i = 0; i < 900; i++) { + adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(0, 0, 0, 0.05), 1, 0); + assertThat(adaptiveAllocationsScaler.scale(), nullValue()); + } + } + + public void testAutoscaling_dontScaleDownToZeroAllocationsWhenMinAllocationsIsSet() { + assumeTrue("Should only run if adaptive allocations feature flag is enabled", ScaleToZeroFeatureFlag.isEnabled()); + + AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1); + adaptiveAllocationsScaler.setMinMaxNumberOfAllocations(1, null); + + // 1 hour with no requests, + for (int i = 0; i < 3600; i++) { + adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(1, 0, 0, 0.05), 1, 1); + assertThat(adaptiveAllocationsScaler.scale(), nullValue()); + } + } } From 052dbb4dacca29c3abf96e1b1579c569a2ec7095 Mon Sep 17 00:00:00 2001 From: Kushal-Dalasaniya <108124477+Kushal-Dalasaniya@users.noreply.github.com> Date: Thu, 26 Sep 2024 20:12:57 +0530 Subject: [PATCH 27/36] Optimize error handling after lazy rollovers (#111572) This commit improves the performance of the error-handling process after a lazy rollover or an index creation failed. --- .../action/bulk/BulkOperation.java | 20 --- .../action/bulk/TransportBulkAction.java | 113 ++++++------ .../action/bulk/BulkOperationTests.java | 6 - ...ActionIndicesThatCannotBeCreatedTests.java | 164 ------------------ .../bulk/TransportBulkActionIngestTests.java | 4 +- .../action/bulk/TransportBulkActionTests.java | 100 ++++++++++- .../bulk/TransportBulkActionTookTests.java | 12 +- 7 files changed, 164 insertions(+), 255 deletions(-) delete mode 100644 server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java index 1789acc1cb7a6..f04d07fb690c4 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java @@ -87,7 +87,6 @@ final class BulkOperation extends ActionRunnable { private final ConcurrentLinkedQueue failureStoreRedirects = new ConcurrentLinkedQueue<>(); private final long startTimeNanos; private final ClusterStateObserver observer; - private final Map indicesThatCannotBeCreated; private final Executor executor; private final LongSupplier relativeTimeProvider; private final FailureStoreDocumentConverter failureStoreDocumentConverter; @@ -107,7 +106,6 @@ final class BulkOperation extends ActionRunnable { BulkRequest bulkRequest, NodeClient client, AtomicArray responses, - Map indicesThatCannotBeCreated, IndexNameExpressionResolver indexNameExpressionResolver, LongSupplier relativeTimeProvider, long startTimeNanos, @@ -122,7 +120,6 @@ final class BulkOperation extends ActionRunnable { bulkRequest, client, responses, - indicesThatCannotBeCreated, indexNameExpressionResolver, relativeTimeProvider, startTimeNanos, @@ -141,7 +138,6 @@ final class BulkOperation extends ActionRunnable { BulkRequest bulkRequest, NodeClient client, AtomicArray responses, - Map indicesThatCannotBeCreated, IndexNameExpressionResolver indexNameExpressionResolver, LongSupplier relativeTimeProvider, long startTimeNanos, @@ -158,7 +154,6 @@ final class BulkOperation extends ActionRunnable { this.bulkRequest = bulkRequest; this.listener = listener; this.startTimeNanos = startTimeNanos; - this.indicesThatCannotBeCreated = indicesThatCannotBeCreated; this.executor = executor; this.relativeTimeProvider = relativeTimeProvider; this.indexNameExpressionResolver = indexNameExpressionResolver; @@ -298,9 +293,6 @@ private Map> groupRequestsByShards( if (addFailureIfRequiresAliasAndAliasIsMissing(docWriteRequest, bulkItemRequest.id(), metadata)) { continue; } - if (addFailureIfIndexCannotBeCreated(docWriteRequest, bulkItemRequest.id())) { - continue; - } if (addFailureIfRequiresDataStreamAndNoParentDataStream(docWriteRequest, bulkItemRequest.id(), metadata)) { continue; } @@ -759,18 +751,6 @@ private boolean addFailureIfIndexIsClosed(DocWriteRequest request, Index conc return false; } - private boolean addFailureIfIndexCannotBeCreated(DocWriteRequest request, int idx) { - IndexNotFoundException cannotCreate = indicesThatCannotBeCreated.get(request.index()); - if (cannotCreate != null) { - var failureStoreStatus = isFailureStoreRequest(request) - ? IndexDocFailureStoreStatus.FAILED - : IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; - addFailureAndDiscardRequest(request, idx, request.index(), cannotCreate, failureStoreStatus); - return true; - } - return false; - } - private static boolean isFailureStoreRequest(DocWriteRequest request) { return request instanceof IndexRequest ir && ir.isWriteToFailureStore(); } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 03768af029141..61adf41a9a276 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -43,7 +43,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.features.FeatureService; -import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.VersionType; import org.elasticsearch.indices.SystemIndices; @@ -60,6 +59,7 @@ import java.util.Objects; import java.util.Set; import java.util.SortedMap; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.function.Function; import java.util.function.LongSupplier; @@ -351,29 +351,36 @@ protected void createMissingIndicesAndIndexData( final AtomicArray responses = new AtomicArray<>(bulkRequest.requests.size()); // Optimizing when there are no prerequisite actions if (indicesToAutoCreate.isEmpty() && dataStreamsToBeRolledOver.isEmpty() && failureStoresToBeRolledOver.isEmpty()) { - executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses, Map.of()); + executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses); return; } - final Map indicesThatCannotBeCreated = new HashMap<>(); + Map indicesExceptions = new ConcurrentHashMap<>(); + Map dataStreamExceptions = new ConcurrentHashMap<>(); + Map failureStoreExceptions = new ConcurrentHashMap<>(); Runnable executeBulkRunnable = () -> executor.execute(new ActionRunnable<>(listener) { @Override protected void doRun() { - executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses, indicesThatCannotBeCreated); + failRequestsWhenPrerequisiteActionFailed( + indicesExceptions, + dataStreamExceptions, + failureStoreExceptions, + bulkRequest, + responses + ); + executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses); } }); try (RefCountingRunnable refs = new RefCountingRunnable(executeBulkRunnable)) { - createIndices(bulkRequest, indicesToAutoCreate, indicesThatCannotBeCreated, responses, refs); - rollOverDataStreams(bulkRequest, dataStreamsToBeRolledOver, false, responses, refs); - rollOverDataStreams(bulkRequest, failureStoresToBeRolledOver, true, responses, refs); + createIndices(indicesToAutoCreate, refs, indicesExceptions); + rollOverDataStreams(bulkRequest, dataStreamsToBeRolledOver, false, refs, dataStreamExceptions); + rollOverDataStreams(bulkRequest, failureStoresToBeRolledOver, true, refs, failureStoreExceptions); } } private void createIndices( - BulkRequest bulkRequest, Map indicesToAutoCreate, - Map indicesThatCannotBeCreated, - AtomicArray responses, - RefCountingRunnable refs + RefCountingRunnable refs, + final Map indicesExceptions ) { for (Map.Entry indexEntry : indicesToAutoCreate.entrySet()) { final String index = indexEntry.getKey(); @@ -384,25 +391,26 @@ public void onResponse(CreateIndexResponse createIndexResponse) {} @Override public void onFailure(Exception e) { final Throwable cause = ExceptionsHelper.unwrapCause(e); - if (cause instanceof IndexNotFoundException indexNotFoundException) { - synchronized (indicesThatCannotBeCreated) { - indicesThatCannotBeCreated.put(index, indexNotFoundException); - } - } else if ((cause instanceof ResourceAlreadyExistsException) == false) { + if ((cause instanceof ResourceAlreadyExistsException) == false) { // fail all requests involving this index, if create didn't work - failRequestsWhenPrerequisiteActionFailed(index, bulkRequest, responses, e); + indicesExceptions.put(index, e); } } }, refs.acquire())); } } + // Separate method to allow for overriding in tests. + void createIndex(CreateIndexRequest createIndexRequest, ActionListener listener) { + client.execute(AutoCreateAction.INSTANCE, createIndexRequest, listener); + } + private void rollOverDataStreams( BulkRequest bulkRequest, Set dataStreamsToBeRolledOver, boolean targetFailureStore, - AtomicArray responses, - RefCountingRunnable refs + RefCountingRunnable refs, + Map dataStreamExceptions ) { for (String dataStream : dataStreamsToBeRolledOver) { RolloverRequest rolloverRequest = new RolloverRequest(dataStream, null); @@ -416,7 +424,7 @@ private void rollOverDataStreams( } // We are executing a lazy rollover because it is an action specialised for this situation, when we want an // unconditional and performant rollover. - rolloverClient.execute(LazyRolloverAction.INSTANCE, rolloverRequest, ActionListener.releaseAfter(new ActionListener<>() { + rollOver(rolloverRequest, ActionListener.releaseAfter(new ActionListener<>() { @Override public void onResponse(RolloverResponse result) { @@ -431,26 +439,52 @@ public void onResponse(RolloverResponse result) { @Override public void onFailure(Exception e) { - failRequestsWhenPrerequisiteActionFailed(dataStream, bulkRequest, responses, e); + dataStreamExceptions.put(dataStream, e); } }, refs.acquire())); } } + // Separate method to allow for overriding in tests. + void rollOver(RolloverRequest rolloverRequest, ActionListener listener) { + rolloverClient.execute(LazyRolloverAction.INSTANCE, rolloverRequest, listener); + } + /** - * Fails all requests involving this index or data stream because the prerequisite action failed too. + * Mark all the requests for which the prerequisite action failed (i.e. index creation or data stream/failure store rollover) as failed. */ - private static void failRequestsWhenPrerequisiteActionFailed( - String target, + private void failRequestsWhenPrerequisiteActionFailed( + Map indicesExceptions, + Map dataStreamExceptions, + Map failureStoreExceptions, BulkRequest bulkRequest, - AtomicArray responses, - Exception error + AtomicArray responses ) { + if (indicesExceptions.isEmpty() && dataStreamExceptions.isEmpty() && failureStoreExceptions.isEmpty()) { + return; + } for (int i = 0; i < bulkRequest.requests.size(); i++) { DocWriteRequest request = bulkRequest.requests.get(i); - if (request != null && setResponseFailureIfIndexMatches(responses, i, request, target, error)) { - bulkRequest.requests.set(i, null); + if (request == null) { + continue; } + var exception = indicesExceptions.get(request.index()); + if (exception == null) { + if (request instanceof IndexRequest indexRequest && indexRequest.isWriteToFailureStore()) { + exception = failureStoreExceptions.get(request.index()); + } else { + exception = dataStreamExceptions.get(request.index()); + } + } + if (exception == null) { + continue; + } + var failureStoreStatus = request instanceof IndexRequest ir && ir.isWriteToFailureStore() + ? IndexDocFailureStoreStatus.FAILED + : IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; + var failure = new BulkItemResponse.Failure(request.index(), request.id(), exception, failureStoreStatus); + responses.set(i, BulkItemResponse.failure(i, request.opType(), failure)); + bulkRequest.requests.set(i, null); } } @@ -532,33 +566,13 @@ private static boolean isSystemIndex(SortedMap indices } } - void createIndex(CreateIndexRequest createIndexRequest, ActionListener listener) { - client.execute(AutoCreateAction.INSTANCE, createIndexRequest, listener); - } - - private static boolean setResponseFailureIfIndexMatches( - AtomicArray responses, - int idx, - DocWriteRequest request, - String index, - Exception e - ) { - if (index.equals(request.index())) { - BulkItemResponse.Failure failure = new BulkItemResponse.Failure(request.index(), request.id(), e); - responses.set(idx, BulkItemResponse.failure(idx, request.opType(), failure)); - return true; - } - return false; - } - void executeBulk( Task task, BulkRequest bulkRequest, long startTimeNanos, ActionListener listener, Executor executor, - AtomicArray responses, - Map indicesThatCannotBeCreated + AtomicArray responses ) { new BulkOperation( task, @@ -568,7 +582,6 @@ void executeBulk( bulkRequest, client, responses, - indicesThatCannotBeCreated, indexNameExpressionResolver, relativeTimeNanosProvider, startTimeNanos, diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java index 3be942bcd291e..b87dfd07181dc 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java @@ -42,7 +42,6 @@ import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.MapperException; import org.elasticsearch.index.shard.ShardId; @@ -1022,7 +1021,6 @@ private BulkOperation newBulkOperation(NodeClient client, BulkRequest request, A client, request, new AtomicArray<>(request.numberOfActions()), - Map.of(), mockObserver(DEFAULT_STATE), listener, new FailureStoreDocumentConverter() @@ -1040,7 +1038,6 @@ private BulkOperation newBulkOperation( client, request, new AtomicArray<>(request.numberOfActions()), - Map.of(), mockObserver(DEFAULT_STATE), listener, failureStoreDocumentConverter @@ -1059,7 +1056,6 @@ private BulkOperation newBulkOperation( client, request, new AtomicArray<>(request.numberOfActions()), - Map.of(), observer, listener, new FailureStoreDocumentConverter() @@ -1071,7 +1067,6 @@ private BulkOperation newBulkOperation( NodeClient client, BulkRequest request, AtomicArray existingResponses, - Map indicesThatCanNotBeCreated, ClusterStateObserver observer, ActionListener listener, FailureStoreDocumentConverter failureStoreDocumentConverter @@ -1100,7 +1095,6 @@ private BulkOperation newBulkOperation( request, client, existingResponses, - indicesThatCanNotBeCreated, indexNameExpressionResolver, () -> endTime, timeZero, diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java deleted file mode 100644 index 2f5e6b22e1a8c..0000000000000 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.action.bulk; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.ActionTestUtils; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlocks; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.AtomicArray; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.features.FeatureService; -import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.index.IndexingPressure; -import org.elasticsearch.index.VersionType; -import org.elasticsearch.indices.EmptySystemIndices; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; - -import java.util.Map; -import java.util.Set; -import java.util.concurrent.Executor; -import java.util.function.Consumer; -import java.util.function.Function; - -import static java.util.Collections.emptySet; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class TransportBulkActionIndicesThatCannotBeCreatedTests extends ESTestCase { - private static final Consumer noop = index -> {}; - - public void testNonExceptional() { - BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(new IndexRequest(randomAlphaOfLength(5))); - bulkRequest.add(new IndexRequest(randomAlphaOfLength(5))); - bulkRequest.add(new DeleteRequest(randomAlphaOfLength(5))); - bulkRequest.add(new UpdateRequest(randomAlphaOfLength(5), randomAlphaOfLength(5))); - // Test emulating that index can be auto-created - indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> true, noop); - // Test emulating that index cannot be auto-created - indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> false, noop); - // Test emulating auto_create_index=true with some indices already created. - indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> randomBoolean(), noop); - } - - public void testAllFail() { - BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(new IndexRequest("no")); - bulkRequest.add(new IndexRequest("can't")); - bulkRequest.add(new DeleteRequest("do").version(0).versionType(VersionType.EXTERNAL)); - bulkRequest.add(new UpdateRequest("nothin", randomAlphaOfLength(5))); - indicesThatCannotBeCreatedTestCase(Set.of("no", "can't", "do", "nothin"), bulkRequest, index -> true, index -> { - throw new IndexNotFoundException("Can't make it because I say so"); - }); - } - - public void testSomeFail() { - BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(new IndexRequest("ok")); - bulkRequest.add(new IndexRequest("bad")); - // Emulate auto_create_index=-bad,+* - indicesThatCannotBeCreatedTestCase(Set.of("bad"), bulkRequest, index -> true, index -> { - if (index.equals("bad")) { - throw new IndexNotFoundException("Can't make it because I say so"); - } - }); - } - - private void indicesThatCannotBeCreatedTestCase( - Set expected, - BulkRequest bulkRequest, - Function shouldAutoCreate, - Consumer simulateAutoCreate - ) { - ClusterService clusterService = mock(ClusterService.class); - ClusterState state = mock(ClusterState.class); - when(state.getMetadata()).thenReturn(Metadata.EMPTY_METADATA); - when(state.metadata()).thenReturn(Metadata.EMPTY_METADATA); - when(state.blocks()).thenReturn(mock(ClusterBlocks.class)); - when(clusterService.state()).thenReturn(state); - when(clusterService.getSettings()).thenReturn(Settings.EMPTY); - - DiscoveryNode localNode = mock(DiscoveryNode.class); - when(clusterService.localNode()).thenReturn(localNode); - when(localNode.isIngestNode()).thenReturn(randomBoolean()); - - final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver( - new ThreadContext(Settings.EMPTY), - EmptySystemIndices.INSTANCE - ) { - @Override - public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state) { - return shouldAutoCreate.apply(indexAbstraction) == false; - } - }; - - final ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); - FeatureService mockFeatureService = mock(FeatureService.class); - when(mockFeatureService.clusterHasFeature(any(), any())).thenReturn(true); - TransportBulkAction action = new TransportBulkAction( - threadPool, - transportService, - clusterService, - null, - mockFeatureService, - new NodeClient(Settings.EMPTY, threadPool), - mock(ActionFilters.class), - indexNameExpressionResolver, - new IndexingPressure(Settings.EMPTY), - EmptySystemIndices.INSTANCE, - FailureStoreMetrics.NOOP - ) { - @Override - void executeBulk( - Task task, - BulkRequest bulkRequest, - long startTimeNanos, - ActionListener listener, - Executor executor, - AtomicArray responses, - Map indicesThatCannotBeCreated - ) { - assertEquals(expected, indicesThatCannotBeCreated.keySet()); - } - - @Override - void createIndex(CreateIndexRequest createIndexRequest, ActionListener listener) { - String index = createIndexRequest.index(); - try { - simulateAutoCreate.accept(index); - // If we try to create an index just immediately assume it worked - listener.onResponse(new CreateIndexResponse(true, true, index)); - } catch (Exception e) { - listener.onFailure(e); - } - } - }; - action.doExecute(null, bulkRequest, ActionTestUtils.assertNoFailureListener(bulkItemResponse -> {})); - } -} diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index abdb02924e26c..112748cd73627 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -39,7 +39,6 @@ import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.core.Nullable; import org.elasticsearch.features.FeatureService; -import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexingPressure; @@ -168,8 +167,7 @@ void executeBulk( long startTimeNanos, ActionListener listener, Executor executor, - AtomicArray responses, - Map indicesThatCannotBeCreated + AtomicArray responses ) { assertTrue(indexCreated); isExecuted = true; diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index 5d55e22a080cd..eae6fbf9cbc7f 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -15,6 +15,8 @@ import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.bulk.TransportBulkActionTookTests.Resolver; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; @@ -39,6 +41,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.features.FeatureService; +import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -90,7 +93,9 @@ public class TransportBulkActionTests extends ESTestCase { class TestTransportBulkAction extends TransportBulkAction { - volatile boolean failIndexCreation = false; + volatile Exception failIndexCreationException; + volatile Exception failDataStreamRolloverException; + volatile Exception failFailureStoreRolloverException; boolean indexCreated = false; // set when the "real" index is created Runnable beforeIndexCreation = null; @@ -116,12 +121,25 @@ void createIndex(CreateIndexRequest createIndexRequest, ActionListener listener) { + if (failDataStreamRolloverException != null && rolloverRequest.targetsFailureStore() == false) { + listener.onFailure(failDataStreamRolloverException); + } else if (failFailureStoreRolloverException != null && rolloverRequest.targetsFailureStore()) { + listener.onFailure(failFailureStoreRolloverException); + } else { + listener.onResponse( + new RolloverResponse(null, null, Map.of(), rolloverRequest.isDryRun(), true, true, true, rolloverRequest.isLazy()) + ); + } + } } @Before @@ -357,7 +375,7 @@ public void testRejectCoordination() { public void testRejectionAfterCreateIndexIsPropagated() { BulkRequest bulkRequest = new BulkRequest().add(new IndexRequest("index").id("id").source(Collections.emptyMap())); - bulkAction.failIndexCreation = randomBoolean(); + bulkAction.failIndexCreationException = randomBoolean() ? new ResourceAlreadyExistsException("index already exists") : null; final var blockingLatch = new CountDownLatch(1); try { bulkAction.beforeIndexCreation = () -> blockWriteThreadPool(blockingLatch); @@ -467,6 +485,76 @@ public void testResolveFailureStoreFromTemplate() throws Exception { assertThat(TransportBulkAction.resolveFailureInternal(indexTemplate + "-1", metadata, testTime), is(nullValue())); } + /** + * This test asserts that any failing prerequisite action that fails (i.e. index creation or data stream/failure store rollover) + * results in a failed response. + */ + public void testFailuresDuringPrerequisiteActions() throws InterruptedException { + // One request for testing a failure during index creation. + BulkRequest bulkRequest = new BulkRequest().add(new IndexRequest("index").source(Map.of())) + // One request for testing a failure during data stream rollover. + .add(new IndexRequest("data-stream").source(Map.of())) + // One request for testing a failure during failure store rollover. + .add(new IndexRequest("failure-store").source(Map.of()).setWriteToFailureStore(true)); + + // Construct a cluster state that contains the required data streams. + var state = clusterService.state() + .copyAndUpdateMetadata( + builder -> builder.put(indexMetadata(".ds-data-stream-01")) + .put(indexMetadata(".ds-failure-store-01")) + .put(indexMetadata(".fs-failure-store-01")) + .put( + DataStream.builder( + "data-stream", + DataStream.DataStreamIndices.backingIndicesBuilder(List.of(new Index(".ds-data-stream-01", randomUUID()))) + .setRolloverOnWrite(true) + .build() + ).build() + ) + .put( + DataStream.builder("failure-store", List.of(new Index(".ds-failure-store-01", randomUUID()))) + .setFailureIndices( + DataStream.DataStreamIndices.failureIndicesBuilder(List.of(new Index(".fs-failure-store-01", randomUUID()))) + .setRolloverOnWrite(true) + .build() + ) + .build() + ) + ); + + // Apply the cluster state. + CountDownLatch latch = new CountDownLatch(1); + clusterService.getClusterApplierService().onNewClusterState("set-state", () -> state, ActionListener.running(latch::countDown)); + // And wait for it to be applied. + latch.await(10L, TimeUnit.SECONDS); + + // Set the exceptions that the transport action should encounter. + bulkAction.failIndexCreationException = new IndexNotFoundException("index"); + bulkAction.failDataStreamRolloverException = new RuntimeException("data-stream-rollover-exception"); + bulkAction.failFailureStoreRolloverException = new RuntimeException("failure-store-rollover-exception"); + + // Execute the action and get the response. + PlainActionFuture future = new PlainActionFuture<>(); + ActionTestUtils.execute(bulkAction, null, bulkRequest, future); + BulkResponse response = future.actionGet(); + assertEquals(3, response.getItems().length); + + var indexFailure = response.getItems()[0]; + assertTrue(indexFailure.isFailed()); + assertTrue(indexFailure.getFailure().getCause() instanceof IndexNotFoundException); + assertNull(bulkRequest.requests.get(0)); + + var dataStreamFailure = response.getItems()[1]; + assertTrue(dataStreamFailure.isFailed()); + assertEquals("data-stream-rollover-exception", dataStreamFailure.getFailure().getCause().getMessage()); + assertNull(bulkRequest.requests.get(1)); + + var failureStoreFailure = response.getItems()[2]; + assertTrue(failureStoreFailure.isFailed()); + assertEquals("failure-store-rollover-exception", failureStoreFailure.getFailure().getCause().getMessage()); + assertNull(bulkRequest.requests.get(2)); + } + private BulkRequest buildBulkRequest(List indices) { BulkRequest request = new BulkRequest(); for (String index : indices) { @@ -488,4 +576,8 @@ private BulkRequest buildBulkStreamRequest(List indices) throws IOExcept StreamInput streamInput = out.bytes().streamInput(); return (new BulkRequest(streamInput)); } + + private static IndexMetadata.Builder indexMetadata(String index) { + return IndexMetadata.builder(index).settings(settings(IndexVersion.current())).numberOfShards(1).numberOfReplicas(1); + } } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java index 7f15fd0ec2582..b3d3ebe5e1357 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.indices.EmptySystemIndices; @@ -50,7 +49,6 @@ import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashSet; -import java.util.Map; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -143,11 +141,10 @@ void executeBulk( long startTimeNanos, ActionListener listener, Executor executor, - AtomicArray responses, - Map indicesThatCannotBeCreated + AtomicArray responses ) { expected.set(1000000); - super.executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses, indicesThatCannotBeCreated); + super.executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses); } }; } else { @@ -168,12 +165,11 @@ void executeBulk( long startTimeNanos, ActionListener listener, Executor executor, - AtomicArray responses, - Map indicesThatCannotBeCreated + AtomicArray responses ) { long elapsed = spinForAtLeastOneMillisecond(); expected.set(elapsed); - super.executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses, indicesThatCannotBeCreated); + super.executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses); } }; } From 457d99a7cc98c0804e256215deb09409bf194371 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 26 Sep 2024 08:59:40 -0600 Subject: [PATCH 28/36] Allow configuring `ignore_dynamic_beyond_limit` in Serverless (#113548) This setting is already set for the Serverless templates, however, it's not marked as available on Serverless, so it's ending up ignored. This allows the setting. Co-authored-by: Elastic Machine --- .../java/org/elasticsearch/index/mapper/MapperService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index a7e1b3e122060..08461525526b9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -125,7 +125,8 @@ public boolean isAutoUpdate() { "index.mapping.total_fields.ignore_dynamic_beyond_limit", false, Property.Dynamic, - Property.IndexScope + Property.IndexScope, + Property.ServerlessPublic ); public static final Setting INDEX_MAPPING_DEPTH_LIMIT_SETTING = Setting.longSetting( "index.mapping.depth.limit", From 0ea8a78ca7692c59615be11109ef3412f8690917 Mon Sep 17 00:00:00 2001 From: Stef Nestor <26751266+stefnestor@users.noreply.github.com> Date: Thu, 26 Sep 2024 09:05:21 -0600 Subject: [PATCH 29/36] (Doc+) Avoid search pile up by setting default timeout (#112846) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 👋! Mini doc PR to say can avoid search task pile-ups by setting [`search.default_search_timeout`](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-your-data.html#search-timeout) under [High JVM > avoid expensive searches](https://www.elastic.co/guide/en/elasticsearch/reference/master/high-jvm-memory-pressure.html#reduce-jvm-memory-pressure). --- .../common-issues/high-jvm-memory-pressure.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc b/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc index 267d6594b8025..3469a0ca5bf42 100644 --- a/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc +++ b/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc @@ -66,6 +66,8 @@ searches, consider the following setting changes: <> cluster setting. +* Set a default search timeout using the <> cluster setting. + [source,console] ---- PUT _settings From ade235bf0f032926ab611a17d553160bc2f254a9 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 26 Sep 2024 16:10:39 +0100 Subject: [PATCH 30/36] Wait for logs in tests without busy-waiting (#113589) Introduces `MockLog#awaitAllExpectationsMatched` to allow tests to wait until all the expected log messages have been seen without having to use `assertBusy()`. --- .../bootstrap/SpawnerNoBootstrapTests.java | 74 ++++-------- .../common/network/ThreadWatchdogIT.java | 42 +++---- .../single/SingleNodeDiscoveryIT.java | 2 +- .../indices/cluster/ShardLockFailureIT.java | 16 ++- .../search/fieldcaps/FieldCapabilitiesIT.java | 2 +- .../service/ClusterApplierServiceTests.java | 2 +- .../cluster/service/MasterServiceTests.java | 2 +- .../index/shard/IndexShardTests.java | 6 +- .../monitor/fs/FsHealthServiceTests.java | 2 +- .../tasks/BanFailureLoggingTests.java | 4 +- .../threadpool/ThreadPoolTests.java | 4 +- .../java/org/elasticsearch/test/MockLog.java | 108 +++++++++++++++--- .../AbstractSimpleTransportTestCase.java | 6 +- .../org/elasticsearch/test/MockLogTests.java | 63 +++++++++- .../SnapshotBasedIndexRecoveryIT.java | 2 +- 15 files changed, 224 insertions(+), 111 deletions(-) diff --git a/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java b/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java index 32416b0d0d77d..168493eb52f60 100644 --- a/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java +++ b/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java @@ -13,7 +13,6 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.core.LogEvent; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.util.Constants; import org.elasticsearch.Version; @@ -36,7 +35,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.function.Function; @@ -70,34 +68,6 @@ public class SpawnerNoBootstrapTests extends LuceneTestCase { MockLog.init(); } - static class ExpectedStreamMessage implements MockLog.LoggingExpectation { - final String expectedLogger; - final String expectedMessage; - final CountDownLatch matched; - volatile boolean saw; - - ExpectedStreamMessage(String logger, String message, CountDownLatch matched) { - this.expectedLogger = logger; - this.expectedMessage = message; - this.matched = matched; - } - - @Override - public void match(LogEvent event) { - if (event.getLoggerName().equals(expectedLogger) - && event.getLevel().equals(Level.WARN) - && event.getMessage().getFormattedMessage().equals(expectedMessage)) { - saw = true; - matched.countDown(); - } - } - - @Override - public void assertMatched() { - assertTrue("Expected to see message [" + expectedMessage + "] on logger [" + expectedLogger + "]", saw); - } - } - /** * Simplest case: a module with no controller daemon. */ @@ -209,32 +179,32 @@ private void assertControllerSpawns(final Function pluginsDir String stderrLoggerName = "test_plugin-controller-stderr"; Loggers.setLevel(LogManager.getLogger(stdoutLoggerName), Level.TRACE); Loggers.setLevel(LogManager.getLogger(stderrLoggerName), Level.TRACE); - CountDownLatch messagesLoggedLatch = new CountDownLatch(2); - try (var mockLog = MockLog.capture(stdoutLoggerName, stderrLoggerName)) { - if (expectSpawn) { - mockLog.addExpectation(new ExpectedStreamMessage(stdoutLoggerName, "I am alive", messagesLoggedLatch)); - mockLog.addExpectation(new ExpectedStreamMessage(stderrLoggerName, "I am an error", messagesLoggedLatch)); + if (expectSpawn) { + final Process process; + try (var mockLog = MockLog.capture(stdoutLoggerName, stderrLoggerName)) { + mockLog.addExpectation(new MockLog.SeenEventExpectation("stdout", stdoutLoggerName, Level.WARN, "I am alive")); + mockLog.addExpectation(new MockLog.SeenEventExpectation("stderr", stderrLoggerName, Level.WARN, "I am an error")); + + try (var spawner = new Spawner()) { + spawner.spawnNativeControllers(environment); + List processes = spawner.getProcesses(); + + // as there should only be a reference in the list for the module that had the controller daemon, we expect one here + assertThat(processes, hasSize(1)); + process = processes.get(0); + // fail if we don't get the expected log messages soonish + mockLog.awaitAllExpectationsMatched(); + } } - Spawner spawner = new Spawner(); - spawner.spawnNativeControllers(environment); - - List processes = spawner.getProcesses(); - - if (expectSpawn) { - // as there should only be a reference in the list for the module that had the controller daemon, we expect one here - assertThat(processes, hasSize(1)); - Process process = processes.get(0); - // fail if we don't get the expected log messages within one second; usually it will be even quicker - assertTrue(messagesLoggedLatch.await(1, TimeUnit.SECONDS)); - spawner.close(); - // fail if the process does not die within one second; usually it will be even quicker but it depends on OS scheduling - assertTrue(process.waitFor(1, TimeUnit.SECONDS)); - } else { - assertThat(processes, is(empty())); + // fail if the process does not die within one second; usually it will be even quicker but it depends on OS scheduling + assertTrue(process.waitFor(1, TimeUnit.SECONDS)); + } else { + try (var spawner = new Spawner()) { + spawner.spawnNativeControllers(environment); + assertThat(spawner.getProcesses(), is(empty())); } - mockLog.assertAllExpectationsMatched(); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java b/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java index 6d3ed45f4015a..f2441e43de8d8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java @@ -9,7 +9,7 @@ package org.elasticsearch.common.network; -import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.Level; import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.client.Request; @@ -23,7 +23,6 @@ import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.common.util.concurrent.RunOnce; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; @@ -43,7 +42,6 @@ import java.io.IOException; import java.util.Collection; import java.util.List; -import java.util.concurrent.CountDownLatch; import java.util.function.Predicate; import java.util.function.Supplier; @@ -103,26 +101,24 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c } private static void blockAndWaitForWatchdogLogs() { - final var threadName = Thread.currentThread().getName(); - final var logsSeenLatch = new CountDownLatch(2); - final var warningSeen = new RunOnce(logsSeenLatch::countDown); - final var threadDumpSeen = new RunOnce(logsSeenLatch::countDown); - MockLog.assertThatLogger(() -> safeAwait(logsSeenLatch), ThreadWatchdog.class, new MockLog.LoggingExpectation() { - @Override - public void match(LogEvent event) { - final var formattedMessage = event.getMessage().getFormattedMessage(); - if (formattedMessage.contains("the following threads are active but did not make progress in the preceding [100ms]:") - && formattedMessage.contains(threadName)) { - warningSeen.run(); - } - if (formattedMessage.contains("hot threads dump due to active threads not making progress")) { - threadDumpSeen.run(); - } - } - - @Override - public void assertMatched() {} - }); + MockLog.awaitLogger( + () -> {}, + ThreadWatchdog.class, + new MockLog.SeenEventExpectation( + "warning", + ThreadWatchdog.class.getCanonicalName(), + Level.WARN, + "*the following threads are active but did not make progress in the preceding [100ms]:*" + + Thread.currentThread().getName() + + "*" + ), + new MockLog.SeenEventExpectation( + "thread dump", + ThreadWatchdog.class.getCanonicalName(), + Level.WARN, + "*hot threads dump due to active threads not making progress*" + ) + ); } public void testThreadWatchdogHttpLogging() throws IOException { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java index 8fd10cdf07310..10f13f6ab152f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java @@ -160,7 +160,7 @@ public boolean innerMatch(final LogEvent event) { other.beforeTest(random()); final ClusterState first = internalCluster().getInstance(ClusterService.class).state(); assertThat(first.nodes().getSize(), equalTo(1)); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); } finally { other.close(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ShardLockFailureIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ShardLockFailureIT.java index 9bcd8528acf9e..0ce3ca53e1c1f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ShardLockFailureIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ShardLockFailureIT.java @@ -72,9 +72,9 @@ public void testShardLockFailure() throws Exception { var ignored1 = internalCluster().getInstance(NodeEnvironment.class, node).shardLock(shardId, "blocked for test"); var mockLog = MockLog.capture(IndicesClusterStateService.class); ) { - final CountDownLatch countDownLatch = new CountDownLatch(1); mockLog.addExpectation(new MockLog.LoggingExpectation() { + private final CountDownLatch countDownLatch = new CountDownLatch(1); int debugMessagesSeen = 0; int warnMessagesSeen = 0; @@ -101,14 +101,20 @@ public synchronized void match(LogEvent event) { } @Override - public void assertMatched() {} + public void assertMatched() { + fail("unused"); + } + + @Override + public void awaitMatched(long millis) throws InterruptedException { + assertTrue(countDownLatch.await(millis, TimeUnit.MILLISECONDS)); + } }); updateIndexSettings(Settings.builder().putNull(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + "._name"), indexName); ensureYellow(indexName); - assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); + mockLog.awaitAllExpectationsMatched(); assertEquals(ClusterHealthStatus.YELLOW, clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT, indexName).get().getStatus()); - mockLog.assertAllExpectationsMatched(); } ensureGreen(indexName); @@ -153,7 +159,7 @@ public void testShardLockTimeout() throws Exception { ); updateIndexSettings(Settings.builder().putNull(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + "._name"), indexName); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); final var clusterHealthResponse = clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT, indexName) .setWaitForEvents(Priority.LANGUID) .setTimeout(TimeValue.timeValueSeconds(10)) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index ac68ff243166b..a754350c8faf7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -693,7 +693,7 @@ public void testCancel() throws Exception { } }, 30, TimeUnit.SECONDS); cancellable.cancel(); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); logger.info("--> waiting for field-caps tasks to be cancelled"); assertBusy(() -> { List tasks = clusterAdmin().prepareListTasks() diff --git a/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java index e37e92d553dcc..7c1c954e7b4e9 100644 --- a/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java @@ -181,7 +181,7 @@ public void onFailure(Exception e) { fail(); } }); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java index 498c04c005304..a540649582177 100644 --- a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java @@ -526,7 +526,7 @@ public void onFailure(Exception e) { fail(); } }); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); } } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index ec70f1f7adcfd..cddda8a76ae60 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -4191,7 +4191,7 @@ protected void commitIndexWriter(final IndexWriter writer, final Translog transl ); shard.flushOnIdle(0); assertFalse(shard.isActive()); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); // While the first flush is happening, index one more doc (to turn the shard's active flag to true), // and issue a second flushOnIdle request which should not wait for the ongoing flush @@ -4206,7 +4206,7 @@ protected void commitIndexWriter(final IndexWriter writer, final Translog transl ) ); shard.flushOnIdle(0); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); // A direct call to flush (with waitIfOngoing=false) should not wait and return false immediately assertFalse(shard.flush(new FlushRequest().waitIfOngoing(false).force(false))); @@ -4223,7 +4223,7 @@ protected void commitIndexWriter(final IndexWriter writer, final Translog transl "released flush lock" ) ); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); // The second flushOnIdle (that did not happen) should have turned the active flag to true assertTrue(shard.isActive()); diff --git a/server/src/test/java/org/elasticsearch/monitor/fs/FsHealthServiceTests.java b/server/src/test/java/org/elasticsearch/monitor/fs/FsHealthServiceTests.java index 05b71693a7fea..b644dfbc3a12c 100644 --- a/server/src/test/java/org/elasticsearch/monitor/fs/FsHealthServiceTests.java +++ b/server/src/test/java/org/elasticsearch/monitor/fs/FsHealthServiceTests.java @@ -146,7 +146,7 @@ public void testLoggingOnHungIO() throws Exception { disruptFileSystemProvider.injectIOException.set(true); fsHealthService.new FsHealthMonitor().run(); assertEquals(env.nodeDataPaths().length, disruptFileSystemProvider.getInjectedPathCount()); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); } finally { PathUtilsForTesting.teardown(); ThreadPool.terminate(testThreadPool, 500, TimeUnit.MILLISECONDS); diff --git a/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java b/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java index 276099e01c9f8..78d76476d06fc 100644 --- a/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java +++ b/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java @@ -184,8 +184,8 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, // acceptable; we mostly ignore the result of cancellation anyway } - // assert busy since failure to remove a ban may be logged after cancellation completed - assertBusy(mockLog::assertAllExpectationsMatched); + // await since failure to remove a ban may be logged after cancellation completed + mockLog.awaitAllExpectationsMatched(); } assertTrue("child tasks did not finish in time", childTaskLock.tryLock(15, TimeUnit.SECONDS)); diff --git a/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java b/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java index 458eeb900071d..310cf467a8391 100644 --- a/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java +++ b/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java @@ -128,7 +128,7 @@ public void testTimerThreadWarningLogging() throws Exception { final ThreadPool.CachedTimeThread thread = new ThreadPool.CachedTimeThread("[timer]", 200, 100); thread.start(); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); thread.interrupt(); thread.join(); @@ -297,7 +297,7 @@ public String toString() { } }; threadPool.schedule(runnable, TimeValue.timeValueMillis(randomLongBetween(0, 300)), EsExecutors.DIRECT_EXECUTOR_SERVICE); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); } finally { assertTrue(terminate(threadPool)); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockLog.java b/test/framework/src/main/java/org/elasticsearch/test/MockLog.java index 57acca08c23e8..4a012bb361e65 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/MockLog.java +++ b/test/framework/src/main/java/org/elasticsearch/test/MockLog.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.TimeValue; import java.util.Arrays; import java.util.List; @@ -23,10 +24,13 @@ import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; /** @@ -112,10 +116,46 @@ public void assertAllExpectationsMatched() { } } + public void awaitAllExpectationsMatched() { + awaitAllExpectationsMatched(ESTestCase.SAFE_AWAIT_TIMEOUT); + } + + // exposed for testing + void awaitAllExpectationsMatched(TimeValue waitTime) { + final var deadlineNanos = System.nanoTime() + waitTime.nanos(); + final var nanosPerMilli = TimeValue.timeValueMillis(1).nanos(); + try { + for (LoggingExpectation expectation : expectations) { + final var remainingMillis = (deadlineNanos - System.nanoTime() + nanosPerMilli - 1) / nanosPerMilli; // round up + assertThat(remainingMillis, greaterThan(0L)); + expectation.awaitMatched(remainingMillis); + } + } catch (InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + throw new AssertionError("interrupted", interruptedException); + } + } + + /** + * Keeps track of whether the {@link LogEvent} instances it receives match the expected content. + */ public interface LoggingExpectation { + /** + * Called on every {@link LogEvent} received by the captured appenders. + */ void match(LogEvent event); + /** + * Returns if this expectation is matched, otherwise throws an {@link AssertionError}. + */ void assertMatched(); + + /** + * Returns if this expectation is matched within the given number of milliseconds, otherwise throws an {@link AssertionError}. + */ + default void awaitMatched(long millis) throws InterruptedException { + assertMatched(); + } } public abstract static class AbstractEventExpectation implements LoggingExpectation { @@ -123,14 +163,13 @@ public abstract static class AbstractEventExpectation implements LoggingExpectat protected final String logger; protected final Level level; protected final String message; - volatile boolean saw; + protected final CountDownLatch seenLatch = new CountDownLatch(1); public AbstractEventExpectation(String name, String logger, Level level, String message) { this.name = name; this.logger = logger; this.level = level; this.message = message; - this.saw = false; } @Override @@ -138,11 +177,11 @@ public void match(LogEvent event) { if (event.getLevel().equals(level) && event.getLoggerName().equals(logger) && innerMatch(event)) { if (Regex.isSimpleMatchPattern(message)) { if (Regex.simpleMatch(message, event.getMessage().getFormattedMessage())) { - saw = true; + seenLatch.countDown(); } } else { if (event.getMessage().getFormattedMessage().contains(message)) { - saw = true; + seenLatch.countDown(); } } } @@ -162,7 +201,7 @@ public UnseenEventExpectation(String name, String logger, Level level, String me @Override public void assertMatched() { - assertThat("expected not to see " + name + " but did", saw, equalTo(false)); + assertThat("expected not to see " + name + " but did", seenLatch.getCount(), equalTo(1L)); } } @@ -174,7 +213,12 @@ public SeenEventExpectation(String name, String logger, Level level, String mess @Override public void assertMatched() { - assertThat("expected to see " + name + " but did not", saw, equalTo(true)); + assertThat("expected to see " + name + " but did not", seenLatch.getCount(), equalTo(0L)); + } + + @Override + public void awaitMatched(long millis) throws InterruptedException { + assertThat("expected to see " + name + " but did not", seenLatch.await(millis, TimeUnit.MILLISECONDS), equalTo(true)); } } @@ -195,7 +239,17 @@ public void assertMatched() { if (expectSeen) { super.assertMatched(); } else { - assertThat("expected not to see " + name + " yet but did", saw, equalTo(false)); + assertThat("expected not to see " + name + " yet but did", seenLatch.getCount(), equalTo(1L)); + } + } + + @Override + public void awaitMatched(long millis) throws InterruptedException { + if (expectSeen) { + super.awaitMatched(millis); + } else { + // do not wait for negative expectation + assertThat("expected not to see " + name + " yet but did", seenLatch.getCount(), equalTo(1L)); } } } @@ -229,11 +283,11 @@ public boolean innerMatch(final LogEvent event) { public static class PatternSeenEventExpectation implements LoggingExpectation { - protected final String name; - protected final String logger; - protected final Level level; - protected final Pattern pattern; - volatile boolean saw; + private final String name; + private final String logger; + private final Level level; + private final Pattern pattern; + private final CountDownLatch seenLatch = new CountDownLatch(1); public PatternSeenEventExpectation(String name, String logger, Level level, String pattern) { this.name = name; @@ -246,16 +300,20 @@ public PatternSeenEventExpectation(String name, String logger, Level level, Stri public void match(LogEvent event) { if (event.getLevel().equals(level) && event.getLoggerName().equals(logger)) { if (pattern.matcher(event.getMessage().getFormattedMessage()).matches()) { - saw = true; + seenLatch.countDown(); } } } @Override public void assertMatched() { - assertThat(name, saw, equalTo(true)); + assertThat(name, seenLatch.getCount(), equalTo(0L)); } + @Override + public void awaitMatched(long millis) throws InterruptedException { + assertThat(name, seenLatch.await(millis, TimeUnit.MILLISECONDS), equalTo(true)); + } } /** @@ -284,6 +342,15 @@ public void assertMatched() { } } + @Override + public void awaitMatched(long millis) throws InterruptedException { + try { + delegate.awaitMatched(millis); + } finally { + assertMatchedCalled = true; + } + } + @Override public String toString() { return delegate.toString(); @@ -336,4 +403,17 @@ public static void assertThatLogger(Runnable action, Class loggerOwner, MockL mockLog.assertAllExpectationsMatched(); } } + + /** + * Executes an action and waits until the given logging expectations are satisfied. + */ + public static void awaitLogger(Runnable action, Class loggerOwner, MockLog.LoggingExpectation... expectations) { + try (var mockLog = MockLog.capture(loggerOwner)) { + for (var expectation : expectations) { + mockLog.addExpectation(expectation); + } + action.run(); + mockLog.awaitAllExpectationsMatched(); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java index ba7aa9977b917..fe3de2218a493 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java @@ -1373,7 +1373,7 @@ public void handleException(TransportException exp) {} serviceA.sendRequest(nodeB, "internal:test", new StringMessageRequest("", 10), noopResponseHandler); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); //////////////////////////////////////////////////////////////////////// // tests for included action type "internal:testError" which returns an error @@ -1420,7 +1420,7 @@ public void handleException(TransportException exp) {} serviceA.sendRequest(nodeB, "internal:testError", new StringMessageRequest(""), noopResponseHandler); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); //////////////////////////////////////////////////////////////////////// // tests for excluded action type "internal:testNotSeen" @@ -1467,7 +1467,7 @@ public void handleException(TransportException exp) {} submitRequest(serviceA, nodeB, "internal:testNotSeen", new StringMessageRequest(""), noopResponseHandler).get(); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/MockLogTests.java b/test/framework/src/test/java/org/elasticsearch/test/MockLogTests.java index 2720da55544d1..89d23c3b345b9 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/MockLogTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/MockLogTests.java @@ -9,8 +9,11 @@ package org.elasticsearch.test; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.junit.annotations.TestLogging; import java.util.concurrent.atomic.AtomicBoolean; @@ -27,7 +30,7 @@ public void testConcurrentLogAndLifecycle() throws Exception { logThread.start(); for (int i = 0; i < 1000; i++) { - try (var mockLog = MockLog.capture(MockLogTests.class)) { + try (var ignored = MockLog.capture(MockLogTests.class)) { Thread.yield(); } } @@ -35,4 +38,62 @@ public void testConcurrentLogAndLifecycle() throws Exception { keepGoing.set(false); logThread.join(); } + + @TestLogging(reason = "checking log behaviour", value = "org.elasticsearch.test.MockLogTests:INFO") + public void testAwaitUnseenEvent() { + try (var mockLog = MockLog.capture(MockLogTests.class)) { + mockLog.addExpectation( + new MockLog.UnseenEventExpectation("unseen", MockLogTests.class.getCanonicalName(), Level.INFO, "unexpected") + ); + Thread.currentThread().interrupt(); // ensures no blocking calls + mockLog.awaitAllExpectationsMatched(); + mockLog.assertAllExpectationsMatched(); + + logger.info("unexpected"); + expectThrows(AssertionError.class, mockLog::awaitAllExpectationsMatched); + expectThrows(AssertionError.class, mockLog::assertAllExpectationsMatched); + + assertTrue(Thread.interrupted()); // clear interrupt flag again + } + } + + @TestLogging(reason = "checking log behaviour", value = "org.elasticsearch.test.MockLogTests:INFO") + public void testAwaitSeenEvent() throws InterruptedException { + try (var mockLog = MockLog.capture(MockLogTests.class)) { + mockLog.addExpectation(new MockLog.SeenEventExpectation("seen", MockLogTests.class.getCanonicalName(), Level.INFO, "expected")); + + expectThrows(AssertionError.class, () -> mockLog.awaitAllExpectationsMatched(TimeValue.timeValueMillis(10))); + expectThrows(AssertionError.class, mockLog::assertAllExpectationsMatched); + + final var logThread = new Thread(() -> { + logger.info("expected"); + mockLog.assertAllExpectationsMatched(); + }); + logThread.start(); + mockLog.awaitAllExpectationsMatched(); + mockLog.assertAllExpectationsMatched(); + logThread.join(); + } + } + + @TestLogging(reason = "checking log behaviour", value = "org.elasticsearch.test.MockLogTests:INFO") + public void testAwaitPatternEvent() throws InterruptedException { + try (var mockLog = MockLog.capture(MockLogTests.class)) { + mockLog.addExpectation( + new MockLog.PatternSeenEventExpectation("seen", MockLogTests.class.getCanonicalName(), Level.INFO, ".*expected.*") + ); + + expectThrows(AssertionError.class, () -> mockLog.awaitAllExpectationsMatched(TimeValue.timeValueMillis(10))); + expectThrows(AssertionError.class, mockLog::assertAllExpectationsMatched); + + final var logThread = new Thread(() -> { + logger.info("blah blah expected blah blah"); + mockLog.assertAllExpectationsMatched(); + }); + logThread.start(); + mockLog.awaitAllExpectationsMatched(); + mockLog.assertAllExpectationsMatched(); + logThread.join(); + } + } } diff --git a/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java b/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java index d1eaff1bef1b2..df8dc54bb7490 100644 --- a/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java +++ b/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java @@ -623,7 +623,7 @@ public void testRecoveryIsCancelledAfterDeletingTheIndex() throws Exception { assertAcked(indicesAdmin().prepareDelete(indexName).get()); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); } respondToRecoverSnapshotFile.countDown(); From 122e7288200ee03e9087c98dff6cebbc94e774aa Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Thu, 26 Sep 2024 12:03:01 -0400 Subject: [PATCH 31/36] [ESQL] Add TO_DATE_NANOS conversion function (#112150) Resolves #111842 This adds a conversion function that yields DATE_NANOS. Mostly this is straight forward. It is worth noting that when converting a millisecond date into a nanosecond date, the conversion function truncates it to 0 nanoseconds (i.e. first nanosecond of that millisecond). This is, of course, a bit of an assumption, but I don't have a better assumption we can make. I'd thought about adding a second, optional, parameter to control this behavior, but it's important that TO_DATE_NANOS extend AbstractConvertFunction, which itself extends UnaryScalarFunction, so that it will work correctly with union types. Also, it's unlikely the user will have any better guess than we do for filling in the nanoseconds. Making that assumption does, however, create some weirdness. Consider two comparisons: TO_DATETIME("2023-03-23T12:15:03.360103847") == TO_DATETIME("2023-03-23T12:15:03.360") will return true while TO_DATE_NANOS("2023-03-23T12:15:03.360103847") == TO_DATE_NANOS("2023-03-23T12:15:03.360") will return false. This is akin to casting between longs and doubles, where things may compare equal in one type that are not equal in the other. This seems fine, and I can't think of a better way to do it, but it's worth being aware of. --------- Co-authored-by: Elastic Machine --- .../description/to_date_nanos.asciidoc | 7 + .../kibana/definition/to_date_nanos.json | 9 + .../esql/functions/kibana/docs/mv_avg.md | 2 +- .../esql/functions/kibana/docs/mv_sum.md | 2 +- .../functions/kibana/docs/to_date_nanos.md | 8 + .../esql/functions/kibana/inline_cast.json | 1 + .../functions/layout/to_date_nanos.asciidoc | 16 ++ .../parameters/to_date_nanos.asciidoc | 6 + .../esql/functions/signature/categorize.svg | 2 +- .../esql/functions/signature/qstr.svg | 2 +- .../functions/signature/to_date_nanos.svg | 1 + .../functions/types/to_date_nanos.asciidoc | 9 + .../src/main/resources/date_nanos.csv | 18 +- .../src/main/resources/date_nanos.csv-spec | 193 ++++++++++++++++++ .../main/resources/mapping-date_nanos.json | 3 + .../src/main/resources/meta.csv-spec | 10 +- .../ToDateNanosFromDatetimeEvaluator.java | 122 +++++++++++ .../ToDateNanosFromDoubleEvaluator.java | 124 +++++++++++ .../convert/ToDateNanosFromLongEvaluator.java | 122 +++++++++++ .../ToDateNanosFromStringEvaluator.java | 126 ++++++++++++ .../xpack/esql/action/EsqlCapabilities.java | 5 + .../function/EsqlFunctionRegistry.java | 2 + .../function/scalar/UnaryScalarFunction.java | 2 + .../function/scalar/convert/ToDateNanos.java | 134 ++++++++++++ .../esql/type/EsqlDataTypeConverter.java | 6 +- .../expression/function/TestCaseSupplier.java | 83 ++++++-- .../scalar/convert/ToDateNanosTests.java | 135 ++++++++++++ 27 files changed, 1115 insertions(+), 35 deletions(-) create mode 100644 docs/reference/esql/functions/description/to_date_nanos.asciidoc create mode 100644 docs/reference/esql/functions/kibana/definition/to_date_nanos.json create mode 100644 docs/reference/esql/functions/kibana/docs/to_date_nanos.md create mode 100644 docs/reference/esql/functions/layout/to_date_nanos.asciidoc create mode 100644 docs/reference/esql/functions/parameters/to_date_nanos.asciidoc create mode 100644 docs/reference/esql/functions/signature/to_date_nanos.svg create mode 100644 docs/reference/esql/functions/types/to_date_nanos.asciidoc create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDatetimeEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDoubleEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromLongEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromStringEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java diff --git a/docs/reference/esql/functions/description/to_date_nanos.asciidoc b/docs/reference/esql/functions/description/to_date_nanos.asciidoc new file mode 100644 index 0000000000000..3fac7295f1bed --- /dev/null +++ b/docs/reference/esql/functions/description/to_date_nanos.asciidoc @@ -0,0 +1,7 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Converts an input to a nanosecond-resolution date value (aka date_nanos). + +NOTE: The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z. Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch. diff --git a/docs/reference/esql/functions/kibana/definition/to_date_nanos.json b/docs/reference/esql/functions/kibana/definition/to_date_nanos.json new file mode 100644 index 0000000000000..bafbcf2bc2038 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/to_date_nanos.json @@ -0,0 +1,9 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "to_date_nanos", + "description" : "Converts an input to a nanosecond-resolution date value (aka date_nanos).", + "note" : "The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z. Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch.", + "signatures" : [ ], + "preview" : true +} diff --git a/docs/reference/esql/functions/kibana/docs/mv_avg.md b/docs/reference/esql/functions/kibana/docs/mv_avg.md index c5163f36129bf..c3d7e5423f724 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_avg.md +++ b/docs/reference/esql/functions/kibana/docs/mv_avg.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### MV_AVG -Converts a multivalued field into a single valued field containing the average of all the values. +Converts a multivalued field into a single valued field containing the average of all of the values. ``` ROW a=[3, 5, 1, 6] diff --git a/docs/reference/esql/functions/kibana/docs/mv_sum.md b/docs/reference/esql/functions/kibana/docs/mv_sum.md index 987017b34b743..16285d3c7229b 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_sum.md +++ b/docs/reference/esql/functions/kibana/docs/mv_sum.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### MV_SUM -Converts a multivalued field into a single valued field containing the sum of all the values. +Converts a multivalued field into a single valued field containing the sum of all of the values. ``` ROW a=[3, 5, 6] diff --git a/docs/reference/esql/functions/kibana/docs/to_date_nanos.md b/docs/reference/esql/functions/kibana/docs/to_date_nanos.md new file mode 100644 index 0000000000000..0294802485ccb --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/to_date_nanos.md @@ -0,0 +1,8 @@ + + +### TO_DATE_NANOS +Converts an input to a nanosecond-resolution date value (aka date_nanos). + +Note: The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z. Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch. diff --git a/docs/reference/esql/functions/kibana/inline_cast.json b/docs/reference/esql/functions/kibana/inline_cast.json index f1aa283c52e95..81a1966773238 100644 --- a/docs/reference/esql/functions/kibana/inline_cast.json +++ b/docs/reference/esql/functions/kibana/inline_cast.json @@ -3,6 +3,7 @@ "boolean" : "to_boolean", "cartesian_point" : "to_cartesianpoint", "cartesian_shape" : "to_cartesianshape", + "date_nanos" : "to_date_nanos", "date_period" : "to_dateperiod", "datetime" : "to_datetime", "double" : "to_double", diff --git a/docs/reference/esql/functions/layout/to_date_nanos.asciidoc b/docs/reference/esql/functions/layout/to_date_nanos.asciidoc new file mode 100644 index 0000000000000..977a0ac969e5d --- /dev/null +++ b/docs/reference/esql/functions/layout/to_date_nanos.asciidoc @@ -0,0 +1,16 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-to_date_nanos]] +=== `TO_DATE_NANOS` + +preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] + +*Syntax* + +[.text-center] +image::esql/functions/signature/to_date_nanos.svg[Embedded,opts=inline] + +include::../parameters/to_date_nanos.asciidoc[] +include::../description/to_date_nanos.asciidoc[] +include::../types/to_date_nanos.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/to_date_nanos.asciidoc b/docs/reference/esql/functions/parameters/to_date_nanos.asciidoc new file mode 100644 index 0000000000000..224f474fa64e3 --- /dev/null +++ b/docs/reference/esql/functions/parameters/to_date_nanos.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`field`:: +Input value. The input can be a single- or multi-valued column or an expression. diff --git a/docs/reference/esql/functions/signature/categorize.svg b/docs/reference/esql/functions/signature/categorize.svg index 3f36f10382840..c52fd1763eea1 100644 --- a/docs/reference/esql/functions/signature/categorize.svg +++ b/docs/reference/esql/functions/signature/categorize.svg @@ -1 +1 @@ -CATEGORIZE(field) +CATEGORIZE(field) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/qstr.svg b/docs/reference/esql/functions/signature/qstr.svg index 0d3841b071cef..fb6114822ae63 100644 --- a/docs/reference/esql/functions/signature/qstr.svg +++ b/docs/reference/esql/functions/signature/qstr.svg @@ -1 +1 @@ -QSTR(query) +QSTR(query) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/to_date_nanos.svg b/docs/reference/esql/functions/signature/to_date_nanos.svg new file mode 100644 index 0000000000000..0b24b56429588 --- /dev/null +++ b/docs/reference/esql/functions/signature/to_date_nanos.svg @@ -0,0 +1 @@ +TO_DATE_NANOS(field) \ No newline at end of file diff --git a/docs/reference/esql/functions/types/to_date_nanos.asciidoc b/docs/reference/esql/functions/types/to_date_nanos.asciidoc new file mode 100644 index 0000000000000..1f50b65f25a77 --- /dev/null +++ b/docs/reference/esql/functions/types/to_date_nanos.asciidoc @@ -0,0 +1,9 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +field | result +date_nanos +|=== diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv index 7e857f5243f58..83a2f3cb1c281 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv @@ -1,9 +1,9 @@ -millis:date,nanos:date_nanos -2023-10-23T13:55:01.543Z,2023-10-23T13:55:01.543123456Z -2023-10-23T13:53:55.832Z,2023-10-23T13:53:55.832987654Z -2023-10-23T13:52:55.015Z,2023-10-23T13:52:55.015787878Z -2023-10-23T13:51:54.732Z,2023-10-23T13:51:54.732102837Z -2023-10-23T13:33:34.937Z,2023-10-23T13:33:34.937193000Z -2023-10-23T12:27:28.948Z,2023-10-23T12:27:28.948000000Z -2023-10-23T12:15:03.360Z,2023-10-23T12:15:03.360103847Z -1999-10-23T12:15:03.360Z,[2023-03-23T12:15:03.360103847Z, 2023-02-23T13:33:34.937193000Z, 2023-01-23T13:55:01.543123456Z] +millis:date,nanos:date_nanos,num:long +2023-10-23T13:55:01.543Z,2023-10-23T13:55:01.543123456Z,1698069301543123456 +2023-10-23T13:53:55.832Z,2023-10-23T13:53:55.832987654Z,1698069235832987654 +2023-10-23T13:52:55.015Z,2023-10-23T13:52:55.015787878Z,1698069175015787878 +2023-10-23T13:51:54.732Z,2023-10-23T13:51:54.732102837Z,1698069114732102837 +2023-10-23T13:33:34.937Z,2023-10-23T13:33:34.937193000Z,1698068014937193000 +2023-10-23T12:27:28.948Z,2023-10-23T12:27:28.948000000Z,1698064048948000000 +2023-10-23T12:15:03.360Z,2023-10-23T12:15:03.360103847Z,1698063303360103847 +1999-10-23T12:15:03.360Z,[2023-03-23T12:15:03.360103847Z, 2023-02-23T13:33:34.937193000Z, 2023-01-23T13:55:01.543123456Z], 0 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec index b77689e1b5768..883010eb484db 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec @@ -70,3 +70,196 @@ FROM date_nanos | SORT millis asc | EVAL nanos = MV_LAST(nanos) | KEEP nanos | L nanos:date_nanos 2023-03-23T12:15:03.360103847Z ; + +string to date nanos +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS("2023-03-23T12:15:03.360103847"); + +d:date_nanos +2023-03-23T12:15:03.360103847Z +; + +string to date nanos, :: notation +required_capability: to_date_nanos + +ROW d = "2023-03-23T12:15:03.360103847"::date_nanos; + +d:date_nanos +2023-03-23T12:15:03.360103847Z +; + +string to date nanos, milliseconds only +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS("2023-03-23T12:15:03.360"); + +d:date_nanos +2023-03-23T12:15:03.360Z +; + +string to date nanos, out of range +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS("2262-04-12T00:00:00.000"); +warning:Line 1:9: evaluation of [TO_DATE_NANOS(\"2262-04-12T00:00:00.000\")] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:9: java.lang.IllegalArgumentException: date[2262-04-12T00:00:00Z] is after 2262-04-11T23:47:16.854775807 and cannot be stored in nanosecond resolution + +d:date_nanos +null +; + +string to date nanos, pre 1970 +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS("1969-04-12T00:00:00.000"); +warning:Line 1:9: evaluation of [TO_DATE_NANOS(\"1969-04-12T00:00:00.000\")] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:9: java.lang.IllegalArgumentException: date[1969-04-12T00:00:00Z] is before the epoch in 1970 and cannot be stored in nanosecond resolution + +d:date_nanos +null +; + +long to date nanos +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(1724160894123456789); + +d:date_nanos +2024-08-20T13:34:54.123456789Z +; + +long to date nanos, :: notation +required_capability: to_date_nanos + +ROW d = 1724160894123456789::date_nanos; + +d:date_nanos +2024-08-20T13:34:54.123456789Z +; + + +long to date nanos, before 1970 +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(TO_LONG(-1)); + +warning:Line 1:9: evaluation of [TO_DATE_NANOS(TO_LONG(-1))] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:9: java.lang.IllegalArgumentException: Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported. +d:date_nanos +null +; + +unsigned long to date nanos +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(TO_UNSIGNED_LONG(1724160894123456789)); + +d:date_nanos +2024-08-20T13:34:54.123456789Z +; + +double to date nanos +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(1724160894123456789.0); + +d:date_nanos +# Note we've lost some precision here +2024-08-20T13:34:54.123456768Z +; + +datetime to date nanos, in range +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(TO_DATETIME("2024-08-20T13:34:54.123Z")); + +d:date_nanos +2024-08-20T13:34:54.123000000Z +; + +datetime to date nanos, with overflow +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(TO_DATETIME("2262-04-12T00:00:00.000")); +warning:Line 1:9: evaluation of [TO_DATE_NANOS(TO_DATETIME(\"2262-04-12T00:00:00.000\"))] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:9: java.lang.IllegalArgumentException: milliSeconds [9223372800000] are after 2262-04-11T23:47:16.854775807 and cannot be converted to nanoseconds + +d:date_nanos +null +; + +datetime to date nanos, pre 1970 +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(TO_DATETIME("1969-04-12T00:00:00.000")); +warning:Line 1:9: evaluation of [TO_DATE_NANOS(TO_DATETIME(\"1969-04-12T00:00:00.000\"))] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:9: java.lang.IllegalArgumentException: milliSeconds [-22809600000] are before the epoch in 1970 and cannot be converted to nanoseconds + +d:date_nanos +null +; + + +date nanos to long, index version +required_capability: to_date_nanos + +FROM date_nanos | WHERE millis > "2020-02-02" | EVAL l = TO_LONG(nanos) | KEEP l; + +l:long +1698069301543123456 +1698069235832987654 +1698069175015787878 +1698069114732102837 +1698068014937193000 +1698064048948000000 +1698063303360103847 +; + +long to date nanos, index version +required_capability: to_date_nanos + +FROM date_nanos | WHERE millis > "2020-02-02" | EVAL d = TO_DATE_NANOS(num) | KEEP d; + +d:date_nanos +2023-10-23T13:55:01.543123456Z +2023-10-23T13:53:55.832987654Z +2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937193000Z +2023-10-23T12:27:28.948000000Z +2023-10-23T12:15:03.360103847Z +; + +date_nanos to date nanos, index version +required_capability: to_date_nanos + +FROM date_nanos | WHERE millis > "2020-02-02" | EVAL d = TO_DATE_NANOS(nanos) | KEEP d; + +d:date_nanos +2023-10-23T13:55:01.543123456Z +2023-10-23T13:53:55.832987654Z +2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937193000Z +2023-10-23T12:27:28.948000000Z +2023-10-23T12:15:03.360103847Z +; + +attempt to cast the result of a fold to date nanos +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(CONCAT("2023-01-01","T12:12:12")); + +d:date_nanos +2023-01-01T12:12:12.000000000Z +; + +attempt to cast nulls to date nanos +required_capability: to_date_nanos + +ROW a = TO_DATE_NANOS(null), b = TO_DATE_NANOS(null + 1::long), c = TO_DATE_NANOS(CONCAT("2024", null)); + +a:date_nanos | b:date_nanos | c:date_nanos +null | null | null +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos.json index 506290d90b4b0..a07f9eeeca7b8 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos.json +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos.json @@ -5,6 +5,9 @@ }, "nanos": { "type": "date_nanos" + }, + "num": { + "type": "long" } } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec index 2b3fa9dec797d..13c3857a5c497 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec @@ -97,6 +97,8 @@ double tau() "boolean to_boolean(field:boolean|keyword|text|double|long|unsigned_long|integer)" "cartesian_point to_cartesianpoint(field:cartesian_point|keyword|text)" "cartesian_shape to_cartesianshape(field:cartesian_point|cartesian_shape|keyword|text)" +"date_nanos to_date_nanos(field:date|date_nanos|keyword|text|double|long|unsigned_long)" +"date_nanos to_datenanos(field:date|date_nanos|keyword|text|double|long|unsigned_long)" "date_period to_dateperiod(field:date_period|keyword|text)" "date to_datetime(field:date|date_nanos|keyword|text|double|long|unsigned_long|integer)" "double to_dbl(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long)" @@ -227,6 +229,8 @@ to_bool |field |"boolean|keyword|text|double to_boolean |field |"boolean|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. to_cartesianpo|field |"cartesian_point|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. to_cartesiansh|field |"cartesian_point|cartesian_shape|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. +to_date_nanos |field |"date|date_nanos|keyword|text|double|long|unsigned_long" |Input value. The input can be a single- or multi-valued column or an expression. +to_datenanos |field |"date|date_nanos|keyword|text|double|long|unsigned_long" |Input value. The input can be a single- or multi-valued column or an expression. to_dateperiod |field |"date_period|keyword|text" |Input value. The input is a valid constant date period expression. to_datetime |field |"date|date_nanos|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. to_dbl |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long" |Input value. The input can be a single- or multi-valued column or an expression. @@ -357,6 +361,8 @@ to_bool |Converts an input value to a boolean value. A string value of *tr to_boolean |Converts an input value to a boolean value. A string value of *true* will be case-insensitive converted to the Boolean *true*. For anything else, including the empty string, the function will return *false*. The numerical value of *0* will be converted to *false*, anything else will be converted to *true*. to_cartesianpo|Converts an input value to a `cartesian_point` value. A string will only be successfully converted if it respects the {wikipedia}/Well-known_text_representation_of_geometry[WKT Point] format. to_cartesiansh|Converts an input value to a `cartesian_shape` value. A string will only be successfully converted if it respects the {wikipedia}/Well-known_text_representation_of_geometry[WKT] format. +to_date_nanos |Converts an input to a nanosecond-resolution date value (aka date_nanos). +to_datenanos |Converts an input to a nanosecond-resolution date value (aka date_nanos). to_dateperiod |Converts an input value into a `date_period` value. to_datetime |Converts an input value to a date value. A string will only be successfully converted if it's respecting the format `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`. To convert dates in other formats, use <>. to_dbl |Converts an input value to a double value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the {wikipedia}/Unix_time[Unix epoch], converted to double. Boolean *true* will be converted to double *1.0*, *false* to *0.0*. @@ -489,6 +495,8 @@ to_bool |boolean to_boolean |boolean |false |false |false to_cartesianpo|cartesian_point |false |false |false to_cartesiansh|cartesian_shape |false |false |false +to_date_nanos |date_nanos |false |false |false +to_datenanos |date_nanos |false |false |false to_dateperiod |date_period |false |false |false to_datetime |date |false |false |false to_dbl |double |false |false |false @@ -536,5 +544,5 @@ required_capability: meta meta functions | stats a = count(*), b = count(*), c = count(*) | mv_expand c; a:long | b:long | c:long -119 | 119 | 119 +121 | 121 | 121 ; diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDatetimeEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDatetimeEvaluator.java new file mode 100644 index 0000000000000..e00e7e044ae12 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDatetimeEvaluator.java @@ -0,0 +1,122 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ToDateNanos}. + * This class is generated. Do not edit it. + */ +public final class ToDateNanosFromDatetimeEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public ToDateNanosFromDatetimeEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "ToDateNanosFromDatetime"; + } + + @Override + public Block evalVector(Vector v) { + LongVector vector = (LongVector) v; + int positionCount = v.getPositionCount(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantLongBlockWith(evalValue(vector, 0), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendLong(evalValue(vector, p)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static long evalValue(LongVector container, int index) { + long value = container.getLong(index); + return ToDateNanos.fromDatetime(value); + } + + @Override + public Block evalBlock(Block b) { + LongBlock block = (LongBlock) b; + int positionCount = block.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + long value = evalValue(block, i); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendLong(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static long evalValue(LongBlock container, int index) { + long value = container.getLong(index); + return ToDateNanos.fromDatetime(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public ToDateNanosFromDatetimeEvaluator get(DriverContext context) { + return new ToDateNanosFromDatetimeEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "ToDateNanosFromDatetimeEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDoubleEvaluator.java new file mode 100644 index 0000000000000..23b30e669241b --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDoubleEvaluator.java @@ -0,0 +1,124 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.DoubleVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.InvalidArgumentException; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ToDateNanos}. + * This class is generated. Do not edit it. + */ +public final class ToDateNanosFromDoubleEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public ToDateNanosFromDoubleEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "ToDateNanosFromDouble"; + } + + @Override + public Block evalVector(Vector v) { + DoubleVector vector = (DoubleVector) v; + int positionCount = v.getPositionCount(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantLongBlockWith(evalValue(vector, 0), positionCount); + } catch (IllegalArgumentException | InvalidArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendLong(evalValue(vector, p)); + } catch (IllegalArgumentException | InvalidArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static long evalValue(DoubleVector container, int index) { + double value = container.getDouble(index); + return ToDateNanos.fromDouble(value); + } + + @Override + public Block evalBlock(Block b) { + DoubleBlock block = (DoubleBlock) b; + int positionCount = block.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + long value = evalValue(block, i); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendLong(value); + valuesAppended = true; + } catch (IllegalArgumentException | InvalidArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static long evalValue(DoubleBlock container, int index) { + double value = container.getDouble(index); + return ToDateNanos.fromDouble(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public ToDateNanosFromDoubleEvaluator get(DriverContext context) { + return new ToDateNanosFromDoubleEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "ToDateNanosFromDoubleEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromLongEvaluator.java new file mode 100644 index 0000000000000..cc52208ce5a25 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromLongEvaluator.java @@ -0,0 +1,122 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ToDateNanos}. + * This class is generated. Do not edit it. + */ +public final class ToDateNanosFromLongEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public ToDateNanosFromLongEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "ToDateNanosFromLong"; + } + + @Override + public Block evalVector(Vector v) { + LongVector vector = (LongVector) v; + int positionCount = v.getPositionCount(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantLongBlockWith(evalValue(vector, 0), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendLong(evalValue(vector, p)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static long evalValue(LongVector container, int index) { + long value = container.getLong(index); + return ToDateNanos.fromLong(value); + } + + @Override + public Block evalBlock(Block b) { + LongBlock block = (LongBlock) b; + int positionCount = block.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + long value = evalValue(block, i); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendLong(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static long evalValue(LongBlock container, int index) { + long value = container.getLong(index); + return ToDateNanos.fromLong(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public ToDateNanosFromLongEvaluator get(DriverContext context) { + return new ToDateNanosFromLongEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "ToDateNanosFromLongEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromStringEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromStringEvaluator.java new file mode 100644 index 0000000000000..c5a20ac298da7 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromStringEvaluator.java @@ -0,0 +1,126 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ToDateNanos}. + * This class is generated. Do not edit it. + */ +public final class ToDateNanosFromStringEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public ToDateNanosFromStringEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "ToDateNanosFromString"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantLongBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendLong(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static long evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return ToDateNanos.fromKeyword(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + long value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendLong(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static long evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return ToDateNanos.fromKeyword(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public ToDateNanosFromStringEvaluator get(DriverContext context) { + return new ToDateNanosFromStringEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "ToDateNanosFromStringEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index f714d4d1808c1..f0fa89dedd9ab 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -273,6 +273,11 @@ public enum Cap { */ DATE_NANOS_TYPE(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG), + /** + * Support for to_date_nanos function + */ + TO_DATE_NANOS(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG), + /** * Support CIDRMatch in CombineDisjunctions rule. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index f96742b5a4d91..96ccf8b01e5bc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -43,6 +43,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToBoolean; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianPoint; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianShape; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanos; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatePeriod; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatetime; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDegrees; @@ -349,6 +350,7 @@ private FunctionDefinition[][] functions() { def(ToCartesianShape.class, ToCartesianShape::new, "to_cartesianshape"), def(ToDatePeriod.class, ToDatePeriod::new, "to_dateperiod"), def(ToDatetime.class, ToDatetime::new, "to_datetime", "to_dt"), + def(ToDateNanos.class, ToDateNanos::new, "to_date_nanos", "to_datenanos"), def(ToDegrees.class, ToDegrees::new, "to_degrees"), def(ToDouble.class, ToDouble::new, "to_double", "to_dbl"), def(ToGeoPoint.class, ToGeoPoint::new, "to_geopoint"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java index bdbc9b649c101..4d34033286f52 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToBoolean; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianPoint; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianShape; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanos; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatetime; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDegrees; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble; @@ -107,6 +108,7 @@ public static List getNamedWriteables() { entries.add(ToBoolean.ENTRY); entries.add(ToCartesianPoint.ENTRY); entries.add(ToDatetime.ENTRY); + entries.add(ToDateNanos.ENTRY); entries.add(ToDegrees.ENTRY); entries.add(ToDouble.ENTRY); entries.add(ToGeoShape.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java new file mode 100644 index 0000000000000..9a6a91b7ccedd --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.time.DateFormatters; +import org.elasticsearch.common.time.DateUtils; +import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.xpack.esql.core.InvalidArgumentException; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.DataTypeConverter; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_NANOS_FORMATTER; + +public class ToDateNanos extends AbstractConvertFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "ToDateNanos", + ToDateNanos::new + ); + + private static final Map EVALUATORS = Map.ofEntries( + Map.entry(DATETIME, ToDateNanosFromDatetimeEvaluator.Factory::new), + Map.entry(DATE_NANOS, (field, source) -> field), + Map.entry(LONG, ToDateNanosFromLongEvaluator.Factory::new), + Map.entry(KEYWORD, ToDateNanosFromStringEvaluator.Factory::new), + Map.entry(TEXT, ToDateNanosFromStringEvaluator.Factory::new), + Map.entry(DOUBLE, ToDateNanosFromDoubleEvaluator.Factory::new), + Map.entry(UNSIGNED_LONG, ToLongFromUnsignedLongEvaluator.Factory::new) + /* + NB: not including an integer conversion, because max int in nanoseconds is like 2 seconds after epoch, and it seems more likely + a user who tries to convert an int to a nanosecond date has made a mistake that we should catch that at parse time. + TO_DATE_NANOS(TO_LONG(intVal)) is still possible if someone really needs to do this. + */ + ); + + @FunctionInfo( + returnType = "date_nanos", + description = "Converts an input to a nanosecond-resolution date value (aka date_nanos).", + note = "The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z. Additionally, integers " + + "cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch.", + preview = true + ) + public ToDateNanos( + Source source, + @Param( + name = "field", + type = { "date", "date_nanos", "keyword", "text", "double", "long", "unsigned_long" }, + description = "Input value. The input can be a single- or multi-valued column or an expression." + ) Expression field + ) { + super(source, field); + } + + protected ToDateNanos(StreamInput in) throws IOException { + super(in); + } + + @Override + public DataType dataType() { + return DATE_NANOS; + } + + @Override + protected Map factories() { + return EVALUATORS; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new ToDateNanos(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, ToDateNanos::new, field()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @ConvertEvaluator(extraName = "FromLong", warnExceptions = { IllegalArgumentException.class }) + static long fromLong(long in) { + if (in < 0L) { + throw new IllegalArgumentException("Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported."); + } + return in; + } + + @ConvertEvaluator(extraName = "FromDouble", warnExceptions = { IllegalArgumentException.class, InvalidArgumentException.class }) + static long fromDouble(double in) { + if (in < 0d) { + throw new IllegalArgumentException("Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported."); + } + return DataTypeConverter.safeDoubleToLong(in); + } + + @ConvertEvaluator(extraName = "FromString", warnExceptions = { IllegalArgumentException.class }) + static long fromKeyword(BytesRef in) { + Instant parsed = DateFormatters.from(DEFAULT_DATE_NANOS_FORMATTER.parse(in.utf8ToString())).toInstant(); + return DateUtils.toLong(parsed); + } + + @ConvertEvaluator(extraName = "FromDatetime", warnExceptions = { IllegalArgumentException.class }) + static long fromDatetime(long in) { + return DateUtils.toNanoSeconds(in); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index 0c530bd0eb273..edc3081a33681 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToBoolean; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianPoint; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianShape; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanos; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatePeriod; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatetime; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble; @@ -63,6 +64,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT; import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; @@ -96,6 +98,7 @@ public class EsqlDataTypeConverter { public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time"); + public static final DateFormatter DEFAULT_DATE_NANOS_FORMATTER = DateFormatter.forPattern("strict_date_optional_time_nanos"); public static final DateFormatter HOUR_MINUTE_SECOND = DateFormatter.forPattern("strict_hour_minute_second_fraction"); @@ -104,6 +107,7 @@ public class EsqlDataTypeConverter { entry(CARTESIAN_POINT, ToCartesianPoint::new), entry(CARTESIAN_SHAPE, ToCartesianShape::new), entry(DATETIME, ToDatetime::new), + entry(DATE_NANOS, ToDateNanos::new), // ToDegrees, typeless entry(DOUBLE, ToDouble::new), entry(GEO_POINT, ToGeoPoint::new), @@ -499,7 +503,7 @@ public static String dateTimeToString(long dateTime) { } public static String nanoTimeToString(long dateTime) { - return DateFormatter.forPattern("strict_date_optional_time_nanos").formatNanos(dateTime); + return DEFAULT_DATE_NANOS_FORMATTER.formatNanos(dateTime); } public static String dateTimeToString(long dateTime, DateFormatter formatter) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index e44ea907518b4..b3942a71edadb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -623,6 +623,7 @@ public static void forUnaryBoolean( /** * Generate positive test cases for a unary function operating on an {@link DataType#DATETIME}. + * This variant defaults to maximum range of possible values */ public static void forUnaryDatetime( List suppliers, @@ -641,6 +642,29 @@ public static void forUnaryDatetime( ); } + /** + * Generate positive test cases for a unary function operating on an {@link DataType#DATETIME}. + * This variant accepts a range of values + */ + public static void forUnaryDatetime( + List suppliers, + String expectedEvaluatorToString, + DataType expectedType, + long min, + long max, + Function expectedValue, + List warnings + ) { + unaryNumeric( + suppliers, + expectedEvaluatorToString, + dateCases(min, max), + expectedType, + n -> expectedValue.apply(Instant.ofEpochMilli(n.longValue())), + warnings + ); + } + /** * Generate positive test cases for a unary function operating on an {@link DataType#DATE_NANOS}. */ @@ -1044,26 +1068,45 @@ public static List booleanCases() { *

*/ public static List dateCases() { - return List.of( - new TypedDataSupplier("<1970-01-01T00:00:00Z>", () -> 0L, DataType.DATETIME), - new TypedDataSupplier( - "", - () -> ESTestCase.randomLongBetween(0, 10 * (long) 10e11), // 1970-01-01T00:00:00Z - 2286-11-20T17:46:40Z - DataType.DATETIME - ), - new TypedDataSupplier( - "", - // 2286-11-20T17:46:40Z - +292278994-08-17T07:12:55.807Z - () -> ESTestCase.randomLongBetween(10 * (long) 10e11, Long.MAX_VALUE), - DataType.DATETIME - ), - new TypedDataSupplier( - "", - // very close to +292278994-08-17T07:12:55.807Z, the maximum supported millis since epoch - () -> ESTestCase.randomLongBetween(Long.MAX_VALUE / 100 * 99, Long.MAX_VALUE), - DataType.DATETIME - ) - ); + return dateCases(Long.MIN_VALUE, Long.MAX_VALUE); + } + + /** + * Generate cases for {@link DataType#DATETIME}. + *

+ * For multi-row parameters, see {@link MultiRowTestCaseSupplier#dateCases}. + *

+ */ + public static List dateCases(long min, long max) { + List cases = new ArrayList<>(); + if (min <= 0 && max >= 0) { + cases.add(new TypedDataSupplier("<1970-01-01T00:00:00Z>", () -> 0L, DataType.DATETIME)); + } + + // 1970-01-01T00:00:00Z - 2286-11-20T17:46:40Z + long lower1 = Math.max(min, 0); + long upper1 = Math.min(max, 10 * (long) 10e11); + if (lower1 < upper1) { + cases.add(new TypedDataSupplier("", () -> ESTestCase.randomLongBetween(lower1, upper1), DataType.DATETIME)); + } + + // 2286-11-20T17:46:40Z - +292278994-08-17T07:12:55.807Z + long lower2 = Math.max(min, 10 * (long) 10e11); + long upper2 = Math.min(max, Long.MAX_VALUE); + if (lower2 < upper2) { + cases.add(new TypedDataSupplier("", () -> ESTestCase.randomLongBetween(lower2, upper2), DataType.DATETIME)); + } + + // very close to +292278994-08-17T07:12:55.807Z, the maximum supported millis since epoch + long lower3 = Math.max(min, Long.MAX_VALUE / 100 * 99); + long upper3 = Math.min(max, Long.MAX_VALUE); + if (lower3 < upper3) { + cases.add( + new TypedDataSupplier("", () -> ESTestCase.randomLongBetween(lower3, upper3), DataType.DATETIME) + ); + } + + return cases; } /** diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java new file mode 100644 index 0000000000000..e91a5cc1ebca4 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.common.time.DateUtils; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +public class ToDateNanosTests extends AbstractScalarFunctionTestCase { + public ToDateNanosTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + final String read = "Attribute[channel=0]"; + final List suppliers = new ArrayList<>(); + + TestCaseSupplier.forUnaryDateNanos(suppliers, read, DataType.DATE_NANOS, DateUtils::toLong, List.of()); + TestCaseSupplier.forUnaryDatetime( + suppliers, + "ToDateNanosFromDatetimeEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + 0, + DateUtils.MAX_NANOSECOND_INSTANT.toEpochMilli(), + i -> DateUtils.toNanoSeconds(i.toEpochMilli()), + List.of() + ); + TestCaseSupplier.forUnaryLong( + suppliers, + "ToDateNanosFromLongEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + l -> l, + 0, + Long.MAX_VALUE, + List.of() + ); + TestCaseSupplier.forUnaryLong( + suppliers, + "ToDateNanosFromLongEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + l -> null, + Long.MIN_VALUE, + -1L, + List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.IllegalArgumentException: Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported." + ) + ); + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + "ToLongFromUnsignedLongEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + BigInteger::longValueExact, + BigInteger.ZERO, + BigInteger.valueOf(Long.MAX_VALUE), + List.of() + ); + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + "ToLongFromUnsignedLongEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + bi -> null, + BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.TWO), + UNSIGNED_LONG_MAX, + bi -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + bi + "] out of [long] range" + ) + ); + TestCaseSupplier.forUnaryDouble( + suppliers, + "ToDateNanosFromDoubleEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + d -> null, + Double.NEGATIVE_INFINITY, + -Double.MIN_VALUE, + d -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.IllegalArgumentException: Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported." + ) + ); + TestCaseSupplier.forUnaryDouble( + suppliers, + "ToDateNanosFromDoubleEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + d -> null, + 9.223372036854777E18, // a "convenient" value larger than `(double) Long.MAX_VALUE` (== ...776E18) + Double.POSITIVE_INFINITY, + d -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [long] range" + ) + ); + TestCaseSupplier.forUnaryStrings( + suppliers, + "ToDateNanosFromStringEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + bytesRef -> null, + bytesRef -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.IllegalArgumentException: " + + (bytesRef.utf8ToString().isEmpty() + ? "cannot parse empty datetime" + : ("failed to parse date field [" + bytesRef.utf8ToString() + "] with format [strict_date_optional_time_nanos]")) + ) + ); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + suppliers, + (v, p) -> "date_nanos or datetime or double or long or string or unsigned_long" + ); + } + + @Override + protected Expression build(Source source, List args) { + return new ToDateNanos(source, args.get(0)); + } +} From 3516b9a67529bd9d2509f252c01079bceb3ed20a Mon Sep 17 00:00:00 2001 From: Fang Xing <155562079+fang-xing-esql@users.noreply.github.com> Date: Thu, 26 Sep 2024 12:06:35 -0400 Subject: [PATCH 32/36] [ES|QL] Check expression resolved before checking its data type in ImplicitCasting (#113314) * check resolved before check data type --- docs/changelog/113314.yaml | 6 +++ .../xpack/esql/analysis/Analyzer.java | 37 ++++++++++--------- .../xpack/esql/analysis/VerifierTests.java | 11 ++++++ 3 files changed, 37 insertions(+), 17 deletions(-) create mode 100644 docs/changelog/113314.yaml diff --git a/docs/changelog/113314.yaml b/docs/changelog/113314.yaml new file mode 100644 index 0000000000000..c496ad3dd86f1 --- /dev/null +++ b/docs/changelog/113314.yaml @@ -0,0 +1,6 @@ +pr: 113314 +summary: "[ES|QL] Check expression resolved before checking its data type in `ImplicitCasting`" +area: ES|QL +type: bug +issues: + - 113242 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 9288e1cf81a15..63462d1721f71 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -1070,12 +1070,12 @@ private static Expression processBinaryOperator(BinaryOperator o) { private static Expression processIn(In in) { Expression left = in.value(); List right = in.list(); - DataType targetDataType = left.dataType(); - if (left.resolved() == false || supportsStringImplicitCasting(targetDataType) == false) { + if (left.resolved() == false || supportsStringImplicitCasting(left.dataType()) == false) { return in; } + DataType targetDataType = left.dataType(); List newChildren = new ArrayList<>(right.size() + 1); boolean childrenChanged = false; @@ -1107,23 +1107,26 @@ private static Expression castMixedNumericTypes(EsqlScalarFunction f, DataType t DataType childDataType; for (Expression e : f.children()) { - childDataType = e.dataType(); - if (childDataType.isNumeric() == false - || childDataType == targetNumericType - || canCastNumeric(childDataType, targetNumericType) == false) { + if (e.resolved()) { + childDataType = e.dataType(); + if (childDataType.isNumeric() == false + || childDataType == targetNumericType + || canCastNumeric(childDataType, targetNumericType) == false) { + newChildren.add(e); + continue; + } + childrenChanged = true; + // add a casting function + switch (targetNumericType) { + case INTEGER -> newChildren.add(new ToInteger(e.source(), e)); + case LONG -> newChildren.add(new ToLong(e.source(), e)); + case DOUBLE -> newChildren.add(new ToDouble(e.source(), e)); + case UNSIGNED_LONG -> newChildren.add(new ToUnsignedLong(e.source(), e)); + default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetNumericType); + } + } else { newChildren.add(e); - continue; } - childrenChanged = true; - // add a casting function - switch (targetNumericType) { - case INTEGER -> newChildren.add(new ToInteger(e.source(), e)); - case LONG -> newChildren.add(new ToLong(e.source(), e)); - case DOUBLE -> newChildren.add(new ToDouble(e.source(), e)); - case UNSIGNED_LONG -> newChildren.add(new ToUnsignedLong(e.source(), e)); - default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetNumericType); - } - } return childrenChanged ? f.replaceChildren(newChildren) : f; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 2012e319510af..35e553de61a78 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -283,6 +283,17 @@ public void testImplicitCastingErrorMessages() { "1:42: Cannot convert string [a] to [DOUBLE], error [Cannot parse number [a]]", error("ROW a=[3, 5, 1, 6] | EVAL avg_a = MV_AVG(\"a\")") ); + assertEquals( + "1:19: Unknown column [languages.*], did you mean any of [languages, languages.byte, languages.long, languages.short]?", + error("from test | where `languages.*` in (1, 2)") + ); + assertEquals("1:22: Unknown function [func]", error("from test | eval x = func(languages) | where x in (1, 2)")); + assertEquals( + "1:32: Unknown column [languages.*], did you mean any of [languages, languages.byte, languages.long, languages.short]?", + error("from test | eval x = coalesce( `languages.*`, languages, 0 )") + ); + String error = error("from test | eval x = func(languages) | eval y = coalesce(x, languages, 0 )"); + assertThat(error, containsString("function [func]")); } public void testAggsExpressionsInStatsAggs() { From d8a32157e555edf536cfbfc46cc1ebed0227f7e1 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 26 Sep 2024 10:07:33 -0600 Subject: [PATCH 33/36] Add allowed warnings to the dot-prefix yaml tests (#113560) This adds the missing optional warning from the global legacy template Relates to #113529 (doesn't close it yet until this has been unmuted, which won't happen until this change is backported to 8.16) --- .../resources/rest-api-spec/test/dot_prefix/10_basic.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml b/modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml index b160af4ee8290..ae256daeb8abb 100644 --- a/modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml +++ b/modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml @@ -164,11 +164,13 @@ teardown: --- "Deprecated index template with a dot prefix index pattern": - requires: - test_runner_features: ["warnings", "headers"] + test_runner_features: ["warnings", "headers", "allowed_warnings"] - do: warnings: - "Index [.data-*] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + allowed_warnings: + - "index template [my-template] has index patterns [regular, .data-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" indices.put_index_template: name: my-template body: @@ -177,7 +179,8 @@ teardown: - do: headers: { X-elastic-product-origin: kibana } - warnings: + allowed_warnings: + - "index template [my-template2] has index patterns [other, .data2-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation" indices.put_index_template: name: my-template2 body: From 1e2c19fb0be8f73caec9b7bb693b89d120020127 Mon Sep 17 00:00:00 2001 From: Pat Whelan Date: Thu, 26 Sep 2024 12:18:42 -0400 Subject: [PATCH 34/36] [ML] Add stream flag to inference providers (#113424) Pass the stream flag from the REST request through to the inference providers via the InferenceInputs. Co-authored-by: Elastic Machine --- .../inference/InferenceService.java | 2 ++ .../TestDenseInferenceServiceExtension.java | 1 + .../mock/TestRerankingServiceExtension.java | 1 + .../TestSparseInferenceServiceExtension.java | 1 + ...stStreamingCompletionServiceExtension.java | 1 + .../action/TransportInferenceAction.java | 1 + .../inference/external/http/HttpUtils.java | 2 +- .../http/sender/DocumentsOnlyInput.java | 14 +++++++++++-- .../http/sender/QueryAndDocsInputs.java | 21 +++++++++++++------ .../inference/services/SenderService.java | 5 +++-- .../inference/services/ServiceUtils.java | 1 + .../AlibabaCloudSearchService.java | 1 + .../ElasticsearchInternalService.java | 1 + .../services/elser/ElserInternalService.java | 1 + .../SimpleServiceIntegrationValidator.java | 1 + .../inference/services/ServiceUtilsTests.java | 21 ++++++++----------- .../AmazonBedrockServiceTests.java | 4 ++++ .../anthropic/AnthropicServiceTests.java | 2 ++ .../AzureAiStudioServiceTests.java | 3 +++ .../azureopenai/AzureOpenAiServiceTests.java | 3 +++ .../services/cohere/CohereServiceTests.java | 6 ++++++ .../elastic/ElasticInferenceServiceTests.java | 2 ++ .../GoogleAiStudioServiceTests.java | 4 ++++ .../HuggingFaceBaseServiceTests.java | 1 + .../huggingface/HuggingFaceServiceTests.java | 2 ++ .../ibmwatsonx/IbmWatsonxServiceTests.java | 3 +++ .../services/mistral/MistralServiceTests.java | 2 ++ .../services/openai/OpenAiServiceTests.java | 3 +++ ...impleServiceIntegrationValidatorTests.java | 5 ++++- 29 files changed, 91 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceService.java b/server/src/main/java/org/elasticsearch/inference/InferenceService.java index f677f75dfb5ae..854c58b4f57ad 100644 --- a/server/src/main/java/org/elasticsearch/inference/InferenceService.java +++ b/server/src/main/java/org/elasticsearch/inference/InferenceService.java @@ -85,6 +85,7 @@ void parseRequestConfig( * @param model The model * @param query Inference query, mainly for re-ranking * @param input Inference input + * @param stream Stream inference results * @param taskSettings Settings in the request to override the model's defaults * @param inputType For search, ingest etc * @param timeout The timeout for the request @@ -94,6 +95,7 @@ void infer( Model model, @Nullable String query, List input, + boolean stream, Map taskSettings, InputType inputType, TimeValue timeout, diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java index 10d8f90efef5b..daa29d33699ef 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java @@ -94,6 +94,7 @@ public void infer( Model model, @Nullable String query, List input, + boolean stream, Map taskSettings, InputType inputType, TimeValue timeout, diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java index fae11d5b53ca3..1894db6db8df6 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java @@ -85,6 +85,7 @@ public void infer( Model model, @Nullable String query, List input, + boolean stream, Map taskSettings, InputType inputType, TimeValue timeout, diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java index fee9855b188c2..1a5df146a3aa4 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java @@ -88,6 +88,7 @@ public void infer( Model model, @Nullable String query, List input, + boolean stream, Map taskSettings, InputType inputType, TimeValue timeout, diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java index 3d72b1f2729b0..4313026e92521 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java @@ -85,6 +85,7 @@ public void infer( Model model, String query, List input, + boolean stream, Map taskSettings, InputType inputType, TimeValue timeout, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java index 803e8f1e07612..4186b281a35b5 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java @@ -114,6 +114,7 @@ private void inferOnService( model, request.getQuery(), request.getInput(), + request.isStreaming(), request.getTaskSettings(), request.getInputType(), request.getInferenceTimeout(), diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpUtils.java index 9f2ceddc92a2e..4282e5d1e7cb9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpUtils.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpUtils.java @@ -46,7 +46,7 @@ private static String getStatusCodeErrorMessage(Request request, HttpResult resu } public static void checkForEmptyBody(ThrottlerManager throttlerManager, Logger logger, Request request, HttpResult result) { - if (result.isBodyEmpty()) { + if (result.isBodyEmpty() && (request.isStreaming() == false)) { String message = format("Response body was empty for request from inference entity id [%s]", request.getInferenceEntityId()); throttlerManager.warn(logger, message); throw new IllegalStateException(message); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/DocumentsOnlyInput.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/DocumentsOnlyInput.java index a32e2018117f8..8cf411d84c932 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/DocumentsOnlyInput.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/DocumentsOnlyInput.java @@ -21,13 +21,23 @@ public static DocumentsOnlyInput of(InferenceInputs inferenceInputs) { } private final List input; + private final boolean stream; - public DocumentsOnlyInput(List chunks) { + public DocumentsOnlyInput(List input) { + this(input, false); + } + + public DocumentsOnlyInput(List input, boolean stream) { super(); - this.input = Objects.requireNonNull(chunks); + this.input = Objects.requireNonNull(input); + this.stream = stream; } public List getInputs() { return this.input; } + + public boolean stream() { + return stream; + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/QueryAndDocsInputs.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/QueryAndDocsInputs.java index 0d5f98c180ba9..50bb77b307db3 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/QueryAndDocsInputs.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/QueryAndDocsInputs.java @@ -21,6 +21,19 @@ public static QueryAndDocsInputs of(InferenceInputs inferenceInputs) { } private final String query; + private final List chunks; + private final boolean stream; + + public QueryAndDocsInputs(String query, List chunks) { + this(query, chunks, false); + } + + public QueryAndDocsInputs(String query, List chunks, boolean stream) { + super(); + this.query = Objects.requireNonNull(query); + this.chunks = Objects.requireNonNull(chunks); + this.stream = stream; + } public String getQuery() { return query; @@ -30,12 +43,8 @@ public List getChunks() { return chunks; } - List chunks; - - public QueryAndDocsInputs(String query, List chunks) { - super(); - this.query = Objects.requireNonNull(query); - this.chunks = Objects.requireNonNull(chunks); + public boolean stream() { + return stream; } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java index 864aebcef124f..21b2df6af1ab6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java @@ -51,6 +51,7 @@ public void infer( Model model, @Nullable String query, List input, + boolean stream, Map taskSettings, InputType inputType, TimeValue timeout, @@ -58,9 +59,9 @@ public void infer( ) { init(); if (query != null) { - doInfer(model, new QueryAndDocsInputs(query, input), taskSettings, inputType, timeout, listener); + doInfer(model, new QueryAndDocsInputs(query, input, stream), taskSettings, inputType, timeout, listener); } else { - doInfer(model, new DocumentsOnlyInput(input), taskSettings, inputType, timeout, listener); + doInfer(model, new DocumentsOnlyInput(input, stream), taskSettings, inputType, timeout, listener); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java index 6eb0331913009..32c1d17373e53 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java @@ -666,6 +666,7 @@ public static void getEmbeddingSize(Model model, InferenceService service, Actio model, null, List.of(TEST_EMBEDDING_INPUT), + false, Map.of(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java index 8f0c9896c6642..994bad194aef6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java @@ -309,6 +309,7 @@ private void checkAlibabaCloudSearchServiceConfig(Model model, InferenceService model, query, List.of(input), + false, Map.of(), InputType.INGEST, DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java index cca8ae63e974c..93408c067098b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java @@ -323,6 +323,7 @@ public void infer( Model model, @Nullable String query, List input, + boolean stream, Map taskSettings, InputType inputType, TimeValue timeout, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java index 948117954a63f..746cb6e89fad0 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java @@ -149,6 +149,7 @@ public void infer( Model model, @Nullable String query, List inputs, + boolean stream, Map taskSettings, InputType inputType, TimeValue timeout, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidator.java index 6233a7e0b6b29..70f01e77b9369 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidator.java @@ -31,6 +31,7 @@ public void validate(InferenceService service, Model model, ActionListener { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[6]; + ActionListener listener = invocation.getArgument(7); listener.onResponse(new InferenceTextEmbeddingFloatResults(List.of())); return Void.TYPE; - }).when(service).infer(any(), any(), any(), any(), any(), any(), any()); + }).when(service).infer(any(), any(), any(), anyBoolean(), any(), any(), any(), any()); PlainActionFuture listener = new PlainActionFuture<>(); getEmbeddingSize(model, service, listener); @@ -878,12 +878,11 @@ public void testGetEmbeddingSize_ReturnsError_WhenTextEmbeddingByteResults_IsEmp when(model.getTaskType()).thenReturn(TaskType.TEXT_EMBEDDING); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[6]; + ActionListener listener = invocation.getArgument(7); listener.onResponse(new InferenceTextEmbeddingByteResults(List.of())); return Void.TYPE; - }).when(service).infer(any(), any(), any(), any(), any(), any(), any()); + }).when(service).infer(any(), any(), any(), anyBoolean(), any(), any(), any(), any()); PlainActionFuture listener = new PlainActionFuture<>(); getEmbeddingSize(model, service, listener); @@ -903,12 +902,11 @@ public void testGetEmbeddingSize_ReturnsSize_ForTextEmbeddingResults() { var textEmbedding = TextEmbeddingResultsTests.createRandomResults(); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[6]; + ActionListener listener = invocation.getArgument(7); listener.onResponse(textEmbedding); return Void.TYPE; - }).when(service).infer(any(), any(), any(), any(), any(), any(), any()); + }).when(service).infer(any(), any(), any(), anyBoolean(), any(), any(), any(), any()); PlainActionFuture listener = new PlainActionFuture<>(); getEmbeddingSize(model, service, listener); @@ -927,12 +925,11 @@ public void testGetEmbeddingSize_ReturnsSize_ForTextEmbeddingByteResults() { var textEmbedding = InferenceTextEmbeddingByteResultsTests.createRandomResults(); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[6]; + ActionListener listener = invocation.getArgument(7); listener.onResponse(textEmbedding); return Void.TYPE; - }).when(service).infer(any(), any(), any(), any(), any(), any(), any()); + }).when(service).infer(any(), any(), any(), anyBoolean(), any(), any(), any(), any()); PlainActionFuture listener = new PlainActionFuture<>(); getEmbeddingSize(model, service, listener); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java index bbf34354e1818..297a42f9d1fa7 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java @@ -671,6 +671,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotAmazonBedrockModel() throws IOExc mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -721,6 +722,7 @@ public void testInfer_SendsRequest_ForEmbeddingsModel() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -762,6 +764,7 @@ public void testInfer_SendsRequest_ForChatCompletionModel() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -1025,6 +1028,7 @@ public void testInfer_UnauthorizedResponse() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java index 5e32344ab3840..c3693c227c435 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java @@ -452,6 +452,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotAValidModel() throws IOException mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -506,6 +507,7 @@ public void testInfer_SendsCompletionRequest() throws IOException { model, null, List.of("input"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java index 6f33c36f42db4..bb736f592fbdb 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java @@ -825,6 +825,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotAzureAiStudioModel() throws IOExc mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -954,6 +955,7 @@ public void testInfer_WithChatCompletionModel() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -1004,6 +1006,7 @@ public void testInfer_UnauthorisedResponse() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java index b3fbd6fc9b425..142877c09180f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java @@ -601,6 +601,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotAzureOpenAiModel() throws IOExcep mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -656,6 +657,7 @@ public void testInfer_SendsRequest() throws IOException, URISyntaxException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -1051,6 +1053,7 @@ public void testInfer_UnauthorisedResponse() throws IOException, URISyntaxExcept model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java index aebc3e3776c40..a577a6664d39d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java @@ -622,6 +622,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotCohereModel() throws IOException mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -689,6 +690,7 @@ public void testInfer_SendsRequest() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -932,6 +934,7 @@ public void testInfer_UnauthorisedResponse() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -991,6 +994,7 @@ public void testInfer_SetsInputTypeToIngest_FromInferParameter_WhenTaskSettingsA model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -1064,6 +1068,7 @@ public void testInfer_SetsInputTypeToIngestFromInferParameter_WhenModelSettingIs model, null, List.of("abc"), + false, CohereEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.SEARCH, null), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -1135,6 +1140,7 @@ public void testInfer_DoesNotSetInputType_WhenNotPresentInTaskSettings_AndUnspec model, null, List.of("abc"), + false, new HashMap<>(), InputType.UNSPECIFIED, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java index 38124b3401aaa..0bbf2be7301d8 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java @@ -346,6 +346,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotAValidModel() throws IOException mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -397,6 +398,7 @@ public void testInfer_SendsEmbeddingsRequest() throws IOException { model, null, List.of("input text"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java index 89d6a010bbc07..5d79d0e01f401 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java @@ -503,6 +503,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotGoogleAiStudioModel() throws IOEx mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -578,6 +579,7 @@ public void testInfer_SendsCompletionRequest() throws IOException { model, null, List.of("input"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -634,6 +636,7 @@ public void testInfer_SendsEmbeddingsRequest() throws IOException { model, null, List.of(input), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -775,6 +778,7 @@ public void testInfer_ResourceNotFound() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseServiceTests.java index 22c3b7895460a..168110ae8f7c7 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseServiceTests.java @@ -69,6 +69,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotHuggingFaceModel() throws IOExcep mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java index 5ea9f82e5b60c..d13dea2ab6b4c 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java @@ -438,6 +438,7 @@ public void testInfer_SendsEmbeddingsRequest() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -481,6 +482,7 @@ public void testInfer_SendsElserRequest() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java index e0936c778c7a7..a2de7c15d54da 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java @@ -409,6 +409,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotIbmWatsonxModel() throws IOExcept mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -465,6 +466,7 @@ public void testInfer_SendsEmbeddingsRequest() throws IOException { model, null, List.of(input), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -588,6 +590,7 @@ public void testInfer_ResourceNotFound() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java index 9d0fd954c44f9..33a2b43caf174 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java @@ -446,6 +446,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotMistralEmbeddingsModel() throws I mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -571,6 +572,7 @@ public void testInfer_UnauthorisedResponse() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java index a5e8c1d7eb26e..32099c4bd0be9 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java @@ -936,6 +936,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotOpenAiModel() throws IOException mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -990,6 +991,7 @@ public void testInfer_SendsRequest() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -1470,6 +1472,7 @@ public void testInfer_UnauthorisedResponse() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java index ef295e4070cc3..767dd4d64a7d3 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java @@ -64,6 +64,7 @@ public void testValidate_ServiceThrowsException() { eq(mockModel), eq(null), eq(TEST_INPUT), + eq(false), eq(Map.of()), eq(InputType.INGEST), eq(InferenceAction.Request.DEFAULT_TIMEOUT), @@ -94,7 +95,7 @@ public void testValidate_SuccessfulCallToServiceForReRankTaskType() { private void mockSuccessfulCallToService(String query, InferenceServiceResults result) { doAnswer(ans -> { - ActionListener responseListener = ans.getArgument(6); + ActionListener responseListener = ans.getArgument(7); responseListener.onResponse(result); return null; }).when(mockInferenceService) @@ -102,6 +103,7 @@ private void mockSuccessfulCallToService(String query, InferenceServiceResults r eq(mockModel), eq(query), eq(TEST_INPUT), + eq(false), eq(Map.of()), eq(InputType.INGEST), eq(InferenceAction.Request.DEFAULT_TIMEOUT), @@ -117,6 +119,7 @@ private void verifyCallToService(boolean withQuery) { eq(mockModel), eq(withQuery ? TEST_QUERY : null), eq(TEST_INPUT), + eq(false), eq(Map.of()), eq(InputType.INGEST), eq(InferenceAction.Request.DEFAULT_TIMEOUT), From f412de7e8ec93631b224e7d350907852e4433121 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 26 Sep 2024 17:32:17 +0100 Subject: [PATCH 35/36] Handle rejections in `IncrementalBulkIT` (#113599) Submitting a bare runnable to a threadpool risks an exception being thrown if the queue is full. This commit moves to submitting `AbstractRunnable` instances that won't be rejected. Closes #113365 --- muted-tests.yml | 3 - .../action/bulk/IncrementalBulkIT.java | 91 ++++++++++++------- 2 files changed, 60 insertions(+), 34 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 5d7474af06d86..7a21957a044c0 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -257,9 +257,6 @@ tests: - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/ccr/apis/follow/post-resume-follow/line_84} issue: https://github.com/elastic/elasticsearch/issues/113343 -- class: org.elasticsearch.action.bulk.IncrementalBulkIT - method: testBulkLevelBulkFailureAfterFirstIncrementalRequest - issue: https://github.com/elastic/elasticsearch/issues/113365 - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testDeleteJob_TimingStatsDocumentIsDeleted issue: https://github.com/elastic/elasticsearch/issues/113370 diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java index d7a5d4e2ac973..75f914f76dd77 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.Releasable; @@ -37,6 +38,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; @@ -214,14 +216,8 @@ public void testGlobalBulkFailure() throws InterruptedException { IncrementalBulkService incrementalBulkService = internalCluster().getInstance(IncrementalBulkService.class, randomNodeName); ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, randomNodeName); - int threadCount = threadPool.info(ThreadPool.Names.WRITE).getMax(); - long queueSize = threadPool.info(ThreadPool.Names.WRITE).getQueueSize().singles(); - blockWritePool(threadCount, threadPool, blockingLatch); - - Runnable runnable = () -> {}; - for (int i = 0; i < queueSize; i++) { - threadPool.executor(ThreadPool.Names.WRITE).execute(runnable); - } + blockWritePool(threadPool, blockingLatch); + fillWriteQueue(threadPool); IncrementalBulkService.Handler handler = incrementalBulkService.newBulkRequest(); if (randomBoolean()) { @@ -253,35 +249,32 @@ public void testBulkLevelBulkFailureAfterFirstIncrementalRequest() throws Except AbstractRefCounted refCounted = AbstractRefCounted.of(() -> {}); PlainActionFuture future = new PlainActionFuture<>(); - int threadCount = threadPool.info(ThreadPool.Names.WRITE).getMax(); - long queueSize = threadPool.info(ThreadPool.Names.WRITE).getQueueSize().singles(); - CountDownLatch blockingLatch1 = new CountDownLatch(1); AtomicBoolean nextRequested = new AtomicBoolean(true); AtomicLong hits = new AtomicLong(0); - try (Releasable ignored2 = blockingLatch1::countDown;) { - blockWritePool(threadCount, threadPool, blockingLatch1); + try { + blockWritePool(threadPool, blockingLatch1); while (nextRequested.get()) { nextRequested.set(false); refCounted.incRef(); handler.addItems(List.of(indexRequest(index)), refCounted::decRef, () -> nextRequested.set(true)); hits.incrementAndGet(); } + } finally { + blockingLatch1.countDown(); } assertBusy(() -> assertTrue(nextRequested.get())); CountDownLatch blockingLatch2 = new CountDownLatch(1); - try (Releasable ignored3 = blockingLatch2::countDown;) { - blockWritePool(threadCount, threadPool, blockingLatch2); - Runnable runnable = () -> {}; - // Fill Queue - for (int i = 0; i < queueSize; i++) { - threadPool.executor(ThreadPool.Names.WRITE).execute(runnable); - } + try { + blockWritePool(threadPool, blockingLatch2); + fillWriteQueue(threadPool); handler.lastItems(List.of(indexRequest(index)), refCounted::decRef, future); + } finally { + blockingLatch2.countDown(); } // Should not throw because some succeeded @@ -459,19 +452,55 @@ public void testShortCircuitShardLevelFailureWithIngestNodeHop() throws Exceptio } } - private static void blockWritePool(int threadCount, ThreadPool threadPool, CountDownLatch blockingLatch) throws InterruptedException { - CountDownLatch startedLatch = new CountDownLatch(threadCount); + private static void blockWritePool(ThreadPool threadPool, CountDownLatch finishLatch) { + final var threadCount = threadPool.info(ThreadPool.Names.WRITE).getMax(); + final var startBarrier = new CyclicBarrier(threadCount + 1); + final var blockingTask = new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + fail(e); + } + + @Override + protected void doRun() { + safeAwait(startBarrier); + safeAwait(finishLatch); + } + + @Override + public boolean isForceExecution() { + return true; + } + }; for (int i = 0; i < threadCount; i++) { - threadPool.executor(ThreadPool.Names.WRITE).execute(() -> { - startedLatch.countDown(); - try { - blockingLatch.await(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - }); + threadPool.executor(ThreadPool.Names.WRITE).execute(blockingTask); + } + safeAwait(startBarrier); + } + + private static void fillWriteQueue(ThreadPool threadPool) { + final var queueSize = Math.toIntExact(threadPool.info(ThreadPool.Names.WRITE).getQueueSize().singles()); + final var queueFilled = new AtomicBoolean(false); + final var queueFillingTask = new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + fail(e); + } + + @Override + protected void doRun() { + assertTrue("thread pool not blocked", queueFilled.get()); + } + + @Override + public boolean isForceExecution() { + return true; + } + }; + for (int i = 0; i < queueSize; i++) { + threadPool.executor(ThreadPool.Names.WRITE).execute(queueFillingTask); } - startedLatch.await(); + queueFilled.set(true); } private BulkResponse executeBulk(long docs, String index, IncrementalBulkService.Handler handler, ExecutorService executorService) { From 80cc765090e4874a0f420d2b0026c801bd7554cb Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 27 Sep 2024 02:56:06 +1000 Subject: [PATCH 36/36] Mute org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT test {date_nanos.Date_nanos to date nanos, index version SYNC} #113632 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 7a21957a044c0..df82cda4277b0 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -302,6 +302,9 @@ tests: - class: org.elasticsearch.integration.KibanaUserRoleIntegTests method: testSearchAndMSearch issue: https://github.com/elastic/elasticsearch/issues/113593 +- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT + method: test {date_nanos.Date_nanos to date nanos, index version SYNC} + issue: https://github.com/elastic/elasticsearch/issues/113632 # Examples: #